From 25aa911c5805aa228a145d1d7029dc5457092e10 Mon Sep 17 00:00:00 2001 From: rockymadden Date: Fri, 4 Sep 2026 11:10:56 -0600 Subject: [PATCH] fix(tests): report a broken tenant as ERROR, not as an agent failure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On 2026-09-04 two flow tasks were scored FAILURE for reasons that had nothing to do with the agent: skill-flow-outlook-trigger-inbox AADSTS50173 — the Outlook grant was revoked on 2026-08-31 skill-flow-generic-dynamic-node the ServiceNow developer instance was hibernating, so every metadata call 403'd Both were root-caused as skill defects first. The 403 is several thousand characters into the checker output, and nothing above it says the tenant is down, so the reports read as "the agent wrote a display name where an ID belongs" — which it did, but only because the lookup it was told to use could not answer. A `pre_run` failure lands the run as FinalStatus.ERROR rather than FAILURE, and PreRunCommand's own docstring gives the reason: "the agent should not run against a broken environment." This uses that. preflight_connections.py takes connector keys, lists their connections with --all-folders (without it an empty result is a false negative), and exits non-zero unless at least one is Enabled. Wired into the two tasks above. Behaviour verified against six tenant states: all-enabled, hibernating, mixed with one up, no connections in any folder, CLI non-zero, and a Failure envelope. Only those two tasks are wired. 23 others name a connector, but that list comes from grepping prompts for `uipath-*` and it catches non-connectors like uipath-rpa and uipath-platform — worth doing deliberately, not from a crude match. Co-Authored-By: Claude Opus 5 (1M context) --- .../_shared/preflight_connections.py | 77 +++++++++++++++++++ .../generic_dynamic_node.yaml | 4 + .../outlook_trigger_inbox.yaml | 4 + 3 files changed, 85 insertions(+) create mode 100644 tests/tasks/uipath-maestro-flow/_shared/preflight_connections.py diff --git a/tests/tasks/uipath-maestro-flow/_shared/preflight_connections.py b/tests/tasks/uipath-maestro-flow/_shared/preflight_connections.py new file mode 100644 index 0000000000..99e5ca7132 --- /dev/null +++ b/tests/tasks/uipath-maestro-flow/_shared/preflight_connections.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +"""Fail a task as ERROR, not FAILURE, when the tenant connection it needs is down. + +Usage: + preflight_connections.py [ ...] + +A `pre_run` failure lands the run as ``FinalStatus.ERROR``; a criterion failure +lands it as ``FAILURE``. Without this, a revoked grant or an asleep tenant reads +as an agent mistake: + + skill-flow-outlook-trigger-inbox AADSTS50173, grant revoked 2026-08-31 + skill-flow-generic-dynamic-node ServiceNow developer instance hibernating + +Both were scored FAILURE on 2026-09-04 and root-caused as skill defects before +anyone read far enough into the checker output to find the 403. + +Passes when at least one connection for each key reports Enabled. Connections +live in several folders, so `--all-folders` is required; without it an empty +result is a false negative. +""" + +from __future__ import annotations + +import json +import subprocess +import sys + + +def _connections(key: str) -> list[dict]: + proc = subprocess.run( + ["uip", "is", "connections", "list", key, "--all-folders", "--output", "json"], + capture_output=True, + text=True, + timeout=90, + ) + if proc.returncode != 0: + raise RuntimeError(f"`uip is connections list {key}` exited {proc.returncode}: {proc.stderr.strip()}") + payload = json.loads(proc.stdout) + if payload.get("Result") != "Success": + raise RuntimeError(f"connections list for {key} failed: {payload.get('Message', payload)}") + return payload.get("Data") or [] + + +def main(keys: list[str]) -> int: + broken: list[str] = [] + for key in keys: + try: + conns = _connections(key) + except Exception as exc: # noqa: BLE001 — any failure here is a blocked tenant + broken.append(f"{key}: {exc}") + continue + if not conns: + broken.append(f"{key}: no connection in any folder") + continue + enabled = [c for c in conns if c.get("State") == "Enabled"] + if not enabled: + states = ", ".join(f"{c.get('Name')}={c.get('State')}" for c in conns) + broken.append(f"{key}: no Enabled connection ({states})") + continue + print(f"OK: {key} — {len(enabled)}/{len(conns)} connection(s) Enabled") + + if broken: + print( + "TENANT NOT READY — this is an environment failure, not an agent failure.\n " + + "\n ".join(broken) + + "\n\nReauthorize the connection, or wake the provider instance, then re-run.", + file=sys.stderr, + ) + return 1 + return 0 + + +if __name__ == "__main__": + if len(sys.argv) < 2: + print(__doc__, file=sys.stderr) + sys.exit(2) + sys.exit(main(sys.argv[1:])) diff --git a/tests/tasks/uipath-maestro-flow/connector_features/generic_dynamic_node/generic_dynamic_node.yaml b/tests/tasks/uipath-maestro-flow/connector_features/generic_dynamic_node/generic_dynamic_node.yaml index 23a701a596..9816a70d59 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/generic_dynamic_node/generic_dynamic_node.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_features/generic_dynamic_node/generic_dynamic_node.yaml @@ -52,6 +52,10 @@ initial_prompt: | up; only then stop on that field rather than inventing a value. Record every decision, assumption, and blocked step in your final response. Instructions in the task take precedence over this paragraph. +pre_run: + - command: 'python3 "$SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/preflight_connections.py" uipath-servicenow-servicenow' + timeout: 120 + success_criteria: # ── Flow file validity ───────────────────────────────────────────────── - type: run_command diff --git a/tests/tasks/uipath-maestro-flow/single_node/outlook_trigger_inbox/outlook_trigger_inbox.yaml b/tests/tasks/uipath-maestro-flow/single_node/outlook_trigger_inbox/outlook_trigger_inbox.yaml index c9bb6ced57..567936b1b4 100644 --- a/tests/tasks/uipath-maestro-flow/single_node/outlook_trigger_inbox/outlook_trigger_inbox.yaml +++ b/tests/tasks/uipath-maestro-flow/single_node/outlook_trigger_inbox/outlook_trigger_inbox.yaml @@ -41,6 +41,10 @@ initial_prompt: | up; only then stop on that field rather than inventing a value. Record every decision, assumption, and blocked step in your final response. Instructions in the task take precedence over this paragraph. +pre_run: + - command: 'python3 "$SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/preflight_connections.py" uipath-microsoft-outlook365' + timeout: 120 + success_criteria: # ── Structural: flow builds and validates ────────────────────────────── - type: run_command