From e86cfc5c92a237abd2c5b7d64b0f74144c461312 Mon Sep 17 00:00:00 2001 From: Tao Ma <51425734+tmatup@users.noreply.github.com> Date: Fri, 21 Aug 2026 13:09:51 -0700 Subject: [PATCH 01/52] test(maestro-flow): add arm-neutral flow discovery (#2745) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ๐Ÿค– Generated with Claude Code Co-Authored-By: [Claude](mailto:noreply@anthropic.com) --- .../uipath-maestro-flow/_shared/flow_check.py | 92 ++++++- .../_shared/flow_contains.py | 13 +- .../_shared/test_flow_check_discovery.py | 98 ++++++- .../_shared/test_flow_contains.py | 16 ++ .../_shared/test_same_ground_advisories.py | 21 ++ .../_shared/test_same_ground_corpus.py | 260 ++++++++++++++++++ .../_shared/validate_flow.py | 24 +- .../bindings/check_bindings.py | 12 +- .../e2e/check_devcon_expense_approval.py | 16 +- .../e2e/test_check_devcon_expense_approval.py | 39 +++ 10 files changed, 551 insertions(+), 40 deletions(-) create mode 100644 tests/tasks/uipath-maestro-flow/_shared/test_same_ground_corpus.py diff --git a/tests/tasks/uipath-maestro-flow/_shared/flow_check.py b/tests/tasks/uipath-maestro-flow/_shared/flow_check.py index ba3024e26c..f1e530c8dc 100644 --- a/tests/tasks/uipath-maestro-flow/_shared/flow_check.py +++ b/tests/tasks/uipath-maestro-flow/_shared/flow_check.py @@ -34,6 +34,7 @@ from __future__ import annotations import glob +import hashlib import json import math import os @@ -524,8 +525,7 @@ def assert_loop_body_nodes_parented( loop's ID. Without ``parentId``, the runtime executes the node outside the loop context โ€” per-iteration variables like ``currentItem`` are inaccessible and outputs come back null.""" - project_dir = _find_project(project_glob) - for path in glob.glob(os.path.join(project_dir, "**/*.flow"), recursive=True): + for path in find_flow_files(project_glob): with open(path) as f: flow = json.load(f) nodes_by_id = {n["id"]: n for n in flow.get("nodes") or []} @@ -1332,12 +1332,10 @@ def node_output_leaves(payload: dict, node_ids) -> set: def _load_flow(project_glob: str) -> dict: - """Load the single .flow next to the matched project.uiproj.""" - proj = _find_project(project_glob) - flows = glob.glob(os.path.join(proj, "*.flow")) - if not flows: - _fail(f"no .flow file under {proj}") - return json.loads(open(flows[0], encoding="utf-8").read()) + """Load the single discovered flow artifact.""" + path = find_flow_file(project_glob) + with open(path, encoding="utf-8") as f: + return json.load(f) def assert_decision_branches_reach( @@ -1520,6 +1518,64 @@ def find_project_dir(pattern: str = "**/project.uiproj") -> str: return _find_project(pattern) +def find_flow_files( + project_pattern: str = "**/project.uiproj", *, flow_glob: str = "*.flow" +) -> list[str]: + """Return flow artifacts from the selected solution, or one root emit. + + A Flow solution project wins whenever one exists. This keeps a root-level + SDK scratch file from competing with the linked solution copy. + When no project exists, validate and source-structure checks may consume a + root-level SDK emit. Multiple genuinely distinct root emits are ambiguous + and fail with their paths rather than selecting one by glob order. + + Debug callers intentionally do not use this helper: ``uip maestro flow + debug`` is project-scoped and must continue through :func:`find_project_dir`. + """ + project_candidates = sorted(glob.glob(project_pattern, recursive=True)) + if any(_is_flow_project(path) for path in project_candidates): + project_dir = _find_project(project_pattern) + matches = sorted( + glob.glob( + os.path.join(project_dir, "**", os.path.basename(flow_glob)), + recursive=True, + ) + ) + if not matches: + _fail( + f"No .flow file matching {flow_glob!r} under selected Flow " + f"project {project_dir}" + ) + return matches + + root_matches = _dedupe_flow_candidates( + sorted(glob.glob(os.path.basename(flow_glob))) + ) + if not root_matches: + _fail( + f"No Flow project found matching {project_pattern!r}, and no " + f"root-level .flow file matching {os.path.basename(flow_glob)!r}" + ) + if len(root_matches) > 1: + joined = "\n - ".join(root_matches) + _fail( + "Multiple distinct root-level .flow files found โ€” refusing to " + f"guess:\n - {joined}" + ) + return root_matches + + +def find_flow_file( + project_pattern: str = "**/project.uiproj", *, flow_glob: str = "*.flow" +) -> str: + """Return one unambiguous flow artifact using :func:`find_flow_files`.""" + matches = find_flow_files(project_pattern, flow_glob=flow_glob) + if len(matches) > 1: + joined = "\n - ".join(matches) + _fail(f"Multiple .flow files match {flow_glob!r}:\n - {joined}") + return matches[0] + + # โ”€โ”€ Internals โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ @@ -1561,8 +1617,7 @@ def _get_ci(mapping: Any, *candidate_keys: str, default: Any = None) -> Any: def _iter_flow_nodes(project_glob: str): - project_dir = _find_project(project_glob) - for path in glob.glob(os.path.join(project_dir, "**/*.flow"), recursive=True): + for path in find_flow_files(project_glob): with open(path) as f: flow = json.load(f) yield from flow.get("nodes") or [] @@ -1579,6 +1634,23 @@ def _non_empty_binding_value(value: Any) -> bool: _HUSK_MAX_NODES = 1 +def _dedupe_flow_candidates(paths: list[str]) -> list[str]: + """Collapse aliases and byte-identical copies before ambiguity checks.""" + by_realpath: dict[str, str] = {} + for path in paths: + by_realpath.setdefault(os.path.realpath(path), path) + + by_content: dict[str, str] = {} + for path in by_realpath.values(): + try: + with open(path, "rb") as f: + key = hashlib.sha256(f.read()).hexdigest() + except OSError: + key = f"unreadable:{path}" + by_content.setdefault(key, path) + return sorted(by_content.values()) + + def _find_project(pattern: str) -> str: """Locate the *Flow* project directory matching ``pattern``. diff --git a/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py b/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py index 466293418a..df70f31014 100644 --- a/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py +++ b/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py @@ -41,13 +41,12 @@ from __future__ import annotations -import glob import os import re import sys sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -from flow_check import find_project_dir # noqa: E402 +from flow_check import find_flow_files # noqa: E402 def _parse(argv: list[str]): @@ -75,11 +74,11 @@ def _parse(argv: list[str]): def main(argv: list[str]) -> int: substrings, regexes, absent, flow_name = _parse(argv) - project_dir = find_project_dir() - flows = sorted(glob.glob(os.path.join(project_dir, "**/*.flow"), recursive=True)) - if not flows: - print(f"FAIL: No .flow file found under {project_dir}", file=sys.stderr) - return 1 + try: + flows = find_flow_files() + except SystemExit as exc: + print(str(exc), file=sys.stderr) + return exc.code if isinstance(exc.code, int) else 1 print(f"Found {len(flows)} .flow file(s): {', '.join(flows)}") if flow_name is not None: diff --git a/tests/tasks/uipath-maestro-flow/_shared/test_flow_check_discovery.py b/tests/tasks/uipath-maestro-flow/_shared/test_flow_check_discovery.py index b685da55b0..6beace45ea 100644 --- a/tests/tasks/uipath-maestro-flow/_shared/test_flow_check_discovery.py +++ b/tests/tasks/uipath-maestro-flow/_shared/test_flow_check_discovery.py @@ -12,13 +12,20 @@ import json import os +import subprocess import sys import pytest sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -from flow_check import _find_project # noqa: E402 +from flow_check import ( # noqa: E402 + _find_project, + find_flow_file, + find_flow_files, + find_project_dir, +) +import validate_flow # noqa: E402 PATTERN = "**/project.uiproj" @@ -143,3 +150,92 @@ def test_single_husk_project_is_still_selected(tmp_path, monkeypatch): _make_flow_project(tmp_path, "OnlySolution", "OnlyBuild", 1) assert _find_project(PATTERN) == os.path.join("OnlySolution", "OnlyBuild") + + +def test_structure_discovery_falls_back_to_one_root_flow(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + (tmp_path / "BareEmit.flow").write_text(json.dumps({"nodes": []})) + + assert find_flow_files() == ["BareEmit.flow"] + assert find_flow_file() == "BareEmit.flow" + + +def test_structure_discovery_ignores_non_flow_sidecar_project(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + _make_flow_project(tmp_path, "AgentSidecar", "Agent", 0) + manifest = tmp_path / "AgentSidecar" / "Agent" / "project.uiproj" + manifest.write_text(json.dumps({"ProjectType": "Agent"})) + (tmp_path / "BareEmit.flow").write_text(json.dumps({"nodes": []})) + + assert find_flow_file() == "BareEmit.flow" + + +def test_solution_flow_beats_root_scratch_emit(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + project = _make_flow_project(tmp_path, "Solution", "Build", 4) + (tmp_path / "Build.flow").write_text(json.dumps({"nodes": [{"id": "scratch"}]})) + + assert find_flow_files() == [os.path.join("Solution", "Build", "Build.flow")] + assert find_flow_file() == os.path.join("Solution", "Build", "Build.flow") + assert project.name == "Build" + + +def test_root_fallback_refuses_distinct_candidates_and_lists_them( + tmp_path, monkeypatch +): + monkeypatch.chdir(tmp_path) + (tmp_path / "A.flow").write_text(json.dumps({"nodes": [{"id": "a"}]})) + (tmp_path / "B.flow").write_text(json.dumps({"nodes": [{"id": "b"}]})) + + with pytest.raises(SystemExit) as excinfo: + find_flow_files() + + message = str(excinfo.value) + assert "Multiple distinct root-level .flow files" in message + assert "A.flow" in message + assert "B.flow" in message + + +def test_root_fallback_collapses_byte_identical_copies(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + body = json.dumps({"nodes": [{"id": "same"}]}) + (tmp_path / "A.flow").write_text(body) + (tmp_path / "B.flow").write_text(body) + + assert find_flow_files() == ["A.flow"] + + +def test_debug_discovery_stays_project_scoped(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + (tmp_path / "BareEmit.flow").write_text(json.dumps({"nodes": []})) + + with pytest.raises(SystemExit, match="No project.uiproj found"): + find_project_dir() + + +def test_validate_flow_uses_root_fallback(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + (tmp_path / "BareEmit.flow").write_text(json.dumps({"nodes": []})) + calls = [] + + def fake_run(command, **kwargs): + calls.append((command, kwargs)) + return subprocess.CompletedProcess(command, 0, stdout="valid\n", stderr="") + + monkeypatch.setattr(validate_flow.subprocess, "run", fake_run) + + assert validate_flow.main() == 0 + assert calls == [ + ( + [ + "uip", + "maestro", + "flow", + "validate", + "BareEmit.flow", + "--output", + "json", + ], + {"capture_output": True, "text": True}, + ) + ] diff --git a/tests/tasks/uipath-maestro-flow/_shared/test_flow_contains.py b/tests/tasks/uipath-maestro-flow/_shared/test_flow_contains.py index 3735a590e5..f0ba95e272 100644 --- a/tests/tasks/uipath-maestro-flow/_shared/test_flow_contains.py +++ b/tests/tasks/uipath-maestro-flow/_shared/test_flow_contains.py @@ -48,6 +48,22 @@ def test_no_args_asserts_flow_exists(sandbox): assert main([]) == 0 +def test_no_args_accepts_one_root_level_sdk_emit(tmp_path, monkeypatch): + (tmp_path / "Demo.flow").write_text(FLOW_BODY) + monkeypatch.chdir(tmp_path) + + assert main([]) == 0 + + +def test_solution_flow_beats_root_level_scratch(tmp_path, monkeypatch): + _make_project(tmp_path) + (tmp_path / "Demo.flow").write_text('{"nodes": ["scratch-only"]}') + monkeypatch.chdir(tmp_path) + + assert main(["quick-form"]) == 0 + assert main(["scratch-only"]) == 1 + + def test_substring_present(sandbox): assert main(['"uipath.human-in-the-loop.quick-form"']) == 0 diff --git a/tests/tasks/uipath-maestro-flow/_shared/test_same_ground_advisories.py b/tests/tasks/uipath-maestro-flow/_shared/test_same_ground_advisories.py index a39023c29b..dc333aa7b0 100644 --- a/tests/tasks/uipath-maestro-flow/_shared/test_same_ground_advisories.py +++ b/tests/tasks/uipath-maestro-flow/_shared/test_same_ground_advisories.py @@ -183,6 +183,27 @@ def test_bindings_checker_accepts_v2_symbolic_key_with_real_values(tmp_path: Pat assert result.returncode == 0, result.stdout + result.stderr +def test_bindings_checker_prefers_solution_flow_over_root_scratch(tmp_path: Path) -> None: + checker = FLOW_TASKS / "bindings" / "check_bindings.py" + project = tmp_path / "BindingsMulti" / "BindingsMulti" + project.mkdir(parents=True) + (project / "project.uiproj").write_text(json.dumps({"ProjectType": "Flow"})) + good = {"nodes": [], "bindings": []} + (project / "BindingsMulti.flow").write_text(json.dumps(good)) + (tmp_path / "BindingsMulti.flow").write_text(json.dumps({"nodes": []})) + + result = subprocess.run( + [sys.executable, str(checker), "structure", "**/BindingsMulti*.flow"], + cwd=tmp_path, + capture_output=True, + text=True, + check=False, + ) + + assert result.returncode == 0, result.stdout + result.stderr + assert str(project.relative_to(tmp_path)) in result.stdout + + def test_inline_agent_ignores_staging_trees(tmp_path: Path) -> None: valid = { "settings": {"model": "gpt-4.1"}, diff --git a/tests/tasks/uipath-maestro-flow/_shared/test_same_ground_corpus.py b/tests/tasks/uipath-maestro-flow/_shared/test_same_ground_corpus.py new file mode 100644 index 0000000000..f7c3a25cf8 --- /dev/null +++ b/tests/tasks/uipath-maestro-flow/_shared/test_same_ground_corpus.py @@ -0,0 +1,260 @@ +"""Permanent same-ground contract for uipath-maestro-flow eval tasks. + +The temporary allowlists name pre-campaign debt. Each family batch removes its +own entries as the task contract becomes neutral; exact equality prevents stale +allowlist entries from hiding completed work. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +FLOW_TASKS = Path(__file__).resolve().parent.parent + +V1_AUTHORING_ALLOWLIST = { + "bindings/idempotent_reconfigure.yaml", + "bindings/multi_connector_independence.yaml", + "bindings/no_duplicate_connection_bindings.yaml", + "bindings/reconfigure_different_connection.yaml", + "connector_features/testmanager_attachments/testmanager_attachments.yaml", + "connector_features/testmanager_crud_grounded/testmanager_crud_grounded.yaml", + "connector_features/testmanager_execution_results/testmanager_execution_results.yaml", + "connector_features/testmanager_generic_records/testmanager_generic_records.yaml", + "connector_features/testmanager_requirement_lifecycle/testmanager_requirement_lifecycle.yaml", + "connector_features/testmanager_testcase_lifecycle/testmanager_testcase_lifecycle.yaml", + "connector_features/testmanager_testset_lifecycle/testmanager_testset_lifecycle.yaml", + "connector_trigger/webhook_waitfor_parallel.yaml", + "e2e/devcon_expense_approval.yaml", + "evaluate/evaluator_type_choice.yaml", + "evaluate/inline_agent_eval/inline_agent_eval.yaml", + "evaluate/local_crud.yaml", + "evaluate/no_auto_upload.yaml", + "evaluate/simulation/simulation_crud.yaml", + "interactive/solution_select.yaml", + "ixp/e2e_02_project_selection.yaml", + "ixp/integration_handle_routing.yaml", + "ixp/routing.yaml", + "ixp/routing_listing.yaml", + "ixp/scaffold_minimal.yaml", + "ixp/scaffold_multinode.yaml", + "smoke/init_validate.yaml", + "smoke/inline_agent_robust.yaml", + "smoke/registry_discovery.yaml", +} + +SKILL_TELEMETRY_ALLOWLIST = { + "interactive/customer_escalation_triage/customer_escalation_triage.yaml", + "ixp/e2e_02_project_selection.yaml", + "ixp/integration_handle_routing.yaml", + "ixp/scaffold_minimal.yaml", + "ixp/scaffold_multinode.yaml", +} + +FORBIDDEN_PROMPT_ALLOWLIST = { + "connector_features/slack-http-fallback/slack_http_fallback.yaml", + "context-grounding/batch_transform/batch_transform.yaml", + "context-grounding/summarize/summarize.yaml", + "evaluate/inline_agent_eval/inline_agent_eval.yaml", + "evaluate/local_crud.yaml", + "evaluate/simulation/simulation_crud.yaml", + "ixp/e2e_02_project_selection.yaml", + "ixp/integration_handle_routing.yaml", + "ixp/routing.yaml", + "ixp/routing_negative.yaml", + "ixp/scaffold_minimal.yaml", + "ixp/scaffold_multinode.yaml", + "single_node/delay/delay.yaml", + "single_node/outlook_waitfor_email/outlook_waitfor_email.yaml", + "single_node/transform_filter/transform_filter.yaml", + "single_node/transform_group_by/transform_group_by.yaml", + "single_node/transform_map/transform_map.yaml", + "smoke/merge_parallel_sync.yaml", + "smoke/scheduled_trigger.yaml", +} + +# These born-neutral escalation tasks are outside the recipe-edit sweep. Their +# prompts already require the same-name solution and tell the agent to leave +# the live execution to the grader; preserving them is part of the campaign's +# explicit no-edit fence. +FORBIDDEN_PROMPT_EXCEPTIONS = { + "e2e/escalation_jira_ticket/escalation_jira_ticket.yaml", + "e2e/escalation_orchestrator_paths/escalation_orchestrator_paths.yaml", + "e2e/escalation_slack_alert/escalation_slack_alert.yaml", +} + +DEBUG_SOLUTION_ALLOWLIST = { + "connector_features/generic_dynamic_node/generic_dynamic_node.yaml", + "connector_features/slack-http-fallback/slack_http_fallback.yaml", + "e2e/escalation_jira_ticket/escalation_jira_ticket.yaml", + "e2e/jira_create_issue/jira_create_issue.yaml", + "e2e/jira_get_issue/jira_get_issue.yaml", + "e2e/jira_lifecycle/jira_lifecycle.yaml", + "e2e/jira_search_triage/jira_search_triage.yaml", + "edit/add_node/add_node.yaml", + "edit/add_output/add_output.yaml", + "edit/group_to_subflow/group_to_subflow.yaml", + "edit/move_node/move_node.yaml", + "edit/remove_node/remove_node.yaml", + "edit/update_node/update_node.yaml", + "interactive/bellevue_weather_simulated/bellevue_weather_simulated.yaml", + "interactive/cli_dice_roller_simulated/cli_dice_roller_simulated.yaml", + "interactive/customer_escalation_triage/customer_escalation_triage.yaml", + "interactive/slack_channel_description_simulated/slack_channel_description_simulated.yaml", + "multi_node/bellevue_weather/bellevue_weather.yaml", + "multi_node/calculator/calculator.yaml", + "multi_node/customer_escalation/customer_escalation.yaml", + "multi_node/dice_roller/dice_roller.yaml", + "multi_node/feet_inches/feet_inches.yaml", + "multi_node/loop_multiply/loop_multiply.yaml", + "multi_node/multi_city_weather/multi_city_weather.yaml", + "multi_node/reading_list/reading_list.yaml", + "multi_node/slack_channel_description/slack_channel_description.yaml", + "multi_node/slack_weather_pipeline/slack_weather_pipeline.yaml", + "multi_node/wiki_pageviews/wiki_pageviews.yaml", + "single_node/api_workflow/api_workflow.yaml", + "single_node/decision/decision.yaml", + "single_node/file_attachment/file_attachment.yaml", + "single_node/lowcode_agent/lowcode_agent.yaml", + "single_node/openmeteo_weather/openmeteo_weather.yaml", + "single_node/rpa/rpa.yaml", + "single_node/subflow/subflow.yaml", + "single_node/switch/switch.yaml", + "single_node/terminate/terminate.yaml", + "connector_features/jdbc_databricks_query/jdbc_databricks_query.yaml", +} + +# F1 freezes the #2557 task even though its prompt predates the exact phrase. +DEBUG_PROJECT_LAYOUT_EXCEPTIONS = {"single_node/coded_agent/coded_agent.yaml"} + +UUID_PATTERN = re.compile( + r"\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b", + re.IGNORECASE, +) + + +def _block(text: str, key: str) -> str: + match = re.search(rf"(?ms)^{re.escape(key)}:[^\n]*\n(.*?)(?=^[A-Za-z_][\w-]*:|\Z)", text) + return match.group(0) if match else "" + + +def _criterion_blocks(text: str) -> list[tuple[str, str]]: + section = _block(text, "success_criteria") + return [ + (match.group(1), match.group(0)) + for match in re.finditer( + r"(?ms)^ - type:\s*([^\s#]+).*?(?=^ - type:|^[A-Za-z_][\w-]*:|\Z)", + section, + ) + ] + + +def _tagged_tasks() -> list[tuple[str, Path, str]]: + tasks = [] + for path in sorted(FLOW_TASKS.rglob("*.yaml")): + text = path.read_text() + if "uipath-maestro-flow" in _block(text, "tags"): + tasks.append((path.relative_to(FLOW_TASKS).as_posix(), path, text)) + return tasks + + +def _referenced_python_files(task_path: Path, criterion: str) -> list[Path]: + paths = [] + for token in re.findall( + r"(?:\$TASK_DIR/|\$SKILLS_REPO_PATH/)?[A-Za-z0-9_./-]+\.py", criterion + ): + if token.startswith("$TASK_DIR/"): + path = task_path.parent / token.removeprefix("$TASK_DIR/") + elif token.startswith("$SKILLS_REPO_PATH/"): + path = FLOW_TASKS.parents[2] / token.removeprefix("$SKILLS_REPO_PATH/") + else: + path = Path(token) + if path.is_file(): + paths.append(path) + return paths + + +def _is_debug_graded(task_path: Path, text: str) -> bool: + for criterion_type, criterion in _criterion_blocks(text): + if criterion_type == "command_not_executed": + continue + normalized = criterion.lower().replace("\\\\", "\\") + if "flow\\s+debug" in normalized or "flow debug" in normalized: + return True + if criterion_type != "run_command": + continue + for script in _referenced_python_files(task_path, criterion): + source = script.read_text(errors="replace") + if "run_debug(" in source or re.search( + r'["\']flow["\']\s*,\s*["\']debug["\']', source + ): + return True + return False + + +def _has_v1_authoring_grammar(criterion: str) -> bool: + normalized = criterion.lower().replace("\\\\", "\\") + markers = ( + "solution\\s+", + "flow\\s+init", + "flow\\s+node\\s+add", + "flow\\s+node\\s+configure", + "flow\\s+node\\s+remove", + "flow\\s+node\\s+update", + "flow\\s+registry", + ) + return any(marker in normalized for marker in markers) or ( + "agent\\s+init" in normalized and "inline-in-flow" in normalized + ) + + +def test_every_tagged_task_has_an_explicit_task_id() -> None: + missing = { + relative + for relative, _, text in _tagged_tasks() + if not re.search(r"(?m)^task_id:\s*\S+", text) + } + assert missing == set() + + +def test_v1_only_authoring_commands_match_the_temporary_allowlist() -> None: + offenders = set() + for relative, _, text in _tagged_tasks(): + for criterion_type, criterion in _criterion_blocks(text): + if criterion_type == "command_executed" and _has_v1_authoring_grammar( + criterion + ): + offenders.add(relative) + assert offenders == V1_AUTHORING_ALLOWLIST + + +def test_gating_skill_telemetry_matches_the_temporary_allowlist() -> None: + offenders = set() + for relative, _, text in _tagged_tasks(): + for criterion_type, criterion in _criterion_blocks(text): + if criterion_type != "skill_triggered": + continue + threshold = re.search(r"(?m)^\s+pass_threshold:\s*([0-9.]+)", criterion) + if threshold is None or float(threshold.group(1)) > 0: + offenders.add(relative) + assert offenders == SKILL_TELEMETRY_ALLOWLIST + + +def test_forbidden_prompt_phrases_match_the_temporary_allowlist() -> None: + offenders = set() + for relative, _, text in _tagged_tasks(): + prompt = " ".join(_block(text, "initial_prompt").lower().split()) + if re.search(r"do not [^.]*\bdebug\b", prompt) or UUID_PATTERN.search(prompt): + offenders.add(relative) + assert offenders == FORBIDDEN_PROMPT_ALLOWLIST | FORBIDDEN_PROMPT_EXCEPTIONS + + +def test_debug_graded_prompts_name_a_same_name_solution() -> None: + offenders = set() + for relative, path, text in _tagged_tasks(): + if not _is_debug_graded(path, text): + continue + prompt = " ".join(_block(text, "initial_prompt").lower().split()) + if not re.search(r"\b(?:in|inside) a solution of the same name\b", prompt): + offenders.add(relative) + assert offenders == DEBUG_SOLUTION_ALLOWLIST | DEBUG_PROJECT_LAYOUT_EXCEPTIONS diff --git a/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py b/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py index ded7ad1886..860a98e32c 100644 --- a/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py +++ b/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py @@ -1,6 +1,5 @@ #!/usr/bin/env python3 -"""Locate the Flow project's ``.flow`` file dynamically and run -``uip maestro flow validate`` on it. +"""Locate emitted ``.flow`` files and run ``uip maestro flow validate``. Usage (from a task's run_command, cwd = sandbox root): python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py @@ -13,10 +12,10 @@ itself is valid (observed on skill-flow-loop-multiply: criterion scored 0.0 purely on the path, while the flow validated fine when addressed correctly). -Discovery mirrors the ``check_*.py`` execution checks: find the lone -``project.uiproj`` whose manifest declares ``ProjectType="Flow"`` (via -:func:`flow_check.find_project_dir`), then validate every ``.flow`` file under -it. Exit 0 iff every file validates; otherwise propagate the failing exit code. +Discovery prefers the lone ``project.uiproj`` whose manifest declares +``ProjectType="Flow"`` and validates every flow under it. If no project exists, +it accepts one unambiguous root-level SDK emit instead. Exit 0 iff every selected +file validates; otherwise propagate the failing exit code. Timeouts are budgeted here rather than left to the harness. ``flow validate`` refreshes the node manifest from the tenant on every invocation, so its wall @@ -29,14 +28,15 @@ from __future__ import annotations -import glob import os import subprocess import sys import time -sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -from flow_check import find_project_dir # noqa: E402 +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from flow_check import find_flow_files # noqa: E402 # The criterion budget to self-terminate inside. coder_eval exports only # TASK_DIR to a criterion's command, not the criterion's own timeout, so there @@ -164,11 +164,7 @@ def _validate(flow: str, deadline: float, budget: int) -> int: def main() -> int: - project_dir = find_project_dir() - flows = sorted(glob.glob(os.path.join(project_dir, "**/*.flow"), recursive=True)) - if not flows: - print(f"FAIL: No .flow file found under {project_dir}", file=sys.stderr) - return 1 + flows = find_flow_files() budget = _budget_seconds() deadline = time.monotonic() + max( diff --git a/tests/tasks/uipath-maestro-flow/bindings/check_bindings.py b/tests/tasks/uipath-maestro-flow/bindings/check_bindings.py index 2dd58579ed..0c7bd027cf 100644 --- a/tests/tasks/uipath-maestro-flow/bindings/check_bindings.py +++ b/tests/tasks/uipath-maestro-flow/bindings/check_bindings.py @@ -67,14 +67,19 @@ from __future__ import annotations +from collections import Counter import glob import hashlib import json import os import sys -from collections import Counter +from pathlib import Path from typing import Any, NoReturn +SHARED_DIR = Path(__file__).resolve().parents[1] / "_shared" +sys.path.insert(0, str(SHARED_DIR)) +from flow_check import find_flow_files # noqa: E402 + def _fail(message: str) -> NoReturn: sys.exit(f"FAIL: {message}") @@ -109,7 +114,10 @@ def _dedupe_aliases(paths: list[str]) -> list[str]: def _load_one(pattern: str) -> tuple[str, dict[str, Any]]: - matches = _dedupe_aliases(glob.glob(pattern, recursive=True)) + if pattern.startswith("**/") and pattern.endswith(".flow"): + matches = find_flow_files(flow_glob=os.path.basename(pattern)) + else: + matches = _dedupe_aliases(glob.glob(pattern, recursive=True)) if not matches: _fail(f"No file found for {pattern!r}") if len(matches) > 1: diff --git a/tests/tasks/uipath-maestro-flow/e2e/check_devcon_expense_approval.py b/tests/tasks/uipath-maestro-flow/e2e/check_devcon_expense_approval.py index 01e8582865..13e7255821 100644 --- a/tests/tasks/uipath-maestro-flow/e2e/check_devcon_expense_approval.py +++ b/tests/tasks/uipath-maestro-flow/e2e/check_devcon_expense_approval.py @@ -9,13 +9,20 @@ from __future__ import annotations -import glob import json import sys from pathlib import Path +SHARED_DIR = Path(__file__).resolve().parents[1] / "_shared" +sys.path.insert(0, str(SHARED_DIR)) +try: + from flow_check import find_flow_file # noqa: E402 +except ModuleNotFoundError as exc: + if exc.name != "flow_check": + raise + from _shared.flow_check import find_flow_file # noqa: E402 -FLOW_GLOB = "ExpenseApproval/ExpenseApproval/ExpenseApproval.flow" +FLOW_GLOB = "ExpenseApproval*.flow" def fail(message: str) -> None: @@ -23,10 +30,7 @@ def fail(message: str) -> None: def load_flow() -> dict: - matches = glob.glob(FLOW_GLOB) - if not matches: - fail(f"No flow file matching {FLOW_GLOB}") - path = Path(matches[0]) + path = Path(find_flow_file(flow_glob=FLOW_GLOB)) try: return json.loads(path.read_text()) except json.JSONDecodeError as exc: diff --git a/tests/tasks/uipath-maestro-flow/e2e/test_check_devcon_expense_approval.py b/tests/tasks/uipath-maestro-flow/e2e/test_check_devcon_expense_approval.py index 0576095281..fb275a9d41 100644 --- a/tests/tasks/uipath-maestro-flow/e2e/test_check_devcon_expense_approval.py +++ b/tests/tasks/uipath-maestro-flow/e2e/test_check_devcon_expense_approval.py @@ -3,6 +3,8 @@ from __future__ import annotations import json +import os +import shutil import subprocess import sys from pathlib import Path @@ -58,6 +60,9 @@ def _flow_doc( def _write_flow(tmp_path: Path, doc: dict[str, Any]) -> None: project = tmp_path / "ExpenseApproval" / "ExpenseApproval" project.mkdir(parents=True) + (project / "project.uiproj").write_text( + json.dumps({"ProjectType": "Flow"}), encoding="utf-8" + ) (project / "ExpenseApproval.flow").write_text(json.dumps(doc), encoding="utf-8") @@ -121,6 +126,40 @@ def test_accepts_expression_hitl_input_binding(tmp_path: Path) -> None: assert result.returncode == 0, result.stderr +def test_accepts_root_level_sdk_emit(tmp_path: Path) -> None: + doc = _flow_doc( + fields=_approve_reject_fields("vars.fetchExpense.output.amount"), + outcomes=_APPROVE_REJECT_OUTCOMES, + ) + (tmp_path / "ExpenseApproval.flow").write_text(json.dumps(doc), encoding="utf-8") + + result = _run_checker(tmp_path) + + assert result.returncode == 0, result.stderr + + +def test_accepts_checker_frozen_without_sibling_shared_directory(tmp_path: Path) -> None: + workspace = tmp_path / "workspace" + workspace.mkdir() + _write_approve_reject_flow(workspace, "vars.fetchExpense.output.amount") + frozen_checker = tmp_path / "frozen" / CHECKER.name + frozen_checker.parent.mkdir() + shutil.copyfile(CHECKER, frozen_checker) + env = os.environ.copy() + env["PYTHONPATH"] = str(CHECKER.parents[1]) + + result = subprocess.run( + [sys.executable, str(frozen_checker)], + cwd=workspace, + env=env, + capture_output=True, + text=True, + check=False, + ) + + assert result.returncode == 0, result.stderr + + def test_rejects_hardcoded_hitl_input_binding(tmp_path: Path) -> None: _write_approve_reject_flow(tmp_path, "hardcoded-value") From 8bd0eb757b40f2378a923217029fabfd75ec1b2f Mon Sep 17 00:00:00 2001 From: Tao Ma <51425734+tmatup@users.noreply.github.com> Date: Fri, 21 Aug 2026 16:30:04 -0700 Subject: [PATCH 02/52] test(maestro-flow): align loop-neutral task batch (#2748) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Apply the ratified same-ground alignment recipe to the 31 Batch 1 tasks and preserve their criterion intent and weights. ๐Ÿค– Generated with Claude Code Co-Authored-By: [Claude](mailto:noreply@anthropic.com) --- .../_shared/check_inline_agent.py | 31 ++++- .../_shared/test_same_ground_advisories.py | 29 +++++ .../_shared/test_same_ground_corpus.py | 40 ++----- .../bellevue_weather/bellevue_weather.yaml | 6 + .../multi_node/calculator/calculator.yaml | 6 + .../check_customer_escalation_flow.py | 24 +--- .../customer_escalation.yaml | 5 +- .../test_check_customer_escalation_flow.py | 17 ++- .../multi_node/dice_roller/dice_roller.yaml | 6 + .../multi_node/feet_inches/feet_inches.yaml | 6 + .../loop_multiply/loop_multiply.yaml | 6 + .../multi_city_weather.yaml | 6 + .../multi_node/reading_list/reading_list.yaml | 6 + .../slack_channel_description.yaml | 6 + .../slack_weather_pipeline.yaml | 8 +- .../wiki_pageviews/wiki_pageviews.yaml | 6 + .../api_workflow/api_workflow.yaml | 6 + .../single_node/decision/decision.yaml | 6 + .../single_node/delay/check_delay_flow.py | 10 +- .../single_node/delay/delay.yaml | 12 +- .../file_attachment/file_attachment.yaml | 10 +- .../lowcode_agent/lowcode_agent.yaml | 6 + .../openmeteo_weather/openmeteo_weather.yaml | 8 +- .../check_outlook_trigger_inbox.py | 30 +++-- .../outlook_trigger_inbox.yaml | 19 ++- .../test_check_outlook_trigger_inbox.py | 26 +++++ .../check_outlook_waitfor_email.py | 10 +- .../outlook_waitfor_email.yaml | 6 +- .../single_node/rpa/rpa.yaml | 6 + .../single_node/subflow/subflow.yaml | 6 + .../single_node/switch/switch.yaml | 6 + .../single_node/terminate/terminate.yaml | 6 + .../test_transform_output_envelopes.py | 108 ++++++++++++++++++ .../check_transform_filter_flow.py | 15 +-- .../transform_filter/transform_filter.yaml | 13 ++- .../check_transform_group_by_flow.py | 15 +-- .../transform_group_by.yaml | 12 +- .../transform_map/check_transform_map_flow.py | 15 +-- .../transform_map/transform_map.yaml | 13 ++- .../smoke/check_merge_parallel_sync_flow.py | 14 +-- .../smoke/check_scheduled_trigger_flow.py | 10 +- .../smoke/init_validate.yaml | 54 ++++----- .../smoke/inline_agent_robust.yaml | 59 +++++----- .../smoke/merge_parallel_sync.yaml | 34 ++---- .../smoke/registry_discovery.yaml | 38 +++--- .../smoke/scheduled_trigger.yaml | 34 ++---- 46 files changed, 537 insertions(+), 278 deletions(-) create mode 100644 tests/tasks/uipath-maestro-flow/single_node/test_transform_output_envelopes.py diff --git a/tests/tasks/uipath-maestro-flow/_shared/check_inline_agent.py b/tests/tasks/uipath-maestro-flow/_shared/check_inline_agent.py index bfff5fc65c..c832bf5fb6 100644 --- a/tests/tasks/uipath-maestro-flow/_shared/check_inline_agent.py +++ b/tests/tasks/uipath-maestro-flow/_shared/check_inline_agent.py @@ -3,12 +3,13 @@ to a production bar, not left on the toy scaffold defaults. Usage (from a task's run_command, cwd = sandbox root): - python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/check_inline_agent.py + python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/check_inline_agent.py [--check ] [] Shell-style glob for the inline agent.json. The inline agent dir is a UUID, so the path is not statically knowable โ€” pass e.g. "EmailTriage/EmailTriage/*/agent.json". Defaults to "**/agent.json" if omitted. + One of ``exists``, ``model``, or ``quality`` (the default). Asserts, on the first matching agent.json (excluding generated .agent-builder/): 1. settings.model is set and is NOT the stale scaffold default gpt-4o-2024-11-20. @@ -54,8 +55,23 @@ } +def _parse_args() -> tuple[str, str]: + args = sys.argv[1:] + scope = "quality" + if args[:1] == ["--check"]: + if len(args) < 2 or args[1] not in {"exists", "model", "quality"}: + print("FAIL: --check must be one of exists, model, or quality") + raise SystemExit(1) + scope = args[1] + args = args[2:] + if len(args) > 1: + print("FAIL: expected at most one agent.json glob") + raise SystemExit(1) + return scope, args[0] if args else "**/agent.json" + + def main() -> int: - pattern = sys.argv[1] if len(sys.argv) > 1 else "**/agent.json" + scope, pattern = _parse_args() paths = [ path for path in glob.glob(pattern, recursive=True) @@ -66,6 +82,10 @@ def main() -> int: return 1 path = min(paths) + if scope == "exists": + print(f"OK ({path}): inline agent sidecar exists") + return 0 + try: with open(path) as source: agent = json.load(source) @@ -81,6 +101,13 @@ def main() -> int: elif model == SCAFFOLD_MODEL: errs.append(f"settings.model not overridden ({model})") + if scope == "model": + if errs: + print(f"FAIL ({path}): " + "; ".join(errs)) + return 1 + print(f"OK ({path}): model={model}") + return 0 + sys_msgs = [m.get("content", "") for m in agent.get("messages", []) if m.get("role") == "system"] prompt = (sys_msgs[0] if sys_msgs else "").strip() if prompt.lower() in PLACEHOLDER_PROMPTS or len(prompt) < MIN_PROMPT_LEN: diff --git a/tests/tasks/uipath-maestro-flow/_shared/test_same_ground_advisories.py b/tests/tasks/uipath-maestro-flow/_shared/test_same_ground_advisories.py index dc333aa7b0..db00cdc9f5 100644 --- a/tests/tasks/uipath-maestro-flow/_shared/test_same_ground_advisories.py +++ b/tests/tasks/uipath-maestro-flow/_shared/test_same_ground_advisories.py @@ -222,6 +222,35 @@ def test_inline_agent_ignores_staging_trees(tmp_path: Path) -> None: assert "Solution" in result.stdout +def test_inline_agent_scoped_checks_preserve_artifact_outcomes(tmp_path: Path) -> None: + agent = tmp_path / "agent-id" / "agent.json" + agent.parent.mkdir() + agent.write_text(json.dumps({"settings": {"model": "gpt-4.1"}})) + + exists = run_script( + "check_inline_agent.py", "--check", "exists", "**/agent.json", cwd=tmp_path + ) + model = run_script( + "check_inline_agent.py", "--check", "model", "**/agent.json", cwd=tmp_path + ) + + assert exists.returncode == 0, exists.stdout + exists.stderr + assert model.returncode == 0, model.stdout + model.stderr + + +def test_inline_agent_model_scope_rejects_scaffold_default(tmp_path: Path) -> None: + agent = tmp_path / "agent-id" / "agent.json" + agent.parent.mkdir() + agent.write_text(json.dumps({"settings": {"model": "gpt-4o-2024-11-20"}})) + + result = run_script( + "check_inline_agent.py", "--check", "model", "**/agent.json", cwd=tmp_path + ) + + assert result.returncode != 0 + assert "not overridden" in result.stdout + + def test_reachability_excludes_the_loop_back_edge() -> None: """A loop body must not reach the End nodes that follow the loop. diff --git a/tests/tasks/uipath-maestro-flow/_shared/test_same_ground_corpus.py b/tests/tasks/uipath-maestro-flow/_shared/test_same_ground_corpus.py index f7c3a25cf8..83e7bea3ca 100644 --- a/tests/tasks/uipath-maestro-flow/_shared/test_same_ground_corpus.py +++ b/tests/tasks/uipath-maestro-flow/_shared/test_same_ground_corpus.py @@ -38,9 +38,6 @@ "ixp/routing_listing.yaml", "ixp/scaffold_minimal.yaml", "ixp/scaffold_multinode.yaml", - "smoke/init_validate.yaml", - "smoke/inline_agent_robust.yaml", - "smoke/registry_discovery.yaml", } SKILL_TELEMETRY_ALLOWLIST = { @@ -64,13 +61,6 @@ "ixp/routing_negative.yaml", "ixp/scaffold_minimal.yaml", "ixp/scaffold_multinode.yaml", - "single_node/delay/delay.yaml", - "single_node/outlook_waitfor_email/outlook_waitfor_email.yaml", - "single_node/transform_filter/transform_filter.yaml", - "single_node/transform_group_by/transform_group_by.yaml", - "single_node/transform_map/transform_map.yaml", - "smoke/merge_parallel_sync.yaml", - "smoke/scheduled_trigger.yaml", } # These born-neutral escalation tasks are outside the recipe-edit sweep. Their @@ -101,26 +91,6 @@ "interactive/cli_dice_roller_simulated/cli_dice_roller_simulated.yaml", "interactive/customer_escalation_triage/customer_escalation_triage.yaml", "interactive/slack_channel_description_simulated/slack_channel_description_simulated.yaml", - "multi_node/bellevue_weather/bellevue_weather.yaml", - "multi_node/calculator/calculator.yaml", - "multi_node/customer_escalation/customer_escalation.yaml", - "multi_node/dice_roller/dice_roller.yaml", - "multi_node/feet_inches/feet_inches.yaml", - "multi_node/loop_multiply/loop_multiply.yaml", - "multi_node/multi_city_weather/multi_city_weather.yaml", - "multi_node/reading_list/reading_list.yaml", - "multi_node/slack_channel_description/slack_channel_description.yaml", - "multi_node/slack_weather_pipeline/slack_weather_pipeline.yaml", - "multi_node/wiki_pageviews/wiki_pageviews.yaml", - "single_node/api_workflow/api_workflow.yaml", - "single_node/decision/decision.yaml", - "single_node/file_attachment/file_attachment.yaml", - "single_node/lowcode_agent/lowcode_agent.yaml", - "single_node/openmeteo_weather/openmeteo_weather.yaml", - "single_node/rpa/rpa.yaml", - "single_node/subflow/subflow.yaml", - "single_node/switch/switch.yaml", - "single_node/terminate/terminate.yaml", "connector_features/jdbc_databricks_query/jdbc_databricks_query.yaml", } @@ -178,7 +148,10 @@ def _is_debug_graded(task_path: Path, text: str) -> bool: for criterion_type, criterion in _criterion_blocks(text): if criterion_type == "command_not_executed": continue - normalized = criterion.lower().replace("\\\\", "\\") + executable = "\n".join( + line for line in criterion.splitlines() if not line.lstrip().startswith("#") + ) + normalized = executable.lower().replace("\\\\", "\\") if "flow\\s+debug" in normalized or "flow debug" in normalized: return True if criterion_type != "run_command": @@ -221,9 +194,12 @@ def test_v1_only_authoring_commands_match_the_temporary_allowlist() -> None: offenders = set() for relative, _, text in _tagged_tasks(): for criterion_type, criterion in _criterion_blocks(text): - if criterion_type == "command_executed" and _has_v1_authoring_grammar( + if criterion_type != "command_executed" or not _has_v1_authoring_grammar( criterion ): + continue + threshold = re.search(r"(?m)^\s+pass_threshold:\s*([0-9.]+)", criterion) + if threshold is None or float(threshold.group(1)) > 0: offenders.add(relative) assert offenders == V1_AUTHORING_ALLOWLIST diff --git a/tests/tasks/uipath-maestro-flow/multi_node/bellevue_weather/bellevue_weather.yaml b/tests/tasks/uipath-maestro-flow/multi_node/bellevue_weather/bellevue_weather.yaml index 6905a1a767..0680ae7e63 100644 --- a/tests/tasks/uipath-maestro-flow/multi_node/bellevue_weather/bellevue_weather.yaml +++ b/tests/tasks/uipath-maestro-flow/multi_node/bellevue_weather/bellevue_weather.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-bellevue-weather description: > Create a UiPath Flow that fetches today's weather in Bellevue from open-meteo, @@ -10,6 +14,8 @@ run_limits: turn_timeout: 1200 initial_prompt: | + Build the flow inside a solution of the same name. + Create a UiPath Flow project named "BellevueWeather" that gets today's weather in Bellevue from open-meteo, formats a summary with a script, and if the temperature is greater than 60F returns a summary with a message field 'nice day', diff --git a/tests/tasks/uipath-maestro-flow/multi_node/calculator/calculator.yaml b/tests/tasks/uipath-maestro-flow/multi_node/calculator/calculator.yaml index 5a6a109a34..239b1b495a 100644 --- a/tests/tasks/uipath-maestro-flow/multi_node/calculator/calculator.yaml +++ b/tests/tasks/uipath-maestro-flow/multi_node/calculator/calculator.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-calculator description: > Create a UiPath Flow that takes two number inputs and calculates their product @@ -9,6 +13,8 @@ run_limits: turn_timeout: 1200 initial_prompt: | + Build the flow inside a solution of the same name. + Create a UiPath Flow project named "Calculator" that takes two numbers as input and calculates their product using a Script node. The result should be returned as an output variable. diff --git a/tests/tasks/uipath-maestro-flow/multi_node/customer_escalation/check_customer_escalation_flow.py b/tests/tasks/uipath-maestro-flow/multi_node/customer_escalation/check_customer_escalation_flow.py index a35e90ec25..00119cca3d 100644 --- a/tests/tasks/uipath-maestro-flow/multi_node/customer_escalation/check_customer_escalation_flow.py +++ b/tests/tasks/uipath-maestro-flow/multi_node/customer_escalation/check_customer_escalation_flow.py @@ -25,15 +25,13 @@ from __future__ import annotations -import glob import json -import os import sys from pathlib import Path from typing import Any, NoReturn -sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) -from _shared.flow_check import find_project_dir # noqa: E402 +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) +from _shared.flow_check import find_flow_file # noqa: E402 def fail(msg: str) -> NoReturn: @@ -41,23 +39,7 @@ def fail(msg: str) -> NoReturn: def load_flow() -> dict[str, Any]: - # Discover the Flow project rather than hardcoding - # `CustomerEscalation/CustomerEscalation/CustomerEscalation.flow`: the prompt - # names only the FLOW ('a Maestro Flow called "CustomerEscalation"'), never - # the enclosing solution, and the two names are independent โ€” `uip solution - # init && cd && uip maestro flow init ` yields `//.flow` - # for any , while a bare `uip maestro flow init ` auto-scaffolds - # `Solution//.flow`. A path-pinned glob graded the solution name, - # an unstated requirement: skill-flow-customer-escalation scored 0.0 on this - # criterion with a fully valid flow at - # `CustomerEscalationSolution/CustomerEscalation/CustomerEscalation.flow`. - # Mirrors the discovery in `_shared/validate_flow.py`, so this criterion and - # the validate criterion address the same file. - project_dir = find_project_dir() - matches = sorted(glob.glob(os.path.join(project_dir, "**/*.flow"), recursive=True)) - if not matches: - fail(f"No .flow file found under {project_dir}") - path = Path(matches[0]) + path = Path(find_flow_file(flow_glob="CustomerEscalation*.flow")) try: return json.loads(path.read_text()) except json.JSONDecodeError as exc: diff --git a/tests/tasks/uipath-maestro-flow/multi_node/customer_escalation/customer_escalation.yaml b/tests/tasks/uipath-maestro-flow/multi_node/customer_escalation/customer_escalation.yaml index c384d2849e..e777656912 100644 --- a/tests/tasks/uipath-maestro-flow/multi_node/customer_escalation/customer_escalation.yaml +++ b/tests/tasks/uipath-maestro-flow/multi_node/customer_escalation/customer_escalation.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-customer-escalation description: > Complex multi-branch escalation flow: Outlook email-received trigger โ†’ @@ -58,4 +62,3 @@ success_criteria: expected_exit_code: 0 weight: 10.0 pass_threshold: 1.0 - diff --git a/tests/tasks/uipath-maestro-flow/multi_node/customer_escalation/test_check_customer_escalation_flow.py b/tests/tasks/uipath-maestro-flow/multi_node/customer_escalation/test_check_customer_escalation_flow.py index 370a2ad82c..bb98f916f7 100644 --- a/tests/tasks/uipath-maestro-flow/multi_node/customer_escalation/test_check_customer_escalation_flow.py +++ b/tests/tasks/uipath-maestro-flow/multi_node/customer_escalation/test_check_customer_escalation_flow.py @@ -11,8 +11,8 @@ - Structural requirements (>=5 nodes, exactly one decision with >=2 branches, >=2 scripts, Slack reference, non-trigger Outlook/Graph reference) are still enforced. -- The verdict is independent of the enclosing SOLUTION directory's name, - which the prompt never specifies (see `test_any_solution_name_passes`). +- The verdict is independent of the authoring layout: any solution directory + name or one root-level SDK emit is accepted. """ from __future__ import annotations @@ -42,8 +42,7 @@ def _flow_dir(tmp_path: Path, solution: str = "CustomerEscalationSolution") -> P against a layout the CLI cannot emit, which is why the checker's path-coupling survived to fail a real run. - `project.uiproj` is required: `find_project_dir` selects the project whose - manifest declares `ProjectType: "Flow"`. + `project.uiproj` makes this the live-v1 solution-layout fixture. """ d = tmp_path / solution / "CustomerEscalation" d.mkdir(parents=True) @@ -154,6 +153,16 @@ def test_any_solution_name_passes(tmp_path: Path, solution: str) -> None: assert result.returncode == 0, result.stdout + result.stderr +def test_root_level_sdk_emit_passes(tmp_path: Path) -> None: + nodes, edges = _outlook_connector_shape() + payload = {"version": "1.0.0", "nodes": nodes, "edges": edges} + (tmp_path / "CustomerEscalation.flow").write_text(json.dumps(payload)) + + result = _run(tmp_path) + + assert result.returncode == 0, result.stdout + result.stderr + + def test_missing_outlook_reference_fails(tmp_path: Path) -> None: """Flow with NO outlook/office365/graph reference on any reply node FAILS.""" nodes, edges = _manual_trigger_with_graph_replies() diff --git a/tests/tasks/uipath-maestro-flow/multi_node/dice_roller/dice_roller.yaml b/tests/tasks/uipath-maestro-flow/multi_node/dice_roller/dice_roller.yaml index dd75fa2522..128e786a79 100644 --- a/tests/tasks/uipath-maestro-flow/multi_node/dice_roller/dice_roller.yaml +++ b/tests/tasks/uipath-maestro-flow/multi_node/dice_roller/dice_roller.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-dice-roller description: > Create a UiPath Flow from scratch that simulates rolling a fair six-sided die. @@ -11,6 +15,8 @@ run_limits: turn_timeout: 1200 initial_prompt: | + Build the flow inside a solution of the same name. + Create a UiPath Flow project named "DiceRoller" that simulates rolling a six-sided die and outputs the result. diff --git a/tests/tasks/uipath-maestro-flow/multi_node/feet_inches/feet_inches.yaml b/tests/tasks/uipath-maestro-flow/multi_node/feet_inches/feet_inches.yaml index d64eab27b1..4653595568 100644 --- a/tests/tasks/uipath-maestro-flow/multi_node/feet_inches/feet_inches.yaml +++ b/tests/tasks/uipath-maestro-flow/multi_node/feet_inches/feet_inches.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-feet-inches description: > Create a UiPath Flow that converts a value between feet and inches based on @@ -11,6 +15,8 @@ run_limits: task_timeout: 2700 # 1380s grading + a turn; inherited 1200 could not fit grading initial_prompt: | + Build the flow inside a solution of the same name. + Create a UiPath Flow project named "FeetInches" that takes `value` (number) and `direction` (string: "f2i", "i2f", or "y2f") as inputs, converts feet to inches (ร— 12), inches to feet (รท 12), or yards to feet (ร— 3) accordingly, diff --git a/tests/tasks/uipath-maestro-flow/multi_node/loop_multiply/loop_multiply.yaml b/tests/tasks/uipath-maestro-flow/multi_node/loop_multiply/loop_multiply.yaml index ab64ffd4da..23d97a8376 100644 --- a/tests/tasks/uipath-maestro-flow/multi_node/loop_multiply/loop_multiply.yaml +++ b/tests/tasks/uipath-maestro-flow/multi_node/loop_multiply/loop_multiply.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-loop-multiply description: > Create a UiPath Flow that uses a Loop node to multiply the numbers @@ -10,6 +14,8 @@ run_limits: turn_timeout: 1200 initial_prompt: | + Build the flow inside a solution of the same name. + Create a UiPath Flow project named "LoopMultiply" that multiplies the numbers [13, 15, 17] together using a Loop node and returns the product. diff --git a/tests/tasks/uipath-maestro-flow/multi_node/multi_city_weather/multi_city_weather.yaml b/tests/tasks/uipath-maestro-flow/multi_node/multi_city_weather/multi_city_weather.yaml index 57e8b3e480..f989c15b0f 100644 --- a/tests/tasks/uipath-maestro-flow/multi_node/multi_city_weather/multi_city_weather.yaml +++ b/tests/tasks/uipath-maestro-flow/multi_node/multi_city_weather/multi_city_weather.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-multi-city-weather description: > Loop over 3 cities, fetch weather from open-meteo for each, classify warm/cold @@ -10,6 +14,8 @@ run_limits: turn_timeout: 1200 initial_prompt: | + Build the flow inside a solution of the same name. + Create a flow called "MultiCityWeather". Loop over Seattle, Phoenix, and New York โ€” for each city, fetch the current temperature from open-meteo (fahrenheit) and classify it as 'warm' (> 60F) or 'cold'. Output an array with all 3 results, each having the city name, temperature, and verdict. Use a Managed HTTP Request node (core.action.http.v2) to call the open-meteo API directly โ€” do not use an Integration Service connector. diff --git a/tests/tasks/uipath-maestro-flow/multi_node/reading_list/reading_list.yaml b/tests/tasks/uipath-maestro-flow/multi_node/reading_list/reading_list.yaml index 305eccd2e3..9c3e8fe1c1 100644 --- a/tests/tasks/uipath-maestro-flow/multi_node/reading_list/reading_list.yaml +++ b/tests/tasks/uipath-maestro-flow/multi_node/reading_list/reading_list.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-reading-list description: > Create a UiPath Flow that curates a reading list from a catalog of math/ML/stats @@ -11,6 +15,8 @@ run_limits: turn_timeout: 1200 initial_prompt: | + Build the flow inside a solution of the same name. + Create a UiPath Flow project named "ReadingList" that curates a reading list from a catalog of math and data science books. diff --git a/tests/tasks/uipath-maestro-flow/multi_node/slack_channel_description/slack_channel_description.yaml b/tests/tasks/uipath-maestro-flow/multi_node/slack_channel_description/slack_channel_description.yaml index 70df43e92c..1e704928c7 100644 --- a/tests/tasks/uipath-maestro-flow/multi_node/slack_channel_description/slack_channel_description.yaml +++ b/tests/tasks/uipath-maestro-flow/multi_node/slack_channel_description/slack_channel_description.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-slack-channel-description description: > Create a UiPath Flow that uses the Slack IS connector to retrieve the @@ -11,6 +15,8 @@ run_limits: turn_timeout: 1200 initial_prompt: | + Build the flow inside a solution of the same name. + Create a UiPath Flow named "SlackChannelDescription" that retrieves the channel description of #office-bellevue and outputs it. diff --git a/tests/tasks/uipath-maestro-flow/multi_node/slack_weather_pipeline/slack_weather_pipeline.yaml b/tests/tasks/uipath-maestro-flow/multi_node/slack_weather_pipeline/slack_weather_pipeline.yaml index 21f962aa1d..833fa90098 100644 --- a/tests/tasks/uipath-maestro-flow/multi_node/slack_weather_pipeline/slack_weather_pipeline.yaml +++ b/tests/tasks/uipath-maestro-flow/multi_node/slack_weather_pipeline/slack_weather_pipeline.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-slack-weather-pipeline description: > Multi-step pipeline: read Slack channel description, extract city with a script, @@ -11,11 +15,13 @@ run_limits: turn_timeout: 1200 initial_prompt: | + Build the flow inside a solution of the same name. + Create a flow called "SlackWeatherPipeline". Read the #office-bellevue Slack channel description โ€” it holds the office's full US mailing address (street, suite/unit, city, state, ZIP) โ€” identify the city (the locality, not the street or suite line). Then fetch the current weather for that city from open-meteo. If it's above 60F the flow must output the text in variable `weatherVerdict` 'warm office today', otherwise 'cold office today'. All connections for this task live in Orchestrator folder `Shared/uipath-maestro-flow`. - Validate the flow, then run `uip maestro flow debug` and iterate until it completes without incidents. + Validate the flow, then run it and iterate until it completes without incidents. Do NOT ask for approval, confirmation, or feedback. Do NOT pause between planning and implementation. Build the complete flow end-to-end in a single pass. success_criteria: diff --git a/tests/tasks/uipath-maestro-flow/multi_node/wiki_pageviews/wiki_pageviews.yaml b/tests/tasks/uipath-maestro-flow/multi_node/wiki_pageviews/wiki_pageviews.yaml index bfcf65dcd1..64f339af92 100644 --- a/tests/tasks/uipath-maestro-flow/multi_node/wiki_pageviews/wiki_pageviews.yaml +++ b/tests/tasks/uipath-maestro-flow/multi_node/wiki_pageviews/wiki_pageviews.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-wiki-pageviews description: > Create a UiPath Flow that fetches a range of Wikipedia pageviews, keeps @@ -13,6 +17,8 @@ run_limits: task_timeout: 3000 # 1620s grading + a turn; inherited 1200 could not fit grading initial_prompt: | + Build the flow inside a solution of the same name. + Create a UiPath Flow project named "WikiPageviews" that takes `article` (string) and `date1`, `date2` (strings in `YYYYMMDD` format) as inputs. Fetch the daily view counts in the range from the Wikimedia pageviews API diff --git a/tests/tasks/uipath-maestro-flow/single_node/api_workflow/api_workflow.yaml b/tests/tasks/uipath-maestro-flow/single_node/api_workflow/api_workflow.yaml index eebfd349f8..7d478d5f51 100644 --- a/tests/tasks/uipath-maestro-flow/single_node/api_workflow/api_workflow.yaml +++ b/tests/tasks/uipath-maestro-flow/single_node/api_workflow/api_workflow.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-api-workflow description: > Create a UiPath Flow that invokes the name-to-age API workflow with the @@ -10,6 +14,8 @@ run_limits: turn_timeout: 1200 initial_prompt: | + Build the flow inside a solution of the same name. + Create a UiPath Flow project named "NameToAge" that invokes the name-to-age API workflow with the name 'tomasz' and returns his age as an output. diff --git a/tests/tasks/uipath-maestro-flow/single_node/decision/decision.yaml b/tests/tasks/uipath-maestro-flow/single_node/decision/decision.yaml index fb3a14b056..52741cbf68 100644 --- a/tests/tasks/uipath-maestro-flow/single_node/decision/decision.yaml +++ b/tests/tasks/uipath-maestro-flow/single_node/decision/decision.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-decision description: > Create a UiPath Flow that takes a temperature in Fahrenheit and uses a Decision @@ -12,6 +16,8 @@ run_limits: task_timeout: 2700 # 1260s grading + a turn; inherited 1200 could not fit grading initial_prompt: | + Build the flow inside a solution of the same name. + Create a UiPath Flow project named "TemperatureChecker" that takes a temperature in Fahrenheit as input. If the temperature is greater than 75, the flow should output "warm". Otherwise it should output "cool". diff --git a/tests/tasks/uipath-maestro-flow/single_node/delay/check_delay_flow.py b/tests/tasks/uipath-maestro-flow/single_node/delay/check_delay_flow.py index f93efeaa86..d03db0d942 100644 --- a/tests/tasks/uipath-maestro-flow/single_node/delay/check_delay_flow.py +++ b/tests/tasks/uipath-maestro-flow/single_node/delay/check_delay_flow.py @@ -19,11 +19,14 @@ `sourceNodeId` is the delay node id (outgoing). """ -import glob import json import sys +from pathlib import Path from typing import NoReturn +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) +from _shared.flow_check import find_flow_file # noqa: E402 + NODE_TYPE = "core.logic.delay" VALID_TIMER_TYPES = {"timeDuration", "timeDate"} @@ -33,10 +36,7 @@ def _fail(msg: str) -> NoReturn: def _read_flow() -> dict: - flows = glob.glob("**/DelayDemo*.flow", recursive=True) - if not flows: - _fail("no DelayDemo*.flow found under cwd") - with open(flows[0]) as f: + with open(find_flow_file(flow_glob="DelayDemo*.flow")) as f: return json.load(f) diff --git a/tests/tasks/uipath-maestro-flow/single_node/delay/delay.yaml b/tests/tasks/uipath-maestro-flow/single_node/delay/delay.yaml index c8b35a7534..da0784eaa3 100644 --- a/tests/tasks/uipath-maestro-flow/single_node/delay/delay.yaml +++ b/tests/tasks/uipath-maestro-flow/single_node/delay/delay.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-delay description: > Create a UiPath Flow with a single OOTB Delay node (`core.logic.delay`) @@ -20,18 +24,16 @@ initial_prompt: | - `inputs.timerType` must be `"timeDuration"`. - `inputs.timerPreset` must be a fixed ISO 8601 duration preset such as `"PT15M"` (15 minutes). Do NOT use `"custom"` for this smoke test. - - Set `typeVersion` from the node registry for `core.logic.delay` โ€” - do NOT hardcode a guessed version. + - Set `typeVersion` from the authoritative definition for + `core.logic.delay`. Wire the Delay node so that it has an incoming edge from the trigger and an outgoing edge to the End node (an `edges[]` entry whose `targetNodeId` is the delay node id, and one whose `sourceNodeId` is the delay node id). - Do NOT run flow debug โ€” just validate the flow. + Validate the flow. Do NOT ask for approval, confirmation, or feedback. Do NOT pause between planning and implementation. Build the complete flow end-to-end in a single pass. - Before starting, follow the documented workflow steps exactly โ€” in particular, the delay plugin's `inputs` shape - (`timerType` + `timerPreset`). success_criteria: # โ”€โ”€ Flow file validity โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ diff --git a/tests/tasks/uipath-maestro-flow/single_node/file_attachment/file_attachment.yaml b/tests/tasks/uipath-maestro-flow/single_node/file_attachment/file_attachment.yaml index eb10dbc135..6a49769da8 100644 --- a/tests/tasks/uipath-maestro-flow/single_node/file_attachment/file_attachment.yaml +++ b/tests/tasks/uipath-maestro-flow/single_node/file_attachment/file_attachment.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-file-attachment-debug description: > Build a Flow whose trigger exposes a file-typed input variable, read the @@ -21,13 +25,15 @@ run_limits: turn_timeout: 1200 initial_prompt: | + Build the flow inside a solution of the same name. + Create a UiPath Flow project named "AttachmentEcho" that accepts a file as input and echoes the uploaded file's name back as a flow output. Use a file-typed input variable named `inputDoc` and surface the name as an output variable named `fileName`. - Validate the flow, then run it with `uip maestro flow debug`, binding a local - file to the input. The task is not complete until the debug run reports + Validate the flow, then run it with a local file bound to the input. The + task is not complete until the run reports `finalStatus: "Completed"` and the output reflects the attached file's name. Do NOT ask for approval, confirmation, or feedback. Do NOT pause between diff --git a/tests/tasks/uipath-maestro-flow/single_node/lowcode_agent/lowcode_agent.yaml b/tests/tasks/uipath-maestro-flow/single_node/lowcode_agent/lowcode_agent.yaml index 415736357b..6fe44d88e7 100644 --- a/tests/tasks/uipath-maestro-flow/single_node/lowcode_agent/lowcode_agent.yaml +++ b/tests/tasks/uipath-maestro-flow/single_node/lowcode_agent/lowcode_agent.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-lowcode-agent description: > Create a UiPath Flow that wires in the existing CountLetters low-code agent @@ -15,6 +19,8 @@ run_limits: turn_timeout: 1200 initial_prompt: | + Build the flow inside a solution of the same name. + Create a UiPath Flow project named "CountLettersLowCode" that uses the existing CountLetters low-code agent to count the number of r's in 'arrow' and return the answer. The agent already exists โ€” discover it and wire it diff --git a/tests/tasks/uipath-maestro-flow/single_node/openmeteo_weather/openmeteo_weather.yaml b/tests/tasks/uipath-maestro-flow/single_node/openmeteo_weather/openmeteo_weather.yaml index a911f49ab7..1bdc369e9c 100644 --- a/tests/tasks/uipath-maestro-flow/single_node/openmeteo_weather/openmeteo_weather.yaml +++ b/tests/tasks/uipath-maestro-flow/single_node/openmeteo_weather/openmeteo_weather.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-openmeteo-weather description: > End-to-end: build a Flow whose process fetches the CURRENT weather in Bellevue @@ -24,13 +28,15 @@ run_limits: turn_timeout: 1200 initial_prompt: | + Build the flow inside a solution of the same name. + Create a UiPath Flow project named "OpenMeteoWeather". The process must fetch the CURRENT weather in Bellevue using the Open-Meteo connector, then surface the current temperature as a flow output variable. Use the Open-Meteo Integration Service connector. Validate the flow. - The task is not complete until `uip maestro flow validate` passes. + The task is not complete until the flow validates. Do NOT ask for approval, confirmation, or feedback. Do NOT pause between planning and implementation. Build and validate the complete flow end-to-end diff --git a/tests/tasks/uipath-maestro-flow/single_node/outlook_trigger_inbox/check_outlook_trigger_inbox.py b/tests/tasks/uipath-maestro-flow/single_node/outlook_trigger_inbox/check_outlook_trigger_inbox.py index 7f268cda54..972dd6cd8d 100644 --- a/tests/tasks/uipath-maestro-flow/single_node/outlook_trigger_inbox/check_outlook_trigger_inbox.py +++ b/tests/tasks/uipath-maestro-flow/single_node/outlook_trigger_inbox/check_outlook_trigger_inbox.py @@ -4,6 +4,8 @@ Two checks: check_trigger_node Structural โ€” flow contains the email-received trigger. + check_folder_binding Structural โ€” trigger binds both an Outlook connection + and a non-empty MailFolder reference. check_folder_id_fresh Regression โ€” parentFolderId written into the flow is a live MailFolder ID on the currently-bound Outlook connection. Catches "agent resolved-but-reused a @@ -16,11 +18,13 @@ Privacy: never logs folder display names. Only counts + truncated IDs. """ -import glob import json -import os import subprocess import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) +from _shared.flow_check import find_flow_file # noqa: E402 CONNECTOR_KEY = "uipath-microsoft-outlook365" TRIGGER_TYPE_MARKER = "uipath.connector.trigger.uipath-microsoft-outlook365.email-received" @@ -75,11 +79,9 @@ def _uip_resources_run(tail_args: list[str]) -> dict: def _read_flow() -> tuple[dict, str]: - flows = glob.glob("**/OutlookTriggerInbox*.flow", recursive=True) - if not flows: - sys.exit("FAIL: no OutlookTriggerInbox*.flow found under cwd") - with open(flows[0]) as f: - return json.load(f), flows[0] + path = find_flow_file(flow_glob="OutlookTriggerInbox*.flow") + with open(path) as f: + return json.load(f), path def _find_test_folder_key() -> str: @@ -138,6 +140,19 @@ def check_trigger_node(): print("OK: Outlook email-received trigger node present") +def check_folder_binding(): + flow, _ = _read_flow() + trigger = _find_trigger_node(flow) + detail = trigger.get("inputs", {}).get("detail", {}) or {} + connection_id = detail.get("connectionId") + parent_folder_id = (detail.get("eventParameters") or {}).get("parentFolderId") + if not connection_id: + sys.exit("FAIL: trigger.inputs.detail.connectionId is missing") + if not parent_folder_id: + sys.exit("FAIL: trigger.inputs.detail.eventParameters.parentFolderId is missing") + print("OK: Outlook connection and MailFolder reference are both bound") + + # โ”€โ”€ subcommand: check_folder_id_fresh โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ def check_folder_id_fresh(): flow, _ = _read_flow() @@ -180,6 +195,7 @@ def check_folder_id_fresh(): DISPATCH = { "check_trigger_node": check_trigger_node, + "check_folder_binding": check_folder_binding, "check_folder_id_fresh": check_folder_id_fresh, } diff --git a/tests/tasks/uipath-maestro-flow/single_node/outlook_trigger_inbox/outlook_trigger_inbox.yaml b/tests/tasks/uipath-maestro-flow/single_node/outlook_trigger_inbox/outlook_trigger_inbox.yaml index 20d188f952..f17f40b6f7 100644 --- a/tests/tasks/uipath-maestro-flow/single_node/outlook_trigger_inbox/outlook_trigger_inbox.yaml +++ b/tests/tasks/uipath-maestro-flow/single_node/outlook_trigger_inbox/outlook_trigger_inbox.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-outlook-trigger-inbox description: > Regression test for PR #348: verifies the agent freshly resolves the @@ -67,16 +71,11 @@ success_criteria: weight: 0 pass_threshold: 0 - - type: command_executed - description: "Agent resolved MailFolder against the current --connection-id (PR #348 regression)" - tool_name: "Bash" - # `[\s\S]` matches across newlines โ€” agents often split `uip` commands with - # backslash line-continuation, which `.*` would miss. - # Verb alternation tolerates both the post-rename verb ("run", current) - # and the legacy "execute" verb that older CLIs in the sandbox PATH may - # still expose (MST-9674). - command_pattern: 'uip\s+is\s+resources\s+(?:run|execute)\s+list[\s\S]+?MailFolder[\s\S]+?--connection-id' - min_count: 1 + - type: run_command + description: "Flow carries a MailFolder reference on its bound Outlook connection" + command: "python3 $TASK_DIR/check_outlook_trigger_inbox.py check_folder_binding" + timeout: 30 + expected_exit_code: 0 weight: 2.5 pass_threshold: 1.0 diff --git a/tests/tasks/uipath-maestro-flow/single_node/outlook_trigger_inbox/test_check_outlook_trigger_inbox.py b/tests/tasks/uipath-maestro-flow/single_node/outlook_trigger_inbox/test_check_outlook_trigger_inbox.py index 60f0e27eda..5eb7330a26 100644 --- a/tests/tasks/uipath-maestro-flow/single_node/outlook_trigger_inbox/test_check_outlook_trigger_inbox.py +++ b/tests/tasks/uipath-maestro-flow/single_node/outlook_trigger_inbox/test_check_outlook_trigger_inbox.py @@ -132,6 +132,32 @@ def fake_run(_args, **_kwargs): checker.check_folder_id_fresh() # must NOT raise (id resolves on the connection) +def test_folder_binding_requires_connection_and_reference(monkeypatch) -> None: + checker = _load_checker() + monkeypatch.setattr( + checker, + "_read_flow", + lambda: ( + { + "nodes": [ + { + "type": checker.TRIGGER_TYPE_MARKER, + "inputs": { + "detail": { + "connectionId": "connection-1", + "eventParameters": {"parentFolderId": "mail-folder-1"}, + } + }, + } + ] + }, + "OutlookTriggerInbox.flow", + ), + ) + + checker.check_folder_binding() + + def test_folder_id_mismatch_fails_with_pr348_signature(monkeypatch) -> None: """If the flow's parentFolderId is not in the live ID set, the checker fails with the PR #348 regression marker (independent of the verb).""" diff --git a/tests/tasks/uipath-maestro-flow/single_node/outlook_waitfor_email/check_outlook_waitfor_email.py b/tests/tasks/uipath-maestro-flow/single_node/outlook_waitfor_email/check_outlook_waitfor_email.py index 0d6f7b397d..b797f13c72 100644 --- a/tests/tasks/uipath-maestro-flow/single_node/outlook_waitfor_email/check_outlook_waitfor_email.py +++ b/tests/tasks/uipath-maestro-flow/single_node/outlook_waitfor_email/check_outlook_waitfor_email.py @@ -24,9 +24,12 @@ MST-8802) cannot satisfy the check on its own. """ -import glob import json import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) +from _shared.flow_check import find_flow_file # noqa: E402 EVENT_MARKERS = ( "uipath.connector.event", @@ -71,10 +74,7 @@ def _filter_trees(detail: dict): def _read_flow() -> dict: - flows = glob.glob("**/OutlookWaitForEmail*.flow", recursive=True) - if not flows: - _fail("no OutlookWaitForEmail*.flow found under cwd") - with open(flows[0]) as f: + with open(find_flow_file(flow_glob="OutlookWaitForEmail*.flow")) as f: return json.load(f) diff --git a/tests/tasks/uipath-maestro-flow/single_node/outlook_waitfor_email/outlook_waitfor_email.yaml b/tests/tasks/uipath-maestro-flow/single_node/outlook_waitfor_email/outlook_waitfor_email.yaml index 52fc561599..8dc9c97eeb 100644 --- a/tests/tasks/uipath-maestro-flow/single_node/outlook_waitfor_email/outlook_waitfor_email.yaml +++ b/tests/tasks/uipath-maestro-flow/single_node/outlook_waitfor_email/outlook_waitfor_email.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-outlook-waitfor-email description: > Build-and-validate: a Flow with a manual start trigger, a mid-flow @@ -39,7 +43,7 @@ initial_prompt: | is received in the **Inbox** folder whose SUBJECT CONTAINS the string "TestWaitFor". Then End the flow. - Validate the flow. Do not debug the flow. + Validate the flow. Do NOT ask for approval, confirmation, or feedback. Do NOT pause between planning and implementation. Build the complete flow end-to-end in a single diff --git a/tests/tasks/uipath-maestro-flow/single_node/rpa/rpa.yaml b/tests/tasks/uipath-maestro-flow/single_node/rpa/rpa.yaml index 54904b6a08..77bfdda93b 100644 --- a/tests/tasks/uipath-maestro-flow/single_node/rpa/rpa.yaml +++ b/tests/tasks/uipath-maestro-flow/single_node/rpa/rpa.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-rpa description: > Create a UiPath Flow that uses the HelpDeskLookup RPA workflow to retrieve @@ -11,6 +15,8 @@ run_limits: task_timeout: 2700 # 1380s grading + a turn; inherited 1200 could not fit grading initial_prompt: | + Build the flow inside a solution of the same name. + Create a UiPath Flow project named "HelpDeskTicketTitle" that uses the HelpDeskLookup RPA workflow to retrieve the title for ticket 123 and return it as an output. diff --git a/tests/tasks/uipath-maestro-flow/single_node/subflow/subflow.yaml b/tests/tasks/uipath-maestro-flow/single_node/subflow/subflow.yaml index 7c359f77f6..1713f9b752 100644 --- a/tests/tasks/uipath-maestro-flow/single_node/subflow/subflow.yaml +++ b/tests/tasks/uipath-maestro-flow/single_node/subflow/subflow.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-subflow description: > Create a UiPath Flow that uses a Subflow node to encapsulate string-reversal @@ -12,6 +16,8 @@ run_limits: turn_timeout: 1200 initial_prompt: | + Build the flow inside a solution of the same name. + Create a UiPath Flow project named "SubflowDemo" that takes a string input called "text" and reverses it. The string reversal logic should be encapsulated inside a subflow rather than done directly in the main flow. diff --git a/tests/tasks/uipath-maestro-flow/single_node/switch/switch.yaml b/tests/tasks/uipath-maestro-flow/single_node/switch/switch.yaml index b35e47ae0b..30828035a9 100644 --- a/tests/tasks/uipath-maestro-flow/single_node/switch/switch.yaml +++ b/tests/tasks/uipath-maestro-flow/single_node/switch/switch.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-switch description: > Create a UiPath Flow that takes a quarter number (1-4) and uses a Switch node @@ -10,6 +14,8 @@ run_limits: turn_timeout: 1200 initial_prompt: | + Build the flow inside a solution of the same name. + Create a UiPath Flow project named "SeasonLookup" that takes a quarter number (1, 2, 3, or 4) as input and returns the corresponding season name: - 1 -> "Spring" diff --git a/tests/tasks/uipath-maestro-flow/single_node/terminate/terminate.yaml b/tests/tasks/uipath-maestro-flow/single_node/terminate/terminate.yaml index 33a2a66d03..366e44789e 100644 --- a/tests/tasks/uipath-maestro-flow/single_node/terminate/terminate.yaml +++ b/tests/tasks/uipath-maestro-flow/single_node/terminate/terminate.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-terminate description: > Create a UiPath Flow with two parallel branches from the trigger. One branch @@ -12,6 +16,8 @@ run_limits: turn_timeout: 1200 initial_prompt: | + Build the flow inside a solution of the same name. + Create a UiPath Flow project named "TerminateParallel" with two parallel branches from the trigger: diff --git a/tests/tasks/uipath-maestro-flow/single_node/test_transform_output_envelopes.py b/tests/tasks/uipath-maestro-flow/single_node/test_transform_output_envelopes.py new file mode 100644 index 0000000000..9e8ff2ec35 --- /dev/null +++ b/tests/tasks/uipath-maestro-flow/single_node/test_transform_output_envelopes.py @@ -0,0 +1,108 @@ +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path + +import pytest + +SINGLE_NODE = Path(__file__).resolve().parent + + +@pytest.mark.parametrize( + ("task", "flow_name", "node_type", "inputs"), + [ + ( + "transform_map", + "TransformMapDemo.flow", + "core.action.transform.map", + { + "collection": "$vars.people", + "operations": [ + { + "type": "map", + "config": { + "mappings": [{"field": "name", "transformation": "uppercase"}] + }, + } + ], + }, + ), + ( + "transform_filter", + "TransformFilterDemo.flow", + "core.action.transform.filter", + { + "collection": "$vars.items", + "operations": [ + { + "type": "filter", + "config": { + "filters": [ + {"field": "amount", "condition": "greater_equal", "value": 100} + ] + }, + } + ], + }, + ), + ( + "transform_group_by", + "TransformGroupByDemo.flow", + "core.action.transform.group-by", + { + "collection": "$vars.employees", + "operations": [ + { + "type": "groupBy", + "config": { + "groupByField": "department", + "aggregations": [{"operation": "count", "alias": "headcount"}], + }, + } + ], + }, + ), + ], +) +def test_transform_checkers_accept_sdk_literal_output_envelopes( + tmp_path: Path, task: str, flow_name: str, node_type: str, inputs: dict +) -> None: + flow = { + "nodes": [ + { + "id": "transform", + "type": node_type, + "typeVersion": "1.0.0", + "inputs": inputs, + "outputs": { + "output": { + "source": { + "type": "literal", + "expression": "=result.response", + "fieldType": "string", + } + }, + "error": { + "source": { + "type": "literal", + "expression": "=Error", + "fieldType": "string", + } + }, + }, + } + ] + } + (tmp_path / flow_name).write_text(json.dumps(flow)) + + result = subprocess.run( + [sys.executable, str(SINGLE_NODE / task / f"check_{task}_flow.py")], + cwd=tmp_path, + capture_output=True, + text=True, + check=False, + ) + + assert result.returncode == 0, result.stdout + result.stderr diff --git a/tests/tasks/uipath-maestro-flow/single_node/transform_filter/check_transform_filter_flow.py b/tests/tasks/uipath-maestro-flow/single_node/transform_filter/check_transform_filter_flow.py index 1faffe0118..ca123e2e63 100644 --- a/tests/tasks/uipath-maestro-flow/single_node/transform_filter/check_transform_filter_flow.py +++ b/tests/tasks/uipath-maestro-flow/single_node/transform_filter/check_transform_filter_flow.py @@ -23,11 +23,15 @@ NOT pin a specific version). """ -import glob import json import sys +from pathlib import Path from typing import NoReturn +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) +from _shared.advisory_flow_utils import unwrap # noqa: E402 +from _shared.flow_check import find_flow_file # noqa: E402 + NODE_TYPE = "core.action.transform.filter" EXPECTED_OUTPUT_SOURCE = "=result.response" EXPECTED_ERROR_SOURCE = "=Error" @@ -55,10 +59,7 @@ def _fail(msg: str) -> NoReturn: def _read_flow() -> dict: - flows = glob.glob("**/TransformFilterDemo*.flow", recursive=True) - if not flows: - _fail("no TransformFilterDemo*.flow found under cwd") - with open(flows[0]) as f: + with open(find_flow_file(flow_glob="TransformFilterDemo*.flow")) as f: return json.load(f) @@ -169,12 +170,12 @@ def _check_operations(inputs: dict) -> None: def _check_output_sources(node: dict) -> None: outputs = node.get("outputs") or {} - out_src = ((outputs.get("output")) or {}).get("source") + out_src = unwrap(((outputs.get("output")) or {}).get("source")) if out_src != EXPECTED_OUTPUT_SOURCE: _fail( f"outputs.output.source={out_src!r}; must be {EXPECTED_OUTPUT_SOURCE!r}" ) - err_src = ((outputs.get("error")) or {}).get("source") + err_src = unwrap(((outputs.get("error")) or {}).get("source")) if err_src != EXPECTED_ERROR_SOURCE: _fail( f"outputs.error.source={err_src!r}; must be {EXPECTED_ERROR_SOURCE!r}" diff --git a/tests/tasks/uipath-maestro-flow/single_node/transform_filter/transform_filter.yaml b/tests/tasks/uipath-maestro-flow/single_node/transform_filter/transform_filter.yaml index d1cdf46d46..1647384f26 100644 --- a/tests/tasks/uipath-maestro-flow/single_node/transform_filter/transform_filter.yaml +++ b/tests/tasks/uipath-maestro-flow/single_node/transform_filter/transform_filter.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-transform-filter description: > Create a UiPath Flow that uses a dedicated `core.action.transform.filter` @@ -34,14 +38,13 @@ initial_prompt: | templates. Use the exact condition name `greater_equal`. - Wire the node `outputs.output.source` to `=result.response` and `outputs.error.source` to `=Error`. - - Set `typeVersion` from the node registry for - `core.action.transform.filter` โ€” do not hardcode a guessed value. + - Set `typeVersion` from the authoritative definition for + `core.action.transform.filter`. - Do NOT run flow debug โ€” just validate the flow. + Validate the flow. Do NOT ask for approval, confirmation, or feedback. Do NOT pause between planning and implementation. Build the complete flow end-to-end in a single - pass. Before starting, follow the documented workflow steps exactly โ€” in particular the transform plugin's filter - node shape (`operations[].type == "filter"` with `config.filters`). + pass. success_criteria: # โ”€โ”€ Flow file validity โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ diff --git a/tests/tasks/uipath-maestro-flow/single_node/transform_group_by/check_transform_group_by_flow.py b/tests/tasks/uipath-maestro-flow/single_node/transform_group_by/check_transform_group_by_flow.py index 1719206f60..ae03d8c123 100644 --- a/tests/tasks/uipath-maestro-flow/single_node/transform_group_by/check_transform_group_by_flow.py +++ b/tests/tasks/uipath-maestro-flow/single_node/transform_group_by/check_transform_group_by_flow.py @@ -21,11 +21,15 @@ deliberately NOT pinned to a specific value here). """ -import glob import json import sys +from pathlib import Path from typing import NoReturn +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) +from _shared.advisory_flow_utils import unwrap # noqa: E402 +from _shared.flow_check import find_flow_file # noqa: E402 + NODE_TYPE = "core.action.transform.group-by" EXPECTED_OUTPUT_SOURCE = "=result.response" EXPECTED_ERROR_SOURCE = "=Error" @@ -36,10 +40,7 @@ def _fail(msg: str) -> NoReturn: def _read_flow() -> dict: - flows = glob.glob("**/TransformGroupByDemo*.flow", recursive=True) - if not flows: - _fail("no TransformGroupByDemo*.flow found under cwd") - with open(flows[0]) as f: + with open(find_flow_file(flow_glob="TransformGroupByDemo*.flow")) as f: return json.load(f) @@ -114,12 +115,12 @@ def _check_operations(inputs: dict) -> None: def _check_ports(node: dict) -> None: outputs = node.get("outputs") or {} - out_src = ((outputs.get("output") or {}).get("source")) + out_src = unwrap((outputs.get("output") or {}).get("source")) if out_src != EXPECTED_OUTPUT_SOURCE: _fail( f"outputs.output.source={out_src!r}; must be {EXPECTED_OUTPUT_SOURCE!r}." ) - err_src = ((outputs.get("error") or {}).get("source")) + err_src = unwrap((outputs.get("error") or {}).get("source")) if err_src != EXPECTED_ERROR_SOURCE: _fail( f"outputs.error.source={err_src!r}; must be {EXPECTED_ERROR_SOURCE!r}." diff --git a/tests/tasks/uipath-maestro-flow/single_node/transform_group_by/transform_group_by.yaml b/tests/tasks/uipath-maestro-flow/single_node/transform_group_by/transform_group_by.yaml index 1e9feaa5dd..20161257c6 100644 --- a/tests/tasks/uipath-maestro-flow/single_node/transform_group_by/transform_group_by.yaml +++ b/tests/tasks/uipath-maestro-flow/single_node/transform_group_by/transform_group_by.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-transform-group-by description: > Create a UiPath Flow with a single Group By transform node @@ -35,14 +39,12 @@ initial_prompt: | `average`) and a non-empty `alias` for the output field. - The node's `outputs.output.source` must be `=result.response` and `outputs.error.source` must be `=Error`. - - Set the node's `typeVersion` from the node registry for - `core.action.transform.group-by` โ€” do NOT hardcode a guessed version. + - Set the node's `typeVersion` from the authoritative definition for + `core.action.transform.group-by`. - Validate the flow. Do NOT run flow debug โ€” just validate. + Validate the flow. Do NOT ask for approval, confirmation, or feedback. Do NOT pause between planning and implementation. Build the complete flow end-to-end in a single pass. - Before starting, follow the documented workflow steps exactly โ€” in particular, the transform plugin's Group By JSON - shape and the collection-input contract. success_criteria: # -- Flow file validity -- diff --git a/tests/tasks/uipath-maestro-flow/single_node/transform_map/check_transform_map_flow.py b/tests/tasks/uipath-maestro-flow/single_node/transform_map/check_transform_map_flow.py index 2239a03655..cc377c09e7 100644 --- a/tests/tasks/uipath-maestro-flow/single_node/transform_map/check_transform_map_flow.py +++ b/tests/tasks/uipath-maestro-flow/single_node/transform_map/check_transform_map_flow.py @@ -21,11 +21,15 @@ pin a specific version string here). """ -import glob import json import sys +from pathlib import Path from typing import NoReturn +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) +from _shared.advisory_flow_utils import unwrap # noqa: E402 +from _shared.flow_check import find_flow_file # noqa: E402 + NODE_TYPE = "core.action.transform.map" EXPECTED_OUTPUT_SOURCE = "=result.response" EXPECTED_ERROR_SOURCE = "=Error" @@ -36,10 +40,7 @@ def _fail(msg: str) -> NoReturn: def _read_flow() -> dict: - flows = glob.glob("**/TransformMapDemo*.flow", recursive=True) - if not flows: - _fail("no TransformMapDemo*.flow found under cwd") - with open(flows[0]) as f: + with open(find_flow_file(flow_glob="TransformMapDemo*.flow")) as f: return json.load(f) @@ -105,12 +106,12 @@ def _check_operations(inputs: dict) -> None: def _check_output_sources(node: dict) -> None: outputs = node.get("outputs") or {} - out_src = ((outputs.get("output")) or {}).get("source") + out_src = unwrap(((outputs.get("output")) or {}).get("source")) if out_src != EXPECTED_OUTPUT_SOURCE: _fail( f"outputs.output.source={out_src!r}; must be {EXPECTED_OUTPUT_SOURCE!r}." ) - err_src = ((outputs.get("error")) or {}).get("source") + err_src = unwrap(((outputs.get("error")) or {}).get("source")) if err_src != EXPECTED_ERROR_SOURCE: _fail( f"outputs.error.source={err_src!r}; must be {EXPECTED_ERROR_SOURCE!r}." diff --git a/tests/tasks/uipath-maestro-flow/single_node/transform_map/transform_map.yaml b/tests/tasks/uipath-maestro-flow/single_node/transform_map/transform_map.yaml index 5ee7f6e3e0..6dfc758cef 100644 --- a/tests/tasks/uipath-maestro-flow/single_node/transform_map/transform_map.yaml +++ b/tests/tasks/uipath-maestro-flow/single_node/transform_map/transform_map.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-transform-map description: > Smoke test: agent builds a pure-OOTB UiPath Flow with a single @@ -32,15 +36,12 @@ initial_prompt: | `transformation: "uppercase"`. - The node `outputs.output.source` must be `=result.response` and `outputs.error.source` must be `=Error`. - - Set the node's `typeVersion` from the node registry for - `core.action.transform.map` โ€” do NOT hardcode a guessed version (the - skill's workflow teaches how to read the registry). + - Set the node's `typeVersion` from the authoritative definition for + `core.action.transform.map`. - Do NOT run flow debug โ€” just validate the flow. + Validate the flow. Do NOT ask for approval, confirmation, or feedback. Do NOT pause between planning and implementation. Build the complete flow end-to-end in a single pass. - Before starting, follow the documented workflow steps exactly โ€” in particular, the transform plugin's Map JSON - shape and the `collection` path contract. success_criteria: # โ”€โ”€ Flow file validity โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ diff --git a/tests/tasks/uipath-maestro-flow/smoke/check_merge_parallel_sync_flow.py b/tests/tasks/uipath-maestro-flow/smoke/check_merge_parallel_sync_flow.py index 55f221bece..00cb1c4a0c 100644 --- a/tests/tasks/uipath-maestro-flow/smoke/check_merge_parallel_sync_flow.py +++ b/tests/tasks/uipath-maestro-flow/smoke/check_merge_parallel_sync_flow.py @@ -22,12 +22,15 @@ source node that feeds the merge itself has an incoming edge. """ -import glob import json import sys from collections import Counter +from pathlib import Path from typing import NoReturn +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from _shared.flow_check import find_flow_file # noqa: E402 + NODE_TYPE = "core.logic.merge" @@ -36,14 +39,7 @@ def _fail(msg: str) -> NoReturn: def _read_flow() -> dict: - flows = sorted(glob.glob("**/ParallelSync*.flow", recursive=True)) - if not flows: - _fail("no ParallelSync*.flow found under cwd") - # Prefer the canonical project path; fall back to the first match so the - # read is deterministic even if stray *.flow files exist. - canonical = "ParallelSync/ParallelSync/ParallelSync.flow" - path = canonical if canonical in flows else flows[0] - with open(path) as f: + with open(find_flow_file(flow_glob="ParallelSync*.flow")) as f: return json.load(f) diff --git a/tests/tasks/uipath-maestro-flow/smoke/check_scheduled_trigger_flow.py b/tests/tasks/uipath-maestro-flow/smoke/check_scheduled_trigger_flow.py index 20735c65e6..4d6e4c23bc 100644 --- a/tests/tasks/uipath-maestro-flow/smoke/check_scheduled_trigger_flow.py +++ b/tests/tasks/uipath-maestro-flow/smoke/check_scheduled_trigger_flow.py @@ -34,12 +34,15 @@ - zero `definitions[]` entries with `nodeType == "core.trigger.manual"`. """ -import glob import json import re import sys +from pathlib import Path from typing import NoReturn +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from _shared.flow_check import find_flow_file # noqa: E402 + SCHEDULED = "core.trigger.scheduled" MANUAL = "core.trigger.manual" TRIGGER_PREFIX = "core.trigger." @@ -60,10 +63,7 @@ def _fail(msg: str) -> NoReturn: def _read_flow() -> dict: - flows = glob.glob("**/ScheduledReport*.flow", recursive=True) - if not flows: - _fail("no ScheduledReport*.flow found under cwd") - with open(flows[0]) as f: + with open(find_flow_file(flow_glob="ScheduledReport*.flow")) as f: return json.load(f) diff --git a/tests/tasks/uipath-maestro-flow/smoke/init_validate.yaml b/tests/tasks/uipath-maestro-flow/smoke/init_validate.yaml index 5b6bb7b6aa..8c26381441 100644 --- a/tests/tasks/uipath-maestro-flow/smoke/init_validate.yaml +++ b/tests/tasks/uipath-maestro-flow/smoke/init_validate.yaml @@ -1,8 +1,11 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-init-validate description: > - Skill-guided evaluation: agent uses the uipath-maestro-flow skill to create - a new UiPath Flow project inside a solution and validate it. Tests whether - the skill teaches the correct solution-first workflow and CLI usage. + Skill-guided evaluation: agent creates a new UiPath Flow project and + validates it. Tests whether the skill produces a valid Flow artifact. tags: [uipath-maestro-flow, smoke, lifecycle:generate] run_limits: @@ -12,36 +15,20 @@ initial_prompt: | Create a new UiPath Flow project called "WeatherAlert" and make sure it validates successfully. - A Flow project MUST be created - inside a solution: - 1. Create the solution first. - 2. Create the Flow project inside that solution. - 3. Confirm the flow project is registered in the parent solution. - - The correct flow-file path is: - WeatherAlert/WeatherAlert/WeatherAlert.flow - - The task is NOT complete until `uip maestro flow validate` has passed for - that exact file path. - - Important: - - The `uip` CLI is already available in the environment. - - Validate locally with `uip maestro flow validate`. - success_criteria: - type: command_executed - description: "Agent created a solution with uip solution init (or legacy new)" + description: "Advisory: live-v1 agent created a solution" tool_name: "Bash" command_pattern: '(uip|\$UIP)\s+solution\s+(init|new)' min_count: 1 weight: 1.5 pass_threshold: 0.0 # advisory: init auto-scaffolds the parent solution (CLI #2470) - - type: command_executed - description: "Agent initialized a Flow project with uip maestro flow init" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+(maestro\s+)?flow\s+init' - min_count: 1 + - type: run_command + description: "Initialized flow has the default manual trigger" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py --flow-name WeatherAlert 'core.trigger.manual'" + timeout: 30 + expected_exit_code: 0 weight: 1.5 pass_threshold: 1.0 @@ -53,10 +40,10 @@ success_criteria: weight: 1.5 pass_threshold: 1.0 - # Outcome-based check (auto-registration via `flow init` OR explicit - # `uip solution project add` both populate Projects[] in the .uipx). + # Solution registration is a live-v1 layout detail. Keep it visible without + # gating the SDK arm, whose native output is a root-level .flow artifact. - type: json_check - description: "Flow project is registered in the parent solution .uipx" + description: "Advisory: live-v1 Flow project is registered in the parent solution .uipx" path: "WeatherAlert/WeatherAlert.uipx" assertions: - expression: "length(Projects)" @@ -66,11 +53,12 @@ success_criteria: operator: "gte" expected: 1 weight: 1.0 - pass_threshold: 1.0 + pass_threshold: 0.0 - - type: file_exists - description: "Flow file was created inside the solution" - path: "WeatherAlert/WeatherAlert/WeatherAlert.flow" + - type: run_command + description: "Flow file was created in a supported authoring layout" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py --flow-name WeatherAlert" + timeout: 30 + expected_exit_code: 0 weight: 1.5 pass_threshold: 1.0 - diff --git a/tests/tasks/uipath-maestro-flow/smoke/inline_agent_robust.yaml b/tests/tasks/uipath-maestro-flow/smoke/inline_agent_robust.yaml index 777c9a77a1..bd65c34ca7 100644 --- a/tests/tasks/uipath-maestro-flow/smoke/inline_agent_robust.yaml +++ b/tests/tasks/uipath-maestro-flow/smoke/inline_agent_robust.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-inline-agent-robust description: > Skill-guided evaluation: agent uses the uipath-maestro-flow skill to build a @@ -12,55 +16,48 @@ run_limits: max_turns: 90 initial_prompt: | - Create a new UiPath Flow project called "EmailTriage" (inside a solution) and + Create a new UiPath Flow project called "EmailTriage" and add an inline agent node that classifies an inbound support email into a category and priority. - A Flow project MUST be created - inside a solution, so the inline agent lands at: - EmailTriage/EmailTriage//agent.json - Configure the inline agent to a production standard, not the scaffold default: - Pick an appropriate current model (do not keep the scaffold default). - Write a real system prompt for the classification task (no placeholder). - Declare a typed outputSchema (e.g. category, priority) โ€” not a bare `content` string. - The task is NOT complete until `uip maestro flow validate` passes for - EmailTriage/EmailTriage/EmailTriage.flow. - - Important: - - The `uip` CLI is already available in the environment. - - Validate locally with `uip maestro flow validate`. + Validate the flow. success_criteria: - - type: command_executed - description: "Agent scaffolded the inline agent with uip agent init --inline-in-flow" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+agent\s+init\s+.*--inline-in-flow' - min_count: 1 + - type: run_command + description: "Inline agent sidecar was created" + command: 'python3 "$SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/check_inline_agent.py" --check exists "**/agent.json"' + timeout: 15 + expected_exit_code: 0 weight: 1.5 pass_threshold: 1.0 - - type: command_executed - description: "Agent discovered available models with uip agent model list" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+agent\s+model\s+list' - min_count: 1 + - type: run_command + description: "Inline agent uses an explicitly selected current model" + command: 'python3 "$SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/check_inline_agent.py" --check model "**/agent.json"' + timeout: 15 + expected_exit_code: 0 weight: 1.0 pass_threshold: 1.0 - - type: file_exists - description: "Flow file was created inside the solution" - path: "EmailTriage/EmailTriage/EmailTriage.flow" + - type: run_command + description: "Flow file was created in a supported authoring layout" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py --flow-name EmailTriage" + timeout: 30 + expected_exit_code: 0 weight: 1.5 pass_threshold: 1.0 - - type: file_contains + - type: run_command description: "Flow contains the inline agent node type" - path: "EmailTriage/EmailTriage/EmailTriage.flow" - includes: - - '"uipath.agent.autonomous"' + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py --flow-name EmailTriage 'uipath.agent.autonomous'" + timeout: 30 + expected_exit_code: 0 weight: 2.0 pass_threshold: 1.0 @@ -68,11 +65,11 @@ success_criteria: # knowable โ€” the shared checker globs it (excluding generated .agent-builder/) # and asserts the three production-quality properties the skill now teaches: # model overridden off gpt-4o, real (non-placeholder) system prompt, and a - # typed outputSchema beyond bare `content`. $TASK_DIR is this YAML's dir; - # _shared is its sibling. + # typed outputSchema beyond bare `content`. Use the mounted checker repository + # because preview freezes only this task directory at $TASK_DIR. - type: run_command description: "Inline agent.json: model overridden, real system prompt, typed outputSchema" - command: 'python3 "$SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/check_inline_agent.py" "EmailTriage/EmailTriage/*/agent.json"' + command: 'python3 "$SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/check_inline_agent.py" "**/agent.json"' timeout: 15 expected_exit_code: 0 weight: 3.0 diff --git a/tests/tasks/uipath-maestro-flow/smoke/merge_parallel_sync.yaml b/tests/tasks/uipath-maestro-flow/smoke/merge_parallel_sync.yaml index 6e8af776a5..e3687a305c 100644 --- a/tests/tasks/uipath-maestro-flow/smoke/merge_parallel_sync.yaml +++ b/tests/tasks/uipath-maestro-flow/smoke/merge_parallel_sync.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-merge-parallel-sync description: > Build a UiPath Flow with two parallel branches that fork from the trigger and @@ -17,12 +21,6 @@ initial_prompt: | Create a UiPath Flow project named "ParallelSync" that runs two branches in parallel and synchronizes them with a Merge node before finishing. - A Flow project MUST live inside a solution: - 1. Create the solution first (`uip solution init`). - 2. Create the Flow project inside it (`uip maestro flow init`). The correct - flow-file path is: - ParallelSync/ParallelSync/ParallelSync.flow - Build this exact fork/join topology (the default manual trigger is fine โ€” keep it as the start node and use it as the fork): @@ -38,7 +36,7 @@ initial_prompt: | `display` object with a `label` โ€” validate rejects a node without it. - Wire these FIVE edges. Every edge needs both `sourcePort` and `targetPort` (validate rejects an edge missing either). Use the correct - port ids per the registry/skill: + port ids from the authoritative node definitions: - trigger `output` โ†’ branchA `input` - trigger `output` โ†’ branchB `input` (this fork = trigger has 2 outgoing edges) - branchA `success` โ†’ merge `input` @@ -46,23 +44,13 @@ initial_prompt: | - merge `output` โ†’ end `input` The two edges into the merge must have `targetNodeId` = the merge node id; the edge out of the merge must have `sourceNodeId` = the merge node id. - - Set every node's `typeVersion` from the registry โ€” do NOT hardcode guessed - versions (`uip maestro flow registry get core.logic.merge --output json`, - etc.). Append each node type's definition to `definitions[]` (the merge - definition carries `model.type: "bpmn:ParallelGateway"`). - - The task is NOT complete until `uip maestro flow validate` passes for that - exact flow-file path. + - Set every node's `typeVersion` from its authoritative node definition. + Include each node type's definition in `definitions[]`; the Merge + definition carries `model.type: "bpmn:ParallelGateway"`. - Important: - - The `uip` CLI is already available; use `--output json` on uip commands. - - After wiring, you may run `uip maestro flow format ` to normalize - layout/variables, then validate with `uip maestro flow validate` โ€” do NOT - run `uip maestro flow debug` (the parallel join only synchronizes in a - live engine). - - Do NOT ask for approval, confirmation, or feedback. Do NOT pause between - planning and implementation. Build the complete flow end-to-end in a - single pass. + Validate the flow. Do NOT ask for approval, confirmation, or feedback. Do + NOT pause between planning and implementation. Build the complete flow + end-to-end in a single pass. success_criteria: # โ”€โ”€ Flow file validity โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ diff --git a/tests/tasks/uipath-maestro-flow/smoke/registry_discovery.yaml b/tests/tasks/uipath-maestro-flow/smoke/registry_discovery.yaml index f944c14bb3..3c7426d74a 100644 --- a/tests/tasks/uipath-maestro-flow/smoke/registry_discovery.yaml +++ b/tests/tasks/uipath-maestro-flow/smoke/registry_discovery.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-registry-discovery description: > Skill-guided evaluation: agent uses the uipath-maestro-flow skill to explore @@ -10,48 +14,42 @@ run_limits: max_turns: 28 initial_prompt: | - I want to build a UiPath Flow that makes an HTTP request and processes the - response with a script. Before building anything, explore what node types - are available in the Flow registry and find the right ones for this task. - - Important: - - The `uip` CLI is already available in the environment. - - Start by refreshing the registry cache with `uip maestro flow registry pull --output json`. - - After list/search identifies the candidates, inspect the exact schemas with - `uip maestro flow registry get core.action.http --output json` and - `uip maestro flow registry get core.action.script --output json`. - - Use `--output json` on all uip commands. - - Do not build the flow โ€” just explore. + Explore the available Flow authoring capabilities for a flow that makes an + HTTP request and processes the response with a script. Identify the exact + HTTP and Script node types and inspect their input and output schemas. + Do not build the flow; report what you find. success_criteria: + # This task deliberately produces no artifact. Under ratified ruling D-2, + # v1-only registry telemetry remains report-only rather than gating the SDK arm. - type: command_executed - description: "Agent pulled registry data" + description: "Advisory: live-v1 agent refreshed registry data" tool_name: "Bash" command_pattern: 'uip\s+(maestro\s+)?flow\s+registry\s+pull' min_count: 1 weight: 1.0 - pass_threshold: 1.0 + pass_threshold: 0.0 - type: command_executed - description: "Agent listed or searched registry nodes" + description: "Advisory: live-v1 agent listed or searched registry nodes" tool_name: "Bash" command_pattern: 'uip\s+(maestro\s+)?flow\s+registry\s+(list|search)' min_count: 1 weight: 1.5 - pass_threshold: 1.0 + pass_threshold: 0.0 - type: command_executed - description: "Agent retrieved details on the HTTP node type via registry get" + description: "Advisory: live-v1 agent retrieved details on the HTTP node type" tool_name: "Bash" command_pattern: 'uip\s+(maestro\s+)?flow\s+registry\s+get\s+.*core\.action\.http' min_count: 1 weight: 1.5 - pass_threshold: 1.0 + pass_threshold: 0.0 - type: command_executed - description: "Agent retrieved details on the script node type via registry get" + description: "Advisory: live-v1 agent retrieved details on the Script node type" tool_name: "Bash" command_pattern: 'uip\s+(maestro\s+)?flow\s+registry\s+get\s+.*core\.action\.script' min_count: 1 weight: 1.5 - pass_threshold: 1.0 + pass_threshold: 0.0 diff --git a/tests/tasks/uipath-maestro-flow/smoke/scheduled_trigger.yaml b/tests/tasks/uipath-maestro-flow/smoke/scheduled_trigger.yaml index 665a01ecb7..b27517d4c8 100644 --- a/tests/tasks/uipath-maestro-flow/smoke/scheduled_trigger.yaml +++ b/tests/tasks/uipath-maestro-flow/smoke/scheduled_trigger.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-scheduled-trigger description: > Scaffold a UiPath Flow whose start node is a Scheduled Trigger @@ -17,40 +21,24 @@ initial_prompt: | Create a UiPath Flow project named "ScheduledReport" whose flow is started by a **scheduled trigger** that runs every hour. - A Flow project MUST live inside a solution: - 1. Create the solution first (`uip solution init`). - 2. Create the Flow project inside it (`uip maestro flow init`). The correct - flow-file path is: - ScheduledReport/ScheduledReport/ScheduledReport.flow - - `uip maestro flow init` scaffolds the flow with a default **manual** trigger - (`core.trigger.manual`). REPLACE that manual trigger with a **scheduled** - trigger so the flow's start node is a scheduled trigger: + Replace the default **manual** trigger (`core.trigger.manual`) with a + **scheduled** trigger so the flow's start node is a scheduled trigger: - The start node `type` must be `core.trigger.scheduled`. - Keep the existing `entryPointId` from the manual trigger's `inputs`. - `inputs.timerType` must be `"timeCycle"`. - `inputs.timerPreset` must be `"R/PT1H"` (every hour โ€” an ISO 8601 repeating interval). Do NOT use `"custom"` for this smoke test. - - Set the node `typeVersion` from the registry for `core.trigger.scheduled` - โ€” do NOT hardcode a guessed version - (`uip maestro flow registry get core.trigger.scheduled --output json`). + - Set the node `typeVersion` from its authoritative node definition. - Remove the `core.trigger.manual` node AND its `definitions[]` entry, and - append the `core.trigger.scheduled` definition from the registry get above + include the authoritative `core.trigger.scheduled` definition (it carries `model.type: "bpmn:StartEvent"` and `model.eventDefinition: "bpmn:TimerEventDefinition"`). A flow must have exactly one trigger. - The task is NOT complete until `uip maestro flow validate` passes for that - exact flow-file path. - - Important: - - The `uip` CLI is already available; use `--output json` on uip commands. - - Validate locally with `uip maestro flow validate` โ€” do NOT run - `uip maestro flow debug` (the schedule fires only in a live engine). - - Do NOT ask for approval, confirmation, or feedback. Do NOT pause between - planning and implementation. Build the complete flow end-to-end in a - single pass. + Validate the flow. Do NOT ask for approval, confirmation, or feedback. Do + NOT pause between planning and implementation. Build the complete flow + end-to-end in a single pass. success_criteria: # โ”€โ”€ Flow file validity โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ From f0fa034a26a00743fe2c282a11263534088b6fc8 Mon Sep 17 00:00:00 2001 From: Tao Ma <51425734+tmatup@users.noreply.github.com> Date: Fri, 21 Aug 2026 16:54:25 -0700 Subject: [PATCH 03/52] test(maestro-flow): align HITL and interactive task batch (#2749) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Apply the ratified same-ground recipe to the 16-task HITL and interactive batch. Preserve criterion intent and weights while replacing loop-coupled paths and telemetry with shared artifact checks. All required CI checks pass; the local paired run remains blocked by the recorded expired UiPath refresh token. ๐Ÿค– Generated with Claude Code Co-Authored-By: [Claude](mailto:noreply@anthropic.com) --- .../_shared/check_simulated_hitl.py | 151 ++++++++++++++++++ .../_shared/test_batch2_alignment.py | 112 +++++++++++++ .../_shared/test_same_ground_corpus.py | 14 +- .../hitl/quality_01_schema_design.yaml | 5 +- .../hitl/quality_02_result_downstream.yaml | 5 +- .../hitl/quality_03_boolean_decision.yaml | 5 +- .../hitl/quality_04_brownfield_insert.yaml | 5 +- .../hitl/smoke_01_hitl_node_placed.yaml | 5 +- .../hitl/smoke_02_completed_port_wired.yaml | 4 + .../hitl/smoke_03_multi_outcome_routing.yaml | 5 +- .../bellevue_weather_simulated.yaml | 23 +-- .../interactive/check_solution_select.py | 116 ++++++++++++++ .../cli_dice_roller_simulated.yaml | 30 ++-- .../check_escalation_behavior.py | 51 ++++-- .../customer_escalation_simulated.yaml | 15 +- .../customer_escalation_triage.yaml | 24 +-- .../expense_approval_simulated.yaml | 15 +- .../hitl_schema_design_simulated.yaml | 17 +- .../ixp_invoice_extraction_simulated.yaml | 15 +- .../slack_channel_description_simulated.yaml | 27 ++-- .../interactive/solution_select.yaml | 50 +++--- 21 files changed, 572 insertions(+), 122 deletions(-) create mode 100644 tests/tasks/uipath-maestro-flow/_shared/check_simulated_hitl.py create mode 100644 tests/tasks/uipath-maestro-flow/_shared/test_batch2_alignment.py create mode 100644 tests/tasks/uipath-maestro-flow/interactive/check_solution_select.py diff --git a/tests/tasks/uipath-maestro-flow/_shared/check_simulated_hitl.py b/tests/tasks/uipath-maestro-flow/_shared/check_simulated_hitl.py new file mode 100644 index 0000000000..1658dda302 --- /dev/null +++ b/tests/tasks/uipath-maestro-flow/_shared/check_simulated_hitl.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python3 +"""Check source semantics for the simulated HITL tasks.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +from typing import Any + +from flow_check import find_flow_file + + +def fail(message: str) -> None: + raise SystemExit(f"FAIL: {message}") + + +def load_flow() -> tuple[dict[str, Any], list[dict[str, Any]]]: + path = Path(find_flow_file()) + try: + flow = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + fail(f"could not read {path}: {error}") + nodes = flow.get("nodes") + if not isinstance(nodes, list): + fail(f"{path} has no nodes array") + return flow, nodes + + +def hitl_nodes(nodes: list[dict[str, Any]]) -> list[dict[str, Any]]: + return [ + node + for node in nodes + if str(node.get("type", "")).startswith("uipath.human-in-the-loop") + ] + + +def field_text(field: dict[str, Any], key: str) -> str: + value = field.get(key) + return value.lower() if isinstance(value, str) else "" + + +def check_schema(flow: dict[str, Any], nodes: list[dict[str, Any]]) -> None: + candidates = hitl_nodes(nodes) + if not candidates: + fail("need a HITL quickform node") + fields = (candidates[0].get("inputs") or {}).get("schema", {}).get("fields", []) + directions = {field.get("direction") for field in fields} + if "input" not in directions: + fail("need an input-direction (read-only) field") + if not directions.intersection({"output", "inOut"}): + fail("need an output/inOut (fill-in) field") + rendered = json.dumps(flow).lower() + if "approve" not in rendered or "reject" not in rendered: + fail("need Approve and Reject outcomes") + print("OK: HITL schema has input/output fields and Approve/Reject outcomes") + + +def check_priority(_flow: dict[str, Any], nodes: list[dict[str, Any]]) -> None: + candidates = hitl_nodes(nodes) + if not candidates: + fail("need a HITL quickform node") + if "high" not in json.dumps(candidates[0]).lower(): + fail("priority should be High") + print("OK: HITL priority is High") + + +def check_expense(flow: dict[str, Any], nodes: list[dict[str, Any]]) -> None: + edges = flow.get("edges") + if not isinstance(edges, list): + fail("Flow must contain edges[]") + candidates = hitl_nodes(nodes) + if len(candidates) != 1: + fail(f"need exactly 1 HITL node, found {len(candidates)}") + hitl = candidates[0] + hitl_id = hitl.get("id") + schema = (hitl.get("inputs") or {}).get("schema", {}) + fields = schema.get("fields", []) + outcomes = schema.get("outcomes", []) or [] + if not fields: + fail("HITL needs fields") + if not any( + (field_text(field, "id") == "amount" or "amount" in field_text(field, "label")) + and field.get("type") == "number" + for field in fields + ): + fail("amount must be a number field") + decision_fields = [ + field + for field in fields + if field.get("direction") in {"output", "inOut"} + and any( + key in field_text(field, "id") + for key in ("approval", "approved", "decision") + ) + ] + outcome_names = { + str(outcome.get("name") or outcome.get("id") or "").lower() + for outcome in outcomes + } + has_decision = any(field.get("type") == "boolean" for field in decision_fields) + has_outcomes = any("approve" in name for name in outcome_names) and any( + "reject" in name for name in outcome_names + ) + if not (has_decision or has_outcomes): + fail("need a boolean decision field or approve/reject outcomes") + reason_keys = ("reason", "comment", "explanation", "justification", "note") + if not any( + field.get("direction") in {"output", "inOut"} + and field.get("type") in {"text", "string", "textarea"} + and any( + key in field_text(field, "id") or key in field_text(field, "label") + for key in reason_keys + ) + for field in fields + ): + fail("need a text reason output field") + if not any( + edge.get("sourceNodeId") == hitl_id + and edge.get("sourcePort") in {"completed", "outcome-completed"} + for edge in edges + ): + fail("HITL completed handle must be wired") + scripts = [ + str((node.get("inputs") or {}).get("script", "")) + for node in nodes + if node.get("type") == "core.action.script" + ] + expected = f"$vars.{hitl_id}.output" + if not any(expected in script for script in scripts): + fail(f"downstream script must read HITL output via {expected}") + print("OK: expense HITL schema, routing, and downstream output access are correct") + + +CHECKS = { + "expense": check_expense, + "priority": check_priority, + "schema": check_schema, +} + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("check", choices=sorted(CHECKS)) + args = parser.parse_args() + flow, nodes = load_flow() + CHECKS[args.check](flow, nodes) + + +if __name__ == "__main__": + main() diff --git a/tests/tasks/uipath-maestro-flow/_shared/test_batch2_alignment.py b/tests/tasks/uipath-maestro-flow/_shared/test_batch2_alignment.py new file mode 100644 index 0000000000..b74c52544b --- /dev/null +++ b/tests/tasks/uipath-maestro-flow/_shared/test_batch2_alignment.py @@ -0,0 +1,112 @@ +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path + +SHARED = Path(__file__).resolve().parent +FLOW_TASKS = SHARED.parent + + +def run_script(script: Path, *args: str, cwd: Path) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, str(script), *args], + cwd=cwd, + capture_output=True, + text=True, + check=False, + ) + + +def test_simulated_hitl_checks_accept_root_sdk_emit(tmp_path: Path) -> None: + flow = { + "nodes": [ + { + "id": "review", + "type": "uipath.human-in-the-loop", + "inputs": { + "priority": "High", + "schema": { + "fields": [ + { + "id": "amount", + "label": "Amount", + "type": "number", + "direction": "input", + }, + { + "id": "reason", + "label": "Reason", + "type": "text", + "direction": "output", + }, + ], + "outcomes": [{"name": "Approve"}, {"name": "Reject"}], + }, + }, + }, + { + "id": "log", + "type": "core.action.script", + "inputs": {"script": "return $vars.review.output.reason;"}, + }, + ], + "edges": [ + { + "sourceNodeId": "review", + "sourcePort": "completed", + "targetNodeId": "log", + "targetPort": "input", + } + ], + } + (tmp_path / "Review.flow").write_text(json.dumps(flow)) + script = SHARED / "check_simulated_hitl.py" + + for check in ("expense", "priority", "schema"): + result = run_script(script, check, cwd=tmp_path) + assert result.returncode == 0, result.stdout + result.stderr + + +def test_solution_select_checks_use_selected_solution(tmp_path: Path) -> None: + for name in ("SolarReports", "TideTracker"): + directory = tmp_path / name + directory.mkdir() + (directory / f"{name}.uipx").write_text(json.dumps({"Projects": []})) + + selected = tmp_path / "WeatherSelection-7K4M" + project = selected / "WeatherAlert" + project.mkdir(parents=True) + (selected / "WeatherSelection-7K4M.uipx").write_text( + json.dumps({"Projects": [{"Name": "WeatherAlert", "Type": "Flow"}]}) + ) + (project / "project.uiproj").write_text( + json.dumps({"Name": "WeatherAlert", "ProjectType": "Flow"}) + ) + (project / "WeatherAlert.flow").write_text(json.dumps({"nodes": []})) + script = FLOW_TASKS / "interactive" / "check_solution_select.py" + + for check in ( + "existing-untouched", + "flow", + "no-extra-solution", + "project", + "solution", + ): + result = run_script(script, check, cwd=tmp_path) + assert result.returncode == 0, result.stdout + result.stderr + + +def test_solution_select_advisory_reports_extra_default_solution( + tmp_path: Path, +) -> None: + extra = tmp_path / "WeatherAlertSol" + extra.mkdir() + (extra / "WeatherAlertSol.uipx").write_text(json.dumps({"Projects": []})) + script = FLOW_TASKS / "interactive" / "check_solution_select.py" + + result = run_script(script, "no-extra-solution", cwd=tmp_path) + + assert result.returncode != 0 + assert "unexpected default solution" in result.stderr diff --git a/tests/tasks/uipath-maestro-flow/_shared/test_same_ground_corpus.py b/tests/tasks/uipath-maestro-flow/_shared/test_same_ground_corpus.py index 83e7bea3ca..3aa1e41572 100644 --- a/tests/tasks/uipath-maestro-flow/_shared/test_same_ground_corpus.py +++ b/tests/tasks/uipath-maestro-flow/_shared/test_same_ground_corpus.py @@ -31,7 +31,6 @@ "evaluate/local_crud.yaml", "evaluate/no_auto_upload.yaml", "evaluate/simulation/simulation_crud.yaml", - "interactive/solution_select.yaml", "ixp/e2e_02_project_selection.yaml", "ixp/integration_handle_routing.yaml", "ixp/routing.yaml", @@ -41,7 +40,6 @@ } SKILL_TELEMETRY_ALLOWLIST = { - "interactive/customer_escalation_triage/customer_escalation_triage.yaml", "ixp/e2e_02_project_selection.yaml", "ixp/integration_handle_routing.yaml", "ixp/scaffold_minimal.yaml", @@ -87,10 +85,6 @@ "edit/move_node/move_node.yaml", "edit/remove_node/remove_node.yaml", "edit/update_node/update_node.yaml", - "interactive/bellevue_weather_simulated/bellevue_weather_simulated.yaml", - "interactive/cli_dice_roller_simulated/cli_dice_roller_simulated.yaml", - "interactive/customer_escalation_triage/customer_escalation_triage.yaml", - "interactive/slack_channel_description_simulated/slack_channel_description_simulated.yaml", "connector_features/jdbc_databricks_query/jdbc_databricks_query.yaml", } @@ -119,6 +113,10 @@ def _criterion_blocks(text: str) -> list[tuple[str, str]]: ] +def _prompt_text(text: str) -> str: + return "\n".join((_block(text, "initial_prompt"), _block(text, "simulation"))) + + def _tagged_tasks() -> list[tuple[str, Path, str]]: tasks = [] for path in sorted(FLOW_TASKS.rglob("*.yaml")): @@ -219,7 +217,7 @@ def test_gating_skill_telemetry_matches_the_temporary_allowlist() -> None: def test_forbidden_prompt_phrases_match_the_temporary_allowlist() -> None: offenders = set() for relative, _, text in _tagged_tasks(): - prompt = " ".join(_block(text, "initial_prompt").lower().split()) + prompt = " ".join(_prompt_text(text).lower().split()) if re.search(r"do not [^.]*\bdebug\b", prompt) or UUID_PATTERN.search(prompt): offenders.add(relative) assert offenders == FORBIDDEN_PROMPT_ALLOWLIST | FORBIDDEN_PROMPT_EXCEPTIONS @@ -230,7 +228,7 @@ def test_debug_graded_prompts_name_a_same_name_solution() -> None: for relative, path, text in _tagged_tasks(): if not _is_debug_graded(path, text): continue - prompt = " ".join(_block(text, "initial_prompt").lower().split()) + prompt = " ".join(_prompt_text(text).lower().split()) if not re.search(r"\b(?:in|inside) a solution of the same name\b", prompt): offenders.add(relative) assert offenders == DEBUG_SOLUTION_ALLOWLIST | DEBUG_PROJECT_LAYOUT_EXCEPTIONS diff --git a/tests/tasks/uipath-maestro-flow/hitl/quality_01_schema_design.yaml b/tests/tasks/uipath-maestro-flow/hitl/quality_01_schema_design.yaml index e900adaa8f..35055f356b 100644 --- a/tests/tasks/uipath-maestro-flow/hitl/quality_01_schema_design.yaml +++ b/tests/tasks/uipath-maestro-flow/hitl/quality_01_schema_design.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-hitl-quality-schema-design description: > Quality test: agent correctly maps a business description to a quickform @@ -66,4 +70,3 @@ success_criteria: expected_exit_code: 0 weight: 1.5 pass_threshold: 1.0 - diff --git a/tests/tasks/uipath-maestro-flow/hitl/quality_02_result_downstream.yaml b/tests/tasks/uipath-maestro-flow/hitl/quality_02_result_downstream.yaml index 89a41eda97..8ee9b07d94 100644 --- a/tests/tasks/uipath-maestro-flow/hitl/quality_02_result_downstream.yaml +++ b/tests/tasks/uipath-maestro-flow/hitl/quality_02_result_downstream.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-hitl-quality-result-downstream description: > Quality test: agent correctly references the HITL node's output via @@ -60,4 +64,3 @@ success_criteria: expected_exit_code: 0 weight: 1.5 pass_threshold: 1.0 - diff --git a/tests/tasks/uipath-maestro-flow/hitl/quality_03_boolean_decision.yaml b/tests/tasks/uipath-maestro-flow/hitl/quality_03_boolean_decision.yaml index 51d392354a..a9bc305813 100644 --- a/tests/tasks/uipath-maestro-flow/hitl/quality_03_boolean_decision.yaml +++ b/tests/tasks/uipath-maestro-flow/hitl/quality_03_boolean_decision.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-hitl-quality-boolean-decision description: > Quality test: agent correctly wires a boolean HITL output field into a @@ -61,4 +65,3 @@ success_criteria: expected_exit_code: 0 weight: 2.0 pass_threshold: 1.0 - diff --git a/tests/tasks/uipath-maestro-flow/hitl/quality_04_brownfield_insert.yaml b/tests/tasks/uipath-maestro-flow/hitl/quality_04_brownfield_insert.yaml index 818884c36e..72b7c02a44 100644 --- a/tests/tasks/uipath-maestro-flow/hitl/quality_04_brownfield_insert.yaml +++ b/tests/tasks/uipath-maestro-flow/hitl/quality_04_brownfield_insert.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-hitl-quality-brownfield-insert description: > Quality test: agent inserts a HITL node into an existing flow without @@ -70,4 +74,3 @@ success_criteria: expected_exit_code: 0 weight: 1.5 pass_threshold: 1.0 - diff --git a/tests/tasks/uipath-maestro-flow/hitl/smoke_01_hitl_node_placed.yaml b/tests/tasks/uipath-maestro-flow/hitl/smoke_01_hitl_node_placed.yaml index f2d648f834..a2870b6d74 100644 --- a/tests/tasks/uipath-maestro-flow/hitl/smoke_01_hitl_node_placed.yaml +++ b/tests/tasks/uipath-maestro-flow/hitl/smoke_01_hitl_node_placed.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-hitl-smoke-node-placed description: > Smoke test: agent builds a simple invoice approval flow containing an inline @@ -48,4 +52,3 @@ success_criteria: expected_exit_code: 0 weight: 3.0 pass_threshold: 1.0 - diff --git a/tests/tasks/uipath-maestro-flow/hitl/smoke_02_completed_port_wired.yaml b/tests/tasks/uipath-maestro-flow/hitl/smoke_02_completed_port_wired.yaml index bcbbb20af0..2e89626275 100644 --- a/tests/tasks/uipath-maestro-flow/hitl/smoke_02_completed_port_wired.yaml +++ b/tests/tasks/uipath-maestro-flow/hitl/smoke_02_completed_port_wired.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-hitl-smoke-completed-port description: > Smoke test: agent wires the HITL node's `outcome-completed` output port diff --git a/tests/tasks/uipath-maestro-flow/hitl/smoke_03_multi_outcome_routing.yaml b/tests/tasks/uipath-maestro-flow/hitl/smoke_03_multi_outcome_routing.yaml index 5adffb2d7b..6edae84d35 100644 --- a/tests/tasks/uipath-maestro-flow/hitl/smoke_03_multi_outcome_routing.yaml +++ b/tests/tasks/uipath-maestro-flow/hitl/smoke_03_multi_outcome_routing.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-hitl-smoke-multi-outcome-routing description: > Smoke test: agent builds a flow where a Decision node reads the HITL @@ -70,4 +74,3 @@ success_criteria: expected_exit_code: 0 weight: 2.0 pass_threshold: 1.0 - diff --git a/tests/tasks/uipath-maestro-flow/interactive/bellevue_weather_simulated/bellevue_weather_simulated.yaml b/tests/tasks/uipath-maestro-flow/interactive/bellevue_weather_simulated/bellevue_weather_simulated.yaml index 65c5c86a50..1498ba5949 100644 --- a/tests/tasks/uipath-maestro-flow/interactive/bellevue_weather_simulated/bellevue_weather_simulated.yaml +++ b/tests/tasks/uipath-maestro-flow/interactive/bellevue_weather_simulated/bellevue_weather_simulated.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-bellevue-weather-simulated description: > Bellevue weather flow (HTTP โ†’ script โ†’ decision), but driven by a simulated @@ -36,10 +40,10 @@ simulation: or below 60 should say 'bring a jacket'; and you want a short written summary of the weather rather than raw numbers. Say that it has to be built as a UiPath Maestro Flow โ€” not an API workflow, not a standalone - script โ€” and that you want it validated before you sign off. Do not name - files or paths and do not use technical wording. Close by inviting the - agent to ask you anything it needs before it starts building. Keep the - withheld items below to yourself until it asks. + script โ€” inside a solution of the same name, and that you want it validated + before you sign off. Do not name files or paths and do not use technical + wording. Close by inviting the agent to ask you anything it needs before it + starts building. Keep the withheld items below to yourself until it asks. WITHHELD โ€” reveal each of these ONLY if the agent asks about it: - The project name should be "BellevueWeather". @@ -62,12 +66,11 @@ simulation: n_trials: 1 success_criteria: - # Name-agnostic: any .flow the agent built must validate. The exact project - # name is scored separately below so a working-but-misnamed flow still earns - # most of the credit. (`**` skips dot-dirs, so .uipath/.skills flows are ignored.) + # Name-agnostic: any unambiguous generated .flow must validate. The exact + # project name is scored separately below. - type: run_command description: "uip flow validate passes (any .flow)" - command: "python3 -c \"import glob,subprocess,sys; flows=glob.glob('**/*.flow',recursive=True); sys.exit('no .flow file found') if not flows else sys.exit(subprocess.run(['uip','maestro','flow','validate',flows[0],'--output','json']).returncode)\"" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py" timeout: 60 expected_exit_code: 0 weight: 3.0 @@ -76,7 +79,7 @@ success_criteria: # Runtime check: - type: run_command description: "Flow debug runs; weather-API node executed and output contains a verdict" - command: "python3 $TASK_DIR/check_weather_flow.py" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/interactive/bellevue_weather_simulated/check_weather_flow.py" timeout: 600 expected_exit_code: 0 weight: 5.0 @@ -89,7 +92,7 @@ success_criteria: # keeps it from dominating the score. - type: run_command description: "Project/flow is named BellevueWeather (elicited from the user)" - command: "python3 -c \"import glob,sys; sys.exit(0 if glob.glob('**/BellevueWeather.flow',recursive=True) else 'expected a flow named BellevueWeather.flow')\"" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py --flow-name BellevueWeather" timeout: 15 expected_exit_code: 0 weight: 1.0 diff --git a/tests/tasks/uipath-maestro-flow/interactive/check_solution_select.py b/tests/tasks/uipath-maestro-flow/interactive/check_solution_select.py new file mode 100644 index 0000000000..97b559facf --- /dev/null +++ b/tests/tasks/uipath-maestro-flow/interactive/check_solution_select.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +"""Check solution-selection outcomes without depending on authoring commands.""" + +from __future__ import annotations + +import argparse +import json +import subprocess +import sys +from pathlib import Path + +SHARED = Path(__file__).resolve().parents[1] / "_shared" +sys.path.insert(0, str(SHARED)) +from flow_check import find_flow_file, find_project_dir # noqa: E402 + +SELECTED = Path("WeatherSelection-7K4M") +PROJECT_PATTERN = "WeatherSelection-7K4M/**/project.uiproj" + + +def fail(message: str) -> None: + raise SystemExit(f"FAIL: {message}") + + +def selected_flow() -> Path: + path = Path(find_flow_file(PROJECT_PATTERN, flow_glob="WeatherAlert.flow")) + if SELECTED not in path.parents: + fail(f"WeatherAlert.flow is outside {SELECTED}: {path}") + return path + + +def read_solution(path: Path) -> dict: + try: + value = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + fail(f"could not read {path}: {error}") + if not isinstance(value, dict): + fail(f"{path} is not a JSON object") + return value + + +def check_flow() -> None: + print(f"OK: selected solution contains {selected_flow()}") + + +def check_existing_untouched() -> None: + for name in ("SolarReports", "TideTracker"): + path = Path(name) / f"{name}.uipx" + projects = read_solution(path).get("Projects") + if projects != []: + fail(f"{path} should still contain no projects, found {projects!r}") + print("OK: pre-existing solutions contain no registered projects") + + +def check_project() -> None: + project_dir = Path(find_project_dir(PROJECT_PATTERN)) + if SELECTED not in project_dir.parents: + fail(f"Flow project is outside {SELECTED}: {project_dir}") + print(f"OK: selected solution contains a Flow project at {project_dir}") + + +def check_solution() -> None: + path = SELECTED / f"{SELECTED.name}.uipx" + projects = read_solution(path).get("Projects") + if not isinstance(projects, list) or not any( + project.get("Type") == "Flow" + for project in projects + if isinstance(project, dict) + ): + fail(f"{path} does not register a Flow project") + print(f"OK: {path} registers a Flow project") + + +def check_no_extra_solution() -> None: + allowed = {"SolarReports", "TideTracker", SELECTED.name} + extras = sorted( + str(path) + for path in Path.cwd().glob("*/*.uipx") + if path.parent.name not in allowed + ) + if extras: + fail(f"unexpected default solution created before selection: {extras}") + print("OK: no extra default solution was created") + + +def check_validate() -> None: + path = selected_flow() + result = subprocess.run( + ["uip", "maestro", "flow", "validate", str(path), "--output", "json"], + capture_output=True, + text=True, + ) + sys.stdout.write(result.stdout) + sys.stderr.write(result.stderr) + if result.returncode: + raise SystemExit(result.returncode) + + +CHECKS = { + "existing-untouched": check_existing_untouched, + "flow": check_flow, + "no-extra-solution": check_no_extra_solution, + "project": check_project, + "solution": check_solution, + "validate": check_validate, +} + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("check", choices=sorted(CHECKS)) + args = parser.parse_args() + CHECKS[args.check]() + + +if __name__ == "__main__": + main() diff --git a/tests/tasks/uipath-maestro-flow/interactive/cli_dice_roller_simulated/cli_dice_roller_simulated.yaml b/tests/tasks/uipath-maestro-flow/interactive/cli_dice_roller_simulated/cli_dice_roller_simulated.yaml index a1a872f20d..dc759b052b 100644 --- a/tests/tasks/uipath-maestro-flow/interactive/cli_dice_roller_simulated/cli_dice_roller_simulated.yaml +++ b/tests/tasks/uipath-maestro-flow/interactive/cli_dice_roller_simulated/cli_dice_roller_simulated.yaml @@ -1,6 +1,10 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-cli-dice-roller-simulated description: > - Dice-roller flow via CLI mode, but driven by a simulated non-technical user + Dice-roller flow in interactive mode, driven by a simulated non-technical user who withholds requirements until asked. Tests the agent's ability to clarify ambiguous asks before building. tags: [uipath-maestro-flow, e2e, mode:build, lifecycle:generate, shape:multi-node, simulation] @@ -40,16 +44,15 @@ simulation: die, six sides, numbers 1 through 6, and you want the rolled number shown as the result rather than buried somewhere nobody looks. Say that it has to be built as a UiPath Maestro Flow โ€” not an API workflow, not a - standalone script โ€” and that you want it validated before you sign off. Do - not name files or paths and do not use technical wording. How the agent - builds it is its own business โ€” you have no opinion on CLI commands vs - editing files. Close by inviting the agent to ask you anything it needs - before it starts building. Keep the withheld items below to yourself until - it asks. + standalone script โ€” inside a solution of the same name, and that you want + it validated before you sign off. Do not name files or paths and do not use + technical wording. How the agent builds it is its own business. Close by + inviting the agent to ask you anything it needs before it starts building. + Keep the withheld items below to yourself until it asks. WITHHELD โ€” reveal each of these ONLY if the agent asks about it: - The project name should be "DiceRoller". - - You want "uip maestro flow validate" to pass before you sign off. + - You want the completed flow to validate before you sign off. - You want to see it actually produce a number when it runs before you sign off. constraints: @@ -66,12 +69,11 @@ simulation: n_trials: 1 success_criteria: - # Name-agnostic: any .flow the agent built must validate. The exact project - # name is scored separately below so a working-but-misnamed flow still earns - # most of the credit. (`**` skips dot-dirs, so .uipath/.skills flows are ignored.) + # Name-agnostic: any unambiguous generated .flow must validate. The exact + # project name is scored separately below. - type: run_command description: "uip flow validate passes (any .flow)" - command: "python3 -c \"import glob,subprocess,sys; flows=glob.glob('**/*.flow',recursive=True); sys.exit('no .flow file found') if not flows else sys.exit(subprocess.run(['uip','maestro','flow','validate',flows[0],'--output','json']).returncode)\"" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py" timeout: 60 expected_exit_code: 0 weight: 4.0 @@ -82,7 +84,7 @@ success_criteria: # static grep for "random", which a flow that never executes would still pass. - type: run_command description: "Flow debug runs; Script node executed and output is an integer in [1,6]" - command: "python3 $TASK_DIR/check_dice_runs.py" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/interactive/cli_dice_roller_simulated/check_dice_runs.py" # Cloud `flow debug` on this task regularly exceeds the old 240s inner # budget (measured: 2 of 3 codex runs raised subprocess.TimeoutExpired), so # the criterion was scoring 0 on runner latency rather than on the flow. @@ -99,7 +101,7 @@ success_criteria: # keeps it from dominating the score. - type: run_command description: "Project/flow is named DiceRoller (elicited from the user)" - command: "python3 -c \"import glob,sys; sys.exit(0 if glob.glob('**/DiceRoller.flow',recursive=True) else 'expected a flow named DiceRoller.flow')\"" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py --flow-name DiceRoller" timeout: 15 expected_exit_code: 0 weight: 1.0 diff --git a/tests/tasks/uipath-maestro-flow/interactive/customer_escalation_simulated/check_escalation_behavior.py b/tests/tasks/uipath-maestro-flow/interactive/customer_escalation_simulated/check_escalation_behavior.py index e3ab96f525..2a0fba0bc0 100644 --- a/tests/tasks/uipath-maestro-flow/interactive/customer_escalation_simulated/check_escalation_behavior.py +++ b/tests/tasks/uipath-maestro-flow/interactive/customer_escalation_simulated/check_escalation_behavior.py @@ -34,13 +34,16 @@ from __future__ import annotations -import glob import json +import os import sys from pathlib import Path from typing import Any, NoReturn -FLOW_GLOB = "**/*.flow" +sys.path.insert( + 0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +) +from _shared.flow_check import find_flow_file # noqa: E402 def fail(msg: str) -> NoReturn: @@ -48,10 +51,7 @@ def fail(msg: str) -> NoReturn: def load_flow() -> dict[str, Any]: - matches = [m for m in glob.glob(FLOW_GLOB, recursive=True) if "/." not in "/" + m] - if not matches: - fail(f"No .flow file found (searched {FLOW_GLOB})") - path = Path(sorted(matches)[0]) + path = Path(find_flow_file()) try: return json.loads(path.read_text()) except json.JSONDecodeError as exc: @@ -75,9 +75,12 @@ def node_refs(node: dict, *needles: str) -> bool: return any(needle in blob for needle in needles) -def any_node_refs(nodes: list[dict], *needles: str, exclude_triggers: bool = False) -> bool: +def any_node_refs( + nodes: list[dict], *needles: str, exclude_triggers: bool = False +) -> bool: return any( - (not exclude_triggers or not is_trigger(n)) and node_refs(n, *needles) for n in nodes + (not exclude_triggers or not is_trigger(n)) and node_refs(n, *needles) + for n in nodes ) @@ -102,17 +105,27 @@ def main() -> None: or "agent" in node_type(n) ] if not routing_nodes: - fail("No routing construct found (need a decision / switch / if / agent-based classifier)") + fail( + "No routing construct found (need a decision / switch / if / agent-based classifier)" + ) def out_edge_count(node_id: Any) -> int: return sum( 1 for e in edges - if node_id in (e.get("sourceNodeId"), e.get("source"), e.get("from"), e.get("sourceId")) + if node_id + in ( + e.get("sourceNodeId"), + e.get("source"), + e.get("from"), + e.get("sourceId"), + ) ) max_branches = max((out_edge_count(n.get("id")) for n in routing_nodes), default=0) - terminals = [n for n in nodes if "end" in node_type(n) or "terminate" in node_type(n)] + terminals = [ + n for n in nodes if "end" in node_type(n) or "terminate" in node_type(n) + ] if max_branches < 2 and len(terminals) < 2: fail("Routing does not fan out into >=2 branches (VIP-urgent vs standard)") @@ -120,18 +133,26 @@ def out_edge_count(node_id: Any) -> int: if not any_node_refs(nodes, "urgent", "urgen"): fail("No 'urgency' signal on any node (flow should classify emails by urgency)") if not any_node_refs(nodes, "vip"): - fail("No 'VIP' signal on any node (flow should classify the sender as VIP or not)") + fail( + "No 'VIP' signal on any node (flow should classify the sender as VIP or not)" + ) # 4. Slack notification (VIP+urgent branch) if not any_node_refs(nodes, "slack"): fail("No Slack node (the high-touch branch should notify on Slack)") # 5. Reply to sender by email on a non-trigger node - if not any_node_refs(nodes, "outlook", "office365", "graph.microsoft.com", exclude_triggers=True): - fail("No non-trigger Outlook/Graph email-reply node (flow should reply to the sender)") + if not any_node_refs( + nodes, "outlook", "office365", "graph.microsoft.com", exclude_triggers=True + ): + fail( + "No non-trigger Outlook/Graph email-reply node (flow should reply to the sender)" + ) # 6. Support ticket on the standard branch (connector or a script that builds one) - if not any_node_refs(nodes, "jira", "servicenow", "create-issue", "createissue", "ticket"): + if not any_node_refs( + nodes, "jira", "servicenow", "create-issue", "createissue", "ticket" + ): fail("No support-ticket node (standard branch should create a ticket)") print( diff --git a/tests/tasks/uipath-maestro-flow/interactive/customer_escalation_simulated/customer_escalation_simulated.yaml b/tests/tasks/uipath-maestro-flow/interactive/customer_escalation_simulated/customer_escalation_simulated.yaml index 4f11364ba1..ad8208940e 100644 --- a/tests/tasks/uipath-maestro-flow/interactive/customer_escalation_simulated/customer_escalation_simulated.yaml +++ b/tests/tasks/uipath-maestro-flow/interactive/customer_escalation_simulated/customer_escalation_simulated.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-customer-escalation-simulated description: > Multi-branch Outlookโ†’classifyโ†’decision escalation flow, but driven by a @@ -67,12 +71,11 @@ simulation: n_trials: 1 success_criteria: - # Name-agnostic: any .flow the agent built must validate. The exact project - # name is scored separately below so a working-but-misnamed flow still earns - # most of the credit. (`**` skips dot-dirs, so .uipath/.skills flows are ignored.) + # Name-agnostic: any unambiguous generated .flow must validate. The exact + # project name is scored separately below. - type: run_command description: "uip flow validate passes (any .flow)" - command: "python3 -c \"import glob,subprocess,sys; flows=glob.glob('**/*.flow',recursive=True); sys.exit('no .flow file found') if not flows else sys.exit(subprocess.run(['uip','maestro','flow','validate',flows[0],'--output','json']).returncode)\"" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py" timeout: 60 expected_exit_code: 0 weight: 5.0 @@ -86,7 +89,7 @@ success_criteria: # provide. See check_escalation_behavior.py for the rationale. - type: run_command description: "Flow performs the triage: trigger, urgency+VIP routing, Slack notify, email reply, ticket creation" - command: "python3 $TASK_DIR/check_escalation_behavior.py" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/interactive/customer_escalation_simulated/check_escalation_behavior.py" timeout: 30 expected_exit_code: 0 weight: 10.0 @@ -99,7 +102,7 @@ success_criteria: # keeps it from dominating the score. - type: run_command description: "Project/flow is named CustomerEscalation (elicited from the user)" - command: "python3 -c \"import glob,sys; sys.exit(0 if glob.glob('**/CustomerEscalation.flow',recursive=True) else 'expected a flow named CustomerEscalation.flow')\"" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py --flow-name CustomerEscalation" timeout: 15 expected_exit_code: 0 weight: 1.0 diff --git a/tests/tasks/uipath-maestro-flow/interactive/customer_escalation_triage/customer_escalation_triage.yaml b/tests/tasks/uipath-maestro-flow/interactive/customer_escalation_triage/customer_escalation_triage.yaml index f59d6fc0b1..8a2da15031 100644 --- a/tests/tasks/uipath-maestro-flow/interactive/customer_escalation_triage/customer_escalation_triage.yaml +++ b/tests/tasks/uipath-maestro-flow/interactive/customer_escalation_triage/customer_escalation_triage.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-interactive-customer-escalation-triage description: > Interactive end-to-end Flow evaluation. A simulated support-operations @@ -47,40 +51,40 @@ simulation: solution of the same name, that classifies an escalation and returns these named outputs: severity, engineeringNeeded, responseMode, and caseKey. The flow inputs are customerTier, productionDown, workaroundAvailable, - businessImpact, and correlationId. The completed flow must validate. Do not - ask the coding agent to run it; the grader supplies seeded cases and runs it. + businessImpact, and correlationId. The completed flow must validate. The + grader supplies seeded cases and handles execution after validation. constraints: - "In the opening request, state the goal and that you need a UiPath Maestro Flow validated before sign-off. State the flow's input and output contract using these EXACT identifiers, verbatim and unaltered โ€” do not paraphrase, rename, abbreviate, or re-case any of them, and tell the coding agent the flow's declared inputs and outputs must use these names exactly as written because downstream systems consume them by name: the inputs are customerTier, productionDown, workaroundAvailable, businessImpact, and correlationId; the outputs are severity, engineeringNeeded, responseMode, and caseKey. Also state, in business terms and in your own words, the policy it has to encode: an Enterprise customer with production unavailable and no workaround is Sev1; informational or routine follow-up with no production impact is Sev3; engineering gets pulled in for Sev1 but not for Sev3; customer acknowledgements are always drafted for a human to review and never sent automatically, so the response mode is Draft; and the incoming correlationId must be carried through as the case key so downstream systems can correlate the run. Make clear that businessImpact arrives as free-form text the requester writes in their own words โ€” a short phrase or sentence describing the impact, not a fixed keyword or code โ€” so its exact wording varies from request to request. Close by inviting the coding agent to ask you anything it needs before it starts building." - "Do NOT volunteer the Sev2 rules. Only if the agent asks about degraded production, partial impact, or what happens when a workaround exists: Sev2 means production is degraded or unavailable but a viable workaround exists, and engineering is needed for Sev2 as well." - "If asked a finite-choice question, choose the option matching these policies." - "Keep each reply concise and internally consistent." - - "If asked for permission to run or debug the flow, decline and say the grader will execute it after validation." + - "If asked about execution, say the grader will handle it after validation." - "If the coding agent asks you anything before it starts building, answer every outstanding question in ONE message and then let it build." - "The moment the coding agent reports the flow is built and validated, the conversation is over: reply with exactly <<>> on its own line. Do not ask for a rebuild, polish, re-checks, or alternative designs, and do not raise a requirement you have not mentioned yet โ€” the grader verifies the result from here." max_turns: 8 success_criteria: - type: skill_triggered - description: "Agent invoked the uipath-maestro-flow skill" + description: "Advisory: agent invoked the uipath-maestro-flow skill" skill_name: "uipath-maestro-flow" expected_skill: "uipath-maestro-flow" weight: 1.0 - pass_threshold: 1.0 + pass_threshold: 0.0 - type: command_not_executed - description: "Agent left execution to the grader instead of running flow debug" + description: "Advisory: agent left execution to the grader instead of running flow debug" tool_name: "Bash" command_pattern: '(uip|\$UIP|\$\{?UIP\}?)\s+(maestro\s+)?flow\s+debug' weight: 1.0 - pass_threshold: 1.0 + pass_threshold: 0.0 - type: command_executed - description: "Agent validated the generated Flow" + description: "Advisory: agent validated the generated Flow" tool_name: "Bash" command_pattern: '(uip|\$UIP|\$\{?UIP\}?)\s+(maestro\s+)?flow\s+validate' min_count: 1 weight: 1.5 - pass_threshold: 1.0 + pass_threshold: 0.0 - type: run_command description: "Generated customer-escalation Flow validates without errors" @@ -92,7 +96,7 @@ success_criteria: - type: run_command description: "Seeded Sev1 and Sev3 runs produce the elicited severity, engineering, draft-response, and correlation outcomes" - command: "python3 $TASK_DIR/check_customer_escalation_triage.py" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/interactive/customer_escalation_triage/check_customer_escalation_triage.py" timeout: 1320 # budget-guard: manual x2. 2 x debug_budget(300) = 1210 + margin expected_exit_code: 0 weight: 8.0 diff --git a/tests/tasks/uipath-maestro-flow/interactive/expense_approval_simulated/expense_approval_simulated.yaml b/tests/tasks/uipath-maestro-flow/interactive/expense_approval_simulated/expense_approval_simulated.yaml index 53664bb840..8c9541cbcc 100644 --- a/tests/tasks/uipath-maestro-flow/interactive/expense_approval_simulated/expense_approval_simulated.yaml +++ b/tests/tasks/uipath-maestro-flow/interactive/expense_approval_simulated/expense_approval_simulated.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-expense-approval-simulated description: > Expense-approval flow with an inline HITL review step (trigger โ†’ script โ†’ @@ -69,12 +73,11 @@ simulation: n_trials: 1 success_criteria: - # Name-agnostic: any .flow the agent built must validate. The exact project - # name is scored separately below so a working-but-misnamed flow still earns - # most of the credit. (`**` skips dot-dirs, so .uipath/.skills flows are ignored.) + # Name-agnostic: any unambiguous generated .flow must validate. The exact + # project name is scored separately below. - type: run_command description: "uip flow validate passes (any .flow)" - command: "python3 -c \"import glob,subprocess,sys; flows=glob.glob('**/*.flow',recursive=True); sys.exit('no .flow file found') if not flows else sys.exit(subprocess.run(['uip','maestro','flow','validate',flows[0],'--output','json']).returncode)\"" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py" timeout: 60 expected_exit_code: 0 weight: 3.0 @@ -87,7 +90,7 @@ success_criteria: # $vars..output. - type: run_command description: "Flow has a correct inline HITL schema, decision capture, completed wiring, and downstream .output access" - command: "python3 -c \"import json,glob; flows=glob.glob('**/*.flow',recursive=True); assert flows,'No flow file'; f=json.load(open(flows[0])); nodes=f['nodes']; edges=f.get('edges',[]); hitl=[n for n in nodes if str(n.get('type','')).startswith('uipath.human-in-the-loop')]; assert len(hitl)==1,'need exactly 1 HITL node'; h=hitl[0]; hid=h.get('id'); schema=h.get('inputs',{}).get('schema',{}); fields=schema.get('fields',[]); outcomes=schema.get('outcomes',[]) or []; assert fields,'HITL needs fields'; ft=lambda fl,k:(fl.get(k) or '').lower() if isinstance(fl.get(k),str) else ''; assert any((ft(x,'id')=='amount' or 'amount' in ft(x,'label')) and x.get('type')=='number' for x in fields),'amount must be a number field'; dec=[x for x in fields if x.get('direction') in ('output','inOut') and any(k in ft(x,'id') for k in ('approval','approved','decision'))]; onames={str(o.get('name') or o.get('id') or '').lower() for o in outcomes}; okdec=any(x.get('type')=='boolean' for x in dec) or (any('approve' in o for o in onames) and any('reject' in o for o in onames)); assert okdec,'need boolean decision field or approve/reject outcomes'; rk=('reason','comment','explanation','justification','note'); assert any(x.get('direction') in ('output','inOut') and x.get('type') in ('text','string','textarea') and any(k in ft(x,'id') or k in ft(x,'label') for k in rk) for x in fields),'need a text reason output field'; assert any(e.get('sourceNodeId')==hid and e.get('sourcePort') in ('completed','outcome-completed') for e in edges),'HITL completed handle must be wired'; scripts=[str(n.get('inputs',{}).get('script','')) for n in nodes if n.get('type')=='core.action.script']; assert any((chr(36)+'vars.'+str(hid)+'.output') in s for s in scripts),'downstream script must read HITL output via vars..output'; print('OK')\"" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/check_simulated_hitl.py expense" timeout: 15 expected_exit_code: 0 weight: 6.0 @@ -100,7 +103,7 @@ success_criteria: # keeps it from dominating the score. - type: run_command description: "Project/flow is named ExpenseApproval (elicited from the user)" - command: "python3 -c \"import glob,sys; sys.exit(0 if glob.glob('**/ExpenseApproval.flow',recursive=True) else 'expected a flow named ExpenseApproval.flow')\"" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py --flow-name ExpenseApproval" timeout: 15 expected_exit_code: 0 weight: 1.0 diff --git a/tests/tasks/uipath-maestro-flow/interactive/hitl_schema_design_simulated/hitl_schema_design_simulated.yaml b/tests/tasks/uipath-maestro-flow/interactive/hitl_schema_design_simulated/hitl_schema_design_simulated.yaml index 2969c12a83..28617e906c 100644 --- a/tests/tasks/uipath-maestro-flow/interactive/hitl_schema_design_simulated/hitl_schema_design_simulated.yaml +++ b/tests/tasks/uipath-maestro-flow/interactive/hitl_schema_design_simulated/hitl_schema_design_simulated.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-hitl-schema-design-simulated description: > Purchase-order review flow with an inline HITL quickform (trigger โ†’ script โ†’ @@ -68,12 +72,11 @@ simulation: n_trials: 1 success_criteria: - # Name-agnostic: any .flow the agent built must validate. The exact project - # name is scored separately below so a working-but-misnamed flow still earns - # most of the credit. (`**` skips dot-dirs, so .uipath/.skills flows are ignored.) + # Name-agnostic: any unambiguous generated .flow must validate. The exact + # project name is scored separately below. - type: run_command description: "uip flow validate passes (any .flow)" - command: "python3 -c \"import glob,subprocess,sys; flows=glob.glob('**/*.flow',recursive=True); sys.exit('no .flow file found') if not flows else sys.exit(subprocess.run(['uip','maestro','flow','validate',flows[0],'--output','json']).returncode)\"" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py" timeout: 60 expected_exit_code: 0 weight: 3.0 @@ -85,7 +88,7 @@ success_criteria: # below so a missing priority does not zero out an otherwise-correct schema. - type: run_command description: "HITL quickform has input + output fields and Approve/Reject outcomes" - command: "python3 -c \"import json,glob; flows=glob.glob('**/*.flow',recursive=True); assert flows,'No flow file'; f=json.load(open(flows[0])); nodes=f['nodes']; hitl=[n for n in nodes if str(n.get('type','')).startswith('uipath.human-in-the-loop')]; assert hitl,'need a HITL quickform node'; fields=hitl[0].get('inputs',{}).get('schema',{}).get('fields',[]); dirs={x.get('direction') for x in fields}; assert 'input' in dirs,'need an input-direction (read-only) field'; assert 'output' in dirs or 'inOut' in dirs,'need an output/inOut (fill-in) field'; s=json.dumps(f).lower(); assert 'approve' in s and 'reject' in s,'need Approve and Reject outcomes'; print('OK')\"" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/check_simulated_hitl.py schema" timeout: 15 expected_exit_code: 0 weight: 4.0 @@ -97,7 +100,7 @@ success_criteria: # expected to elicit urgency when the persona flags the review as time-sensitive). - type: run_command description: "HITL review priority is set to High (elicited from the user)" - command: "python3 -c \"import json,glob; flows=glob.glob('**/*.flow',recursive=True); assert flows,'No flow file'; f=json.load(open(flows[0])); hitl=[n for n in f['nodes'] if str(n.get('type','')).startswith('uipath.human-in-the-loop')]; assert hitl,'need a HITL quickform node'; assert 'high' in json.dumps(hitl[0]).lower(),'priority should be High'; print('OK')\"" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/check_simulated_hitl.py priority" timeout: 15 expected_exit_code: 0 weight: 1.5 @@ -110,7 +113,7 @@ success_criteria: # keeps it from dominating the score. - type: run_command description: "Project/flow is named VendorPOReview (elicited from the user)" - command: "python3 -c \"import glob,sys; sys.exit(0 if glob.glob('**/VendorPOReview.flow',recursive=True) else 'expected a flow named VendorPOReview.flow')\"" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py --flow-name VendorPOReview" timeout: 15 expected_exit_code: 0 weight: 1.0 diff --git a/tests/tasks/uipath-maestro-flow/interactive/ixp_invoice_extraction_simulated/ixp_invoice_extraction_simulated.yaml b/tests/tasks/uipath-maestro-flow/interactive/ixp_invoice_extraction_simulated/ixp_invoice_extraction_simulated.yaml index 941c937101..8b129c81ae 100644 --- a/tests/tasks/uipath-maestro-flow/interactive/ixp_invoice_extraction_simulated/ixp_invoice_extraction_simulated.yaml +++ b/tests/tasks/uipath-maestro-flow/interactive/ixp_invoice_extraction_simulated/ixp_invoice_extraction_simulated.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-ixp-invoice-extraction-simulated description: > Invoice-processing flow (SharePoint trigger โ†’ IxP extraction โ†’ HTTP POST to @@ -72,12 +76,11 @@ simulation: n_trials: 1 success_criteria: - # Name-agnostic: any .flow the agent built must validate. The exact project - # name is scored separately below so a working-but-misnamed flow still earns - # most of the credit. (`**` skips dot-dirs, so .uipath/.skills flows are ignored.) + # Name-agnostic: any unambiguous generated .flow must validate. The exact + # project name is scored separately below. - type: run_command description: "uip flow validate passes (any .flow)" - command: "python3 -c \"import glob,subprocess,sys; flows=glob.glob('**/*.flow',recursive=True); sys.exit('no .flow file found') if not flows else sys.exit(subprocess.run(['uip','maestro','flow','validate',flows[0],'--output','json']).returncode)\"" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py" timeout: 60 expected_exit_code: 0 weight: 3.0 @@ -88,7 +91,7 @@ success_criteria: # node, and a $vars. reference consuming the extraction output downstream. - type: run_command description: "Flow has IxP extraction + SharePoint + HTTP nodes and a $vars downstream reference" - command: "python3 -c \"import json,glob; flows=glob.glob('**/*.flow',recursive=True); assert flows,'No flow file'; txt=open(flows[0]).read(); low=txt.lower(); f=json.loads(txt); types=[str(n.get('type','')).lower() for n in f.get('nodes',[])]; assert 'uipath.ixp' in low or 'core.logic.mock' in low,'no IxP extraction node (or mock fallback)'; assert 'sharepoint' in low,'no SharePoint trigger/connector'; assert any('http' in t for t in types),'no HTTP action node'; assert (chr(36)+'vars.') in txt,'no vars downstream reference to extraction output'; print('OK')\"" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py '$vars.' --regex '(?i)(uipath\\.ixp|core\\.logic\\.mock)' --regex '(?i)sharepoint' --regex '(?i)\"type\"\\s*:\\s*\"[^\"]*http'" timeout: 15 expected_exit_code: 0 weight: 6.0 @@ -101,7 +104,7 @@ success_criteria: # keeps it from dominating the score. - type: run_command description: "Project/flow is named InvoiceProcessing (elicited from the user)" - command: "python3 -c \"import glob,sys; sys.exit(0 if glob.glob('**/InvoiceProcessing.flow',recursive=True) else 'expected a flow named InvoiceProcessing.flow')\"" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py --flow-name InvoiceProcessing" timeout: 15 expected_exit_code: 0 weight: 1.0 diff --git a/tests/tasks/uipath-maestro-flow/interactive/slack_channel_description_simulated/slack_channel_description_simulated.yaml b/tests/tasks/uipath-maestro-flow/interactive/slack_channel_description_simulated/slack_channel_description_simulated.yaml index cae6ae91fe..422667cf98 100644 --- a/tests/tasks/uipath-maestro-flow/interactive/slack_channel_description_simulated/slack_channel_description_simulated.yaml +++ b/tests/tasks/uipath-maestro-flow/interactive/slack_channel_description_simulated/slack_channel_description_simulated.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-slack-channel-description-simulated description: > Single-connector Slack flow (read a channel's description and output it), @@ -37,11 +41,11 @@ simulation: you: the channel is #office-bellevue in Slack, you want that channel's description text as the result, and it has to work on its own without you supplying the channel every time it runs. Say that it has to be built as a - UiPath Maestro Flow โ€” not an API workflow, not a standalone script โ€” and - that you want it validated before you sign off. Do not name files or paths - and do not use technical wording. Close by inviting the agent to ask you - anything it needs before it starts building. Keep the withheld items below - to yourself until it asks. + UiPath Maestro Flow โ€” not an API workflow, not a standalone script โ€” inside + a solution of the same name, and that you want it validated before you sign + off. Do not name files or paths and do not use technical wording. Close by + inviting the agent to ask you anything it needs before it starts building. + Keep the withheld items below to yourself until it asks. WITHHELD โ€” reveal each of these ONLY if the agent asks about it: - The project name should be "SlackChannelDescription". @@ -62,12 +66,11 @@ simulation: n_trials: 1 success_criteria: - # Name-agnostic: any .flow the agent built must validate. The exact project - # name is scored separately below so a working-but-misnamed flow still earns - # most of the credit. (`**` skips dot-dirs, so .uipath/.skills flows are ignored.) + # Name-agnostic: any unambiguous generated .flow must validate. The exact + # project name is scored separately below. - type: run_command description: "uip flow validate passes (any .flow)" - command: "python3 -c \"import glob,subprocess,sys; flows=glob.glob('**/*.flow',recursive=True); sys.exit('no .flow file found') if not flows else sys.exit(subprocess.run(['uip','maestro','flow','validate',flows[0],'--output','json']).returncode)\"" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py" timeout: 60 expected_exit_code: 0 weight: 3.0 @@ -81,7 +84,7 @@ success_criteria: # unresolved channel fails here regardless of what the static check below reports. - type: run_command description: "Flow debug runs; Slack connector executed and output has the channel description" - command: "python3 $TASK_DIR/check_channel_description.py" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/interactive/slack_channel_description_simulated/check_channel_description.py" timeout: 600 expected_exit_code: 0 weight: 5.0 @@ -107,7 +110,7 @@ success_criteria: # hard-fail the whole task โ€” see the block comment; scored for signal only. - type: run_command description: "Flow targets the #office-bellevue channel (elicited from the user)" - command: "python3 -c \"import glob,re,sys; flows=glob.glob('**/*.flow',recursive=True); t=(open(flows[0]).read() if flows else ''); sys.exit('no .flow file found') if not flows else sys.exit(0 if ('office-bellevue' in t.lower() or re.search(r'conversationsInfoId[^A-Za-z0-9]+C[A-Z0-9]{6,}', t)) else 'flow neither names #office-bellevue nor hardwires a resolved Slack channel ID')\"" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py --regex '(?i)(office-bellevue|conversationsInfoId[^A-Za-z0-9]+C[A-Z0-9]{6,})'" timeout: 15 expected_exit_code: 0 weight: 3.0 @@ -120,7 +123,7 @@ success_criteria: # keeps it from dominating the score. - type: run_command description: "Project/flow is named SlackChannelDescription (elicited from the user)" - command: "python3 -c \"import glob,sys; sys.exit(0 if glob.glob('**/SlackChannelDescription.flow',recursive=True) else 'expected a flow named SlackChannelDescription.flow')\"" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py --flow-name SlackChannelDescription" timeout: 15 expected_exit_code: 0 weight: 1.0 diff --git a/tests/tasks/uipath-maestro-flow/interactive/solution_select.yaml b/tests/tasks/uipath-maestro-flow/interactive/solution_select.yaml index 29b8b58fb6..8a50c5af48 100644 --- a/tests/tasks/uipath-maestro-flow/interactive/solution_select.yaml +++ b/tests/tasks/uipath-maestro-flow/interactive/solution_select.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-solution-select-ask description: > Interactive-mode variant of init-validate. The working directory already @@ -39,7 +43,7 @@ simulation: You are a UiPath developer working in a directory that already holds a couple of unrelated solutions (SolarReports and TideTracker). You want a fresh, separate solution for this new work and you trust the agent to - drive the CLI. + drive the work. goal: | Get a new Flow project called WeatherAlert created in a separate solution and validated successfully. @@ -69,19 +73,20 @@ success_criteria: weight: 3.0 pass_threshold: 1.0 - - type: file_exists + - type: run_command description: "Flow file created inside the user-selected solution" - path: "WeatherSelection-7K4M/WeatherAlert/WeatherAlert.flow" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/interactive/check_solution_select.py flow" + timeout: 30 + expected_exit_code: 0 weight: 2.0 pass_threshold: 1.0 # โ”€โ”€ Did NOT silently adopt an existing solution โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ - - type: command_executed + - type: run_command description: "Agent did not init the Flow project inside a pre-existing solution" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+(maestro\s+)?flow\s+init\b.*(SolarReports|TideTracker)' - min_count: 0 - max_count: 0 + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/interactive/check_solution_select.py existing-untouched" + timeout: 30 + expected_exit_code: 0 weight: 1.5 pass_threshold: 1.0 @@ -108,32 +113,33 @@ success_criteria: # โ”€โ”€ Build is valid in the selected solution โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ - type: run_command description: "Flow validates in the selected solution" - command: "uip maestro flow validate WeatherSelection-7K4M/WeatherAlert/WeatherAlert.flow --output json" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/interactive/check_solution_select.py validate" timeout: 60 expected_exit_code: 0 weight: 1.5 pass_threshold: 1.0 - - type: command_executed - description: "Agent initialized a Flow project with uip maestro flow init" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+(maestro\s+)?flow\s+init' - min_count: 1 + - type: run_command + description: "Selected solution contains an initialized Flow project" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/interactive/check_solution_select.py project" + timeout: 30 + expected_exit_code: 0 weight: 1.0 pass_threshold: 1.0 # โ”€โ”€ Advisory: recorded in the report, never gates the run โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ - - type: command_executed - description: "Agent created a solution with uip solution init (or legacy new)" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+solution\s+(init|new)' - min_count: 1 + - type: run_command + description: "Agent created the selected solution artifact" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/interactive/check_solution_select.py solution" + timeout: 30 + expected_exit_code: 0 weight: 1.0 pass_threshold: 0.0 # advisory: outcome already enforced by the WeatherSelection-7K4M json_check; records the explicit-init convention only - - type: command_not_executed + - type: run_command description: "Advisory: no default WeatherAlert solution before the name was given โ€” records the default-then-repair inefficiency; nondeterministic on simulator name-timing, so non-gating" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+solution\s+(init|new)\s+["'']?WeatherAlert(?:["'']?\s|$)' + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/interactive/check_solution_select.py no-extra-solution" + timeout: 30 + expected_exit_code: 0 weight: 1.0 pass_threshold: 0.0 From 7d57a6d16162425e0431436759bd1c3c69118b11 Mon Sep 17 00:00:00 2001 From: Tao Ma <51425734+tmatup@users.noreply.github.com> Date: Fri, 21 Aug 2026 17:15:35 -0700 Subject: [PATCH 04/52] test(maestro-flow): align bindings and connector features (#2750) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Apply the ratified same-ground recipe to the 26-task bindings and connector-features batch. ๐Ÿค– Generated with Claude Code Co-Authored-By: [Claude](mailto:noreply@anthropic.com) --- .../_shared/test_batch3_alignment.py | 112 ++++++++++++++++++ .../_shared/test_same_ground_corpus.py | 15 --- .../bindings/check_bindings.py | 18 +++ .../bindings/idempotent_reconfigure.yaml | 38 +++--- .../multi_connector_independence.yaml | 42 ++++--- .../no_duplicate_connection_bindings.yaml | 35 +++--- .../reconfigure_different_connection.yaml | 65 ++++------ .../connector_features/ceql_where.yaml | 18 +-- .../check_ceql_where_flow.py | 7 +- .../connector_features/check_enum_flow.py | 19 +-- .../check_managed_http_fallback.py | 15 +-- .../check_multiselect_flow.py | 28 +++-- .../check_path_param_value.py | 17 +-- .../connector_features/complex_array.yaml | 22 ++-- .../dtl_load_by_default_false.yaml | 19 +-- .../dtl_load_by_default_true.yaml | 17 +-- .../connector_features/enhanced_enum.yaml | 19 +-- .../connector_features/enum.yaml | 28 +++-- .../check_generic_dynamic_node.py | 13 +- .../generic_dynamic_node.yaml | 17 ++- .../check_databricks_query.py | 7 +- .../jdbc_databricks_query.yaml | 23 ++-- .../connector_features/multiselect.yaml | 19 +-- .../check_non_catalog_http_fallback.py | 16 +-- .../non_catalog_http_fallback.yaml | 17 +-- .../connector_features/path_params.yaml | 27 +++-- .../connector_features/query_params.yaml | 19 +-- .../connector_features/searchable_joins.yaml | 19 +-- .../check_slack_http_fallback.py | 14 +-- .../slack_http_fallback.yaml | 32 ++--- .../testmanager_attachments.yaml | 36 +++--- .../testmanager_crud_grounded/check.py | 6 + .../testmanager_crud_grounded.yaml | 45 +++---- .../testmanager_execution_results.yaml | 36 +++--- .../testmanager_generic_records.yaml | 36 +++--- .../testmanager_requirement_lifecycle.yaml | 36 +++--- .../testmanager_testcase_lifecycle.yaml | 36 +++--- .../testmanager_testset_lifecycle.yaml | 36 +++--- 38 files changed, 604 insertions(+), 420 deletions(-) create mode 100644 tests/tasks/uipath-maestro-flow/_shared/test_batch3_alignment.py diff --git a/tests/tasks/uipath-maestro-flow/_shared/test_batch3_alignment.py b/tests/tasks/uipath-maestro-flow/_shared/test_batch3_alignment.py new file mode 100644 index 0000000000..bbf1247662 --- /dev/null +++ b/tests/tasks/uipath-maestro-flow/_shared/test_batch3_alignment.py @@ -0,0 +1,112 @@ +"""Regression coverage for Batch 3 same-ground checker routing.""" + +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path + + +TASK_ROOT = Path(__file__).resolve().parent.parent + + +def _run(relative: str, *args: str, cwd: Path) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, str(TASK_ROOT / relative), *args], + cwd=cwd, + capture_output=True, + text=True, + check=False, + ) + + +def _write_flow(path: Path, nodes: list[dict], bindings: list[dict] | None = None) -> None: + path.write_text( + json.dumps({"nodes": nodes, "edges": [], "bindings": bindings or []}), + encoding="utf-8", + ) + + +def test_bindings_checker_accepts_root_sdk_emit(tmp_path: Path) -> None: + _write_flow( + tmp_path / "BindingsMulti.flow", + [ + {"id": "a", "type": "uipath.connector.one.operation"}, + {"id": "b", "type": "uipath.connector.two.operation"}, + ], + ) + + result = _run( + "bindings/check_bindings.py", + "connector_node_count", + "**/BindingsMulti*.flow", + "2", + cwd=tmp_path, + ) + + assert result.returncode == 0, result.stderr or result.stdout + + +def test_multiselect_checker_accepts_root_sdk_emit(tmp_path: Path) -> None: + _write_flow( + tmp_path / "ComplexArrayTest.flow", + [ + { + "id": "slack", + "type": "uipath.connector.uipath-salesforce-slack.create-group-direct-message", + "inputs": {"detail": {"bodyParameters": {"users": ["U1", "U2"]}}}, + } + ], + ) + + result = _run( + "connector_features/check_multiselect_flow.py", + "populated", + cwd=tmp_path, + ) + + assert result.returncode == 0, result.stderr or result.stdout + + +def test_enum_checker_prefers_solution_flow_over_root_scratch(tmp_path: Path) -> None: + _write_flow(tmp_path / "EnumTest.flow", []) + project = tmp_path / "EnumTest" / "project.uiproj" + project.parent.mkdir() + project.write_text('{"ProjectType":"Flow"}', encoding="utf-8") + _write_flow( + project.parent / "EnumTest.flow", + [{"id": "start", "type": "core.trigger.manual"}], + ) + + result = _run( + "connector_features/check_enum_flow.py", + "**/EnumTest*.flow", + "structure", + cwd=tmp_path, + ) + + assert result.returncode == 0, result.stderr or result.stdout + assert "EnumTest/EnumTest.flow" in result.stdout + + +def test_testmanager_execution_evidence_is_distinct_from_round_trip( + tmp_path: Path, +) -> None: + (tmp_path / "seed.json").write_text('{"name": "expected"}') + (tmp_path / "result.json").write_text( + '{"created_name": "different", "retrieved_name": "different", "id": "123"}' + ) + + evidence = _run( + "connector_features/testmanager_crud_grounded/check.py", + "--execution-evidence", + cwd=tmp_path, + ) + semantic = _run( + "connector_features/testmanager_crud_grounded/check.py", + cwd=tmp_path, + ) + + assert evidence.returncode == 0, evidence.stderr or evidence.stdout + assert semantic.returncode == 1 diff --git a/tests/tasks/uipath-maestro-flow/_shared/test_same_ground_corpus.py b/tests/tasks/uipath-maestro-flow/_shared/test_same_ground_corpus.py index 3aa1e41572..3c888d746a 100644 --- a/tests/tasks/uipath-maestro-flow/_shared/test_same_ground_corpus.py +++ b/tests/tasks/uipath-maestro-flow/_shared/test_same_ground_corpus.py @@ -13,17 +13,6 @@ FLOW_TASKS = Path(__file__).resolve().parent.parent V1_AUTHORING_ALLOWLIST = { - "bindings/idempotent_reconfigure.yaml", - "bindings/multi_connector_independence.yaml", - "bindings/no_duplicate_connection_bindings.yaml", - "bindings/reconfigure_different_connection.yaml", - "connector_features/testmanager_attachments/testmanager_attachments.yaml", - "connector_features/testmanager_crud_grounded/testmanager_crud_grounded.yaml", - "connector_features/testmanager_execution_results/testmanager_execution_results.yaml", - "connector_features/testmanager_generic_records/testmanager_generic_records.yaml", - "connector_features/testmanager_requirement_lifecycle/testmanager_requirement_lifecycle.yaml", - "connector_features/testmanager_testcase_lifecycle/testmanager_testcase_lifecycle.yaml", - "connector_features/testmanager_testset_lifecycle/testmanager_testset_lifecycle.yaml", "connector_trigger/webhook_waitfor_parallel.yaml", "e2e/devcon_expense_approval.yaml", "evaluate/evaluator_type_choice.yaml", @@ -47,7 +36,6 @@ } FORBIDDEN_PROMPT_ALLOWLIST = { - "connector_features/slack-http-fallback/slack_http_fallback.yaml", "context-grounding/batch_transform/batch_transform.yaml", "context-grounding/summarize/summarize.yaml", "evaluate/inline_agent_eval/inline_agent_eval.yaml", @@ -72,8 +60,6 @@ } DEBUG_SOLUTION_ALLOWLIST = { - "connector_features/generic_dynamic_node/generic_dynamic_node.yaml", - "connector_features/slack-http-fallback/slack_http_fallback.yaml", "e2e/escalation_jira_ticket/escalation_jira_ticket.yaml", "e2e/jira_create_issue/jira_create_issue.yaml", "e2e/jira_get_issue/jira_get_issue.yaml", @@ -85,7 +71,6 @@ "edit/move_node/move_node.yaml", "edit/remove_node/remove_node.yaml", "edit/update_node/update_node.yaml", - "connector_features/jdbc_databricks_query/jdbc_databricks_query.yaml", } # F1 freezes the #2557 task even though its prompt predates the exact phrase. diff --git a/tests/tasks/uipath-maestro-flow/bindings/check_bindings.py b/tests/tasks/uipath-maestro-flow/bindings/check_bindings.py index 0c7bd027cf..232dfbf9c9 100644 --- a/tests/tasks/uipath-maestro-flow/bindings/check_bindings.py +++ b/tests/tasks/uipath-maestro-flow/bindings/check_bindings.py @@ -158,6 +158,23 @@ def cmd_structure(pattern: str) -> None: print(f"OK: {path} valid JSON with nodes + bindings") +def cmd_connector_node_count(pattern: str, minimum: str) -> None: + path, flow = _load_one(pattern) + nodes = flow.get("nodes") or [] + connector_nodes = [ + node + for node in nodes + if "uipath.connector." in str(node.get("type") or "").lower() + ] + required = int(minimum) + if len(connector_nodes) < required: + _fail( + f"{path}: expected at least {required} connector node(s), " + f"found {len(connector_nodes)}" + ) + print(f"OK: {path} has {len(connector_nodes)} connector node(s)") + + def cmd_no_empty_stubs(pattern: str) -> None: path, bindings = _load_flow_bindings(pattern) empty = [b for b in _connection_rows(bindings) if not b.get("resourceKey")] @@ -329,6 +346,7 @@ def cmd_both_ids_present_from_json(pattern: str, json_path: str) -> None: _COMMANDS = { "structure": cmd_structure, + "connector_node_count": cmd_connector_node_count, "no_empty_stubs": cmd_no_empty_stubs, "no_duplicates": cmd_no_duplicates, "matched_default": cmd_matched_default, diff --git a/tests/tasks/uipath-maestro-flow/bindings/idempotent_reconfigure.yaml b/tests/tasks/uipath-maestro-flow/bindings/idempotent_reconfigure.yaml index aaa5cefe2a..210b467b1a 100644 --- a/tests/tasks/uipath-maestro-flow/bindings/idempotent_reconfigure.yaml +++ b/tests/tasks/uipath-maestro-flow/bindings/idempotent_reconfigure.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-bindings-idempotent-reconfigure description: > Step 1 of the DAPField-mirrored upsert in `upsertConnectionResourceBinding` @@ -15,31 +19,29 @@ run_limits: turn_timeout: 1200 initial_prompt: | - Build a flow named "BindingsIdempotent" โ€” Manual Trigger โ†’ IS connector - node โ†’ End โ€” saved as `BindingsIdempotent/flow_files/BindingsIdempotent.flow` - with `BindingsIdempotent/project.uiproj`. + Build a flow named "BindingsIdempotent" with a Manual Trigger, an Integration + Service connector node, and an End node. - Snapshot the .flow after the first configure: - cp BindingsIdempotent/flow_files/BindingsIdempotent.flow after_first_configure.flow + Configure the connector node with an available connection. Save a copy of the + resulting flow at the sandbox root as `after_first_configure.flow`. - Then run the EXACT SAME `node configure` invocation a second time (same - node id, same --detail JSON). Snapshot again: - cp BindingsIdempotent/flow_files/BindingsIdempotent.flow after_second_configure.flow + Configure the exact same node a second time with the exact same connection + details. Save that result at the sandbox root as `after_second_configure.flow`. - Both snapshots go at the sandbox root. + Both snapshots must contain the configured connector bindings. success_criteria: - - type: command_executed - description: "Agent called `node configure` at least twice" - tool_name: "Bash" - command_pattern: "(uip|\\$UIP)\\s+(maestro\\s+)?flow\\s+node\\s+configure" - min_count: 2 + - type: run_command + description: "Both saved configure snapshots retain the same ConnectionId binding" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/bindings/check_bindings.py same_connection_id 'after_first_configure.flow' 'after_second_configure.flow'" + timeout: 10 + expected_exit_code: 0 weight: 1.5 pass_threshold: 1.0 - type: run_command description: "Both snapshots exist, valid JSON with bindings" - command: "python3 $TASK_DIR/check_bindings.py structure 'after_first_configure.flow' && python3 $TASK_DIR/check_bindings.py structure 'after_second_configure.flow'" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/bindings/check_bindings.py structure 'after_first_configure.flow' && python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/bindings/check_bindings.py structure 'after_second_configure.flow'" timeout: 10 expected_exit_code: 0 weight: 1.5 @@ -47,7 +49,7 @@ success_criteria: - type: run_command description: "Bindings count did not grow on second configure (idempotent)" - command: "python3 $TASK_DIR/check_bindings.py count_equal 'after_first_configure.flow' 'after_second_configure.flow'" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/bindings/check_bindings.py count_equal 'after_first_configure.flow' 'after_second_configure.flow'" timeout: 10 expected_exit_code: 0 weight: 4.0 @@ -55,7 +57,7 @@ success_criteria: - type: run_command description: "No empty-keyed Connection bindings in final snapshot" - command: "python3 $TASK_DIR/check_bindings.py no_empty_stubs 'after_second_configure.flow'" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/bindings/check_bindings.py no_empty_stubs 'after_second_configure.flow'" timeout: 10 expected_exit_code: 0 weight: 3.0 @@ -63,7 +65,7 @@ success_criteria: - type: run_command description: "ConnectionId resourceKey stable across both snapshots" - command: "python3 $TASK_DIR/check_bindings.py same_connection_id 'after_first_configure.flow' 'after_second_configure.flow'" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/bindings/check_bindings.py same_connection_id 'after_first_configure.flow' 'after_second_configure.flow'" timeout: 10 expected_exit_code: 0 weight: 2.0 diff --git a/tests/tasks/uipath-maestro-flow/bindings/multi_connector_independence.yaml b/tests/tasks/uipath-maestro-flow/bindings/multi_connector_independence.yaml index d470704f33..3d1d308a19 100644 --- a/tests/tasks/uipath-maestro-flow/bindings/multi_connector_independence.yaml +++ b/tests/tasks/uipath-maestro-flow/bindings/multi_connector_independence.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-bindings-multi-connector-independence description: > Two distinct connector nodes (different connector keys) in the same flow, @@ -19,10 +23,8 @@ run_limits: turn_timeout: 1200 initial_prompt: | - Build a flow named "BindingsMulti" with TWO IS connector nodes backed - by DIFFERENT connector keys, saved as - `BindingsMulti/flow_files/BindingsMulti.flow` with - `BindingsMulti/project.uiproj`. + Build a flow named "BindingsMulti" with TWO Integration Service connector + nodes backed by DIFFERENT connector keys. Configure each node with its own distinct connection. Record the connection ids and connector keys at the sandbox root: @@ -30,23 +32,19 @@ initial_prompt: | echo '{"a":"","ka":"","b":"","kb":""}' > conn_ids.json success_criteria: - # min_count: 1 (not 2) โ€” agents commonly chain two `uip ... node add` calls - # into a single Bash invocation with `&&`, which the checker counts as one - # match (re.search returns the first hit). Behavioral checks below verify - # both nodes actually exist in the flow. - - type: command_executed - description: "Agent used `uip flow node add` at least once" - tool_name: "Bash" - command_pattern: "(uip|\\$UIP)\\s+(maestro\\s+)?flow\\s+node\\s+add" - min_count: 1 + - type: run_command + description: "The flow contains at least two connector nodes" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/bindings/check_bindings.py connector_node_count '**/BindingsMulti*.flow' 2" + timeout: 10 + expected_exit_code: 0 weight: 1.0 pass_threshold: 1.0 - - type: command_executed - description: "Agent used `uip flow node configure` at least once" - tool_name: "Bash" - command_pattern: "(uip|\\$UIP)\\s+(maestro\\s+)?flow\\s+node\\s+configure" - min_count: 1 + - type: run_command + description: "Both recorded connection ids appear in the flow bindings" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/bindings/check_bindings.py both_ids_present_from_json '**/BindingsMulti*.flow' conn_ids.json" + timeout: 10 + expected_exit_code: 0 weight: 1.0 pass_threshold: 1.0 @@ -60,7 +58,7 @@ success_criteria: - type: run_command description: "Flow file exists and is valid JSON" - command: "python3 $TASK_DIR/check_bindings.py structure '**/BindingsMulti*.flow'" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/bindings/check_bindings.py structure '**/BindingsMulti*.flow'" timeout: 10 expected_exit_code: 0 weight: 2.0 @@ -68,7 +66,7 @@ success_criteria: - type: run_command description: "Both connection ids appear in Connection bindings, each on its own ConnectionId row" - command: "python3 $TASK_DIR/check_bindings.py both_ids_present_from_json '**/BindingsMulti*.flow' conn_ids.json" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/bindings/check_bindings.py both_ids_present_from_json '**/BindingsMulti*.flow' conn_ids.json" timeout: 10 expected_exit_code: 0 weight: 3.0 @@ -76,7 +74,7 @@ success_criteria: - type: run_command description: "No empty-keyed Connection bindings (cross-node aliasing surface)" - command: "python3 $TASK_DIR/check_bindings.py no_empty_stubs '**/BindingsMulti*.flow'" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/bindings/check_bindings.py no_empty_stubs '**/BindingsMulti*.flow'" timeout: 10 expected_exit_code: 0 weight: 4.0 @@ -84,7 +82,7 @@ success_criteria: - type: run_command description: "No (name, propertyAttribute, resourceKey) triplet duplicates" - command: "python3 $TASK_DIR/check_bindings.py no_triplet_dups '**/BindingsMulti*.flow'" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/bindings/check_bindings.py no_triplet_dups '**/BindingsMulti*.flow'" timeout: 10 expected_exit_code: 0 weight: 3.0 diff --git a/tests/tasks/uipath-maestro-flow/bindings/no_duplicate_connection_bindings.yaml b/tests/tasks/uipath-maestro-flow/bindings/no_duplicate_connection_bindings.yaml index 4c505c7629..8f0a91d17c 100644 --- a/tests/tasks/uipath-maestro-flow/bindings/no_duplicate_connection_bindings.yaml +++ b/tests/tasks/uipath-maestro-flow/bindings/no_duplicate_connection_bindings.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-bindings-no-duplicates description: > Regression - `uip maestro flow node configure` previously appended @@ -17,26 +21,25 @@ run_limits: turn_timeout: 1200 initial_prompt: | - Build a flow named "BindingsRegression" โ€” Manual Trigger โ†’ IS connector - node โ†’ End โ€” saved as `BindingsRegression/flow_files/BindingsRegression.flow` - with `BindingsRegression/project.uiproj`. + Build a flow named "BindingsRegression" with a Manual Trigger, an Integration + Service connector node, and an End node. - Configure the connector node exactly once. Do not invoke any "refresh - schema" workaround afterwards โ€” the bindings shape immediately after - `node configure` is what this eval inspects. + Configure the connector node exactly once. Do not apply any schema-refresh + workaround afterwards โ€” the bindings shape immediately after configuration + is what this eval inspects. success_criteria: - - type: command_executed - description: "Agent called `node configure`" - tool_name: "Bash" - command_pattern: "(uip|\\$UIP)\\s+(maestro\\s+)?flow\\s+node\\s+configure" - min_count: 1 + - type: run_command + description: "The connector is configured with a non-empty ConnectionId binding" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/bindings/check_bindings.py matched_default '**/BindingsRegression*.flow'" + timeout: 10 + expected_exit_code: 0 weight: 1.0 pass_threshold: 1.0 - type: run_command description: "Flow file exists and is valid JSON with nodes + bindings" - command: "python3 $TASK_DIR/check_bindings.py structure '**/BindingsRegression*.flow'" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/bindings/check_bindings.py structure '**/BindingsRegression*.flow'" timeout: 10 expected_exit_code: 0 weight: 2.0 @@ -44,7 +47,7 @@ success_criteria: - type: run_command description: "No empty-keyed Connection bindings remain after configure" - command: "python3 $TASK_DIR/check_bindings.py no_empty_stubs '**/BindingsRegression*.flow'" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/bindings/check_bindings.py no_empty_stubs '**/BindingsRegression*.flow'" timeout: 10 expected_exit_code: 0 weight: 4.0 @@ -52,7 +55,7 @@ success_criteria: - type: run_command description: "No duplicate (name, propertyAttribute) under Connection resource" - command: "python3 $TASK_DIR/check_bindings.py no_duplicates '**/BindingsRegression*.flow'" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/bindings/check_bindings.py no_duplicates '**/BindingsRegression*.flow'" timeout: 10 expected_exit_code: 0 weight: 4.0 @@ -60,7 +63,7 @@ success_criteria: - type: run_command description: "ConnectionId binding has non-empty resourceKey matching default" - command: "python3 $TASK_DIR/check_bindings.py matched_default '**/BindingsRegression*.flow'" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/bindings/check_bindings.py matched_default '**/BindingsRegression*.flow'" timeout: 10 expected_exit_code: 0 weight: 2.0 @@ -68,7 +71,7 @@ success_criteria: - type: run_command description: "bindings_v2.json (derived artifact), if emitted, also has no empty-keyed Connection rows" - command: "python3 $TASK_DIR/check_bindings.py bindings_v2_no_empty_stubs '**/bindings_v2.json'" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/bindings/check_bindings.py bindings_v2_no_empty_stubs '**/bindings_v2.json'" timeout: 10 expected_exit_code: 0 weight: 2.0 diff --git a/tests/tasks/uipath-maestro-flow/bindings/reconfigure_different_connection.yaml b/tests/tasks/uipath-maestro-flow/bindings/reconfigure_different_connection.yaml index 0fd6c28a2a..708b00d987 100644 --- a/tests/tasks/uipath-maestro-flow/bindings/reconfigure_different_connection.yaml +++ b/tests/tasks/uipath-maestro-flow/bindings/reconfigure_different_connection.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-bindings-reconfigure-different-connection description: > When the same connector node is reconfigured against a different connection, @@ -15,55 +19,38 @@ run_limits: initial_prompt: | PRECONDITION CHECK โ€” do this FIRST, before any flow work: - Run `uip is connections list --all-folders --output json` once. Group the - enabled connections by `ConnectorKey` and find a connector with two or - more enabled connections. That connector supplies A and B. + Inspect the available Integration Service connections once. Group the enabled + connections by connector key and find a connector with two or more enabled + connections. That connector supplies A and B. If no connector has two or more enabled connections, the precondition is not met for this tenant. Stop IMMEDIATELY: write `PRECONDITION_NOT_MET.txt` at the sandbox root containing one line per ConnectorKey with its enabled-connection count, then end your turn. - Do NOT try to create a second connection via `uip is connections create` - or `--no-wait`. OAuth authorization requires an interactive browser and - cannot complete in this sandbox; pending connections are not a valid - substitute. Do NOT scan for mocks or alternative connectors beyond the - single `connections list` call above. + Do NOT try to create a second connection. OAuth authorization requires an + interactive browser and cannot complete in this sandbox; pending connections + are not a valid substitute. Do NOT scan for mocks or alternative connectors + beyond that single connection inventory. If the precondition IS met, proceed: - Build a flow named "BindingsReconfigure" โ€” Manual Trigger โ†’ IS connector - node โ†’ End โ€” saved as `BindingsReconfigure/flow_files/BindingsReconfigure.flow` - with `BindingsReconfigure/project.uiproj`. + Build a flow named "BindingsReconfigure" with a Manual Trigger, an Integration + Service connector node, and an End node. Pick the connector that has two or more enabled connections, and use two distinct connection ids from that connector as A and B. - Record the ids at the sandbox root and snapshot the .flow after each configure: - - echo '{"a":"","b":""}' > conn_ids.json - - After configuring with A: - cp BindingsReconfigure/flow_files/BindingsReconfigure.flow after_a.flow - - After reconfiguring the SAME node with B: - cp BindingsReconfigure/flow_files/BindingsReconfigure.flow after_b.flow + Record the ids at the sandbox root in `conn_ids.json` as keys `a` and `b`. + Save the configured flow after A as `after_a.flow`, then reconfigure the SAME + node with B and save that result as `after_b.flow`, both at the sandbox root. success_criteria: - # min_count: 1 (not 2). Per uipath-maestro-flow SKILL.md rule #10, agents - # chain sequential `uip` calls โ€” e.g. `node configure && ... && node - # configure && validate && format` โ€” into ONE Bash event. command_executed - # counts matching Bash events, not command occurrences within an event, so a - # correct batched run registers only 1 match and this criterion false-fails. - # Multiplicity is instead proven by the A/B snapshot checks below: after_a - # references connection A while after_b references ONLY B (no leak, equal - # count) โ€” a result a single configure cannot produce. Mirrors the min_count:1 - # convention in multi_connector_independence.yaml / no_duplicate_connection_bindings.yaml. - - type: command_executed - description: "Agent used `uip flow node configure` at least once" - tool_name: "Bash" - command_pattern: "(uip|\\$UIP)\\s+(maestro\\s+)?flow\\s+node\\s+configure" - min_count: 1 + - type: run_command + description: "The final snapshot is configured only with connection B" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/bindings/check_bindings.py connection_id_is_from_json after_b.flow conn_ids.json b" + timeout: 10 + expected_exit_code: 0 weight: 1.0 pass_threshold: 1.0 @@ -77,7 +64,7 @@ success_criteria: - type: run_command description: "after_a.flow references connection A" - command: "python3 $TASK_DIR/check_bindings.py has_connection_key_from_json after_a.flow conn_ids.json a" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/bindings/check_bindings.py has_connection_key_from_json after_a.flow conn_ids.json a" timeout: 10 expected_exit_code: 0 weight: 2.0 @@ -85,7 +72,7 @@ success_criteria: - type: run_command description: "after_b.flow has zero references to connection A (no stale rows)" - command: "python3 $TASK_DIR/check_bindings.py no_connection_leak_from_json after_b.flow conn_ids.json a" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/bindings/check_bindings.py no_connection_leak_from_json after_b.flow conn_ids.json a" timeout: 10 expected_exit_code: 0 weight: 4.0 @@ -93,7 +80,7 @@ success_criteria: - type: run_command description: "after_b.flow ConnectionId binding points at connection B" - command: "python3 $TASK_DIR/check_bindings.py connection_id_is_from_json after_b.flow conn_ids.json b" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/bindings/check_bindings.py connection_id_is_from_json after_b.flow conn_ids.json b" timeout: 10 expected_exit_code: 0 weight: 3.0 @@ -101,7 +88,7 @@ success_criteria: - type: run_command description: "Reconfigure did not grow bindings count" - command: "python3 $TASK_DIR/check_bindings.py count_equal 'after_a.flow' 'after_b.flow'" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/bindings/check_bindings.py count_equal 'after_a.flow' 'after_b.flow'" timeout: 10 expected_exit_code: 0 weight: 2.0 @@ -109,7 +96,7 @@ success_criteria: - type: run_command description: "No empty-keyed Connection bindings in final snapshot" - command: "python3 $TASK_DIR/check_bindings.py no_empty_stubs 'after_b.flow'" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/bindings/check_bindings.py no_empty_stubs 'after_b.flow'" timeout: 10 expected_exit_code: 0 weight: 2.0 diff --git a/tests/tasks/uipath-maestro-flow/connector_features/ceql_where.yaml b/tests/tasks/uipath-maestro-flow/connector_features/ceql_where.yaml index 0fee1a29a9..6a96f7c287 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/ceql_where.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_features/ceql_where.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-ipe-ceql-where description: > Tests the CEQL where query IS feature โ€” agent plans a structured filter @@ -31,18 +35,16 @@ initial_prompt: | success_criteria: - type: run_command description: "where_detail.json (found anywhere under the solution) has a canonical CEQL filter tree on displayName='active'; flow references the registered Azure AD / Entra connector key with List Groups + Terminate nodes" - command: "python3 $TASK_DIR/check_ceql_where_flow.py" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/connector_features/check_ceql_where_flow.py" timeout: 60 expected_exit_code: 0 weight: 8.0 pass_threshold: 1.0 - - type: command_executed - description: "uip maestro flow validate was called" - tool_name: "Bash" - command_pattern: "(uip|\\$UIP)\\s+(maestro\\s+)?flow\\s+validate" - min_count: 1 - require_success: true + - type: run_command + description: "The completed flow validates" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py" + timeout: 30 + expected_exit_code: 0 weight: 2.0 pass_threshold: 1.0 - diff --git a/tests/tasks/uipath-maestro-flow/connector_features/check_ceql_where_flow.py b/tests/tasks/uipath-maestro-flow/connector_features/check_ceql_where_flow.py index 405f186c6a..3c38307f80 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/check_ceql_where_flow.py +++ b/tests/tasks/uipath-maestro-flow/connector_features/check_ceql_where_flow.py @@ -23,7 +23,7 @@ import sys sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")) -from _shared.flow_check import assert_flow_has_node_type # noqa: E402 +from _shared.flow_check import assert_flow_has_node_type, find_flow_file # noqa: E402 CONNECTOR_KEY = "uipath-microsoft-azureactivedirectory" FLOW_GLOB = "**/CeqlWhereTest*.flow" @@ -134,10 +134,7 @@ def _check_where_detail() -> None: def _find_flow() -> str: - flows = glob.glob(FLOW_GLOB, recursive=True) - if not flows: - sys.exit(f"FAIL: No flow file matching {FLOW_GLOB}") - return flows[0] + return find_flow_file(flow_glob=os.path.basename(FLOW_GLOB)) def _check_flow_structure() -> None: diff --git a/tests/tasks/uipath-maestro-flow/connector_features/check_enum_flow.py b/tests/tasks/uipath-maestro-flow/connector_features/check_enum_flow.py index b5efadf0e9..e321406a10 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/check_enum_flow.py +++ b/tests/tasks/uipath-maestro-flow/connector_features/check_enum_flow.py @@ -13,27 +13,28 @@ from __future__ import annotations -import glob import json +import os import sys +from pathlib import Path from typing import Any, NoReturn +SHARED_DIR = Path(__file__).resolve().parent.parent / "_shared" +sys.path.insert(0, str(SHARED_DIR)) +from flow_check import find_flow_file # noqa: E402 + def _fail(message: str) -> NoReturn: sys.exit(f"FAIL: {message}") def _load_flow(pattern: str) -> tuple[str, dict[str, Any]]: - matches = sorted(glob.glob(pattern, recursive=True)) - if not matches: - _fail(f"No flow found for {pattern!r}") - if len(matches) > 1: - _fail(f"Multiple flows found for {pattern!r}: {matches}") + path = find_flow_file(flow_glob=os.path.basename(pattern)) try: - with open(matches[0], encoding="utf-8") as flow_file: - return matches[0], json.load(flow_file) + with open(path, encoding="utf-8") as flow_file: + return path, json.load(flow_file) except json.JSONDecodeError as exc: - _fail(f"Flow {matches[0]} is not valid JSON: {exc}") + _fail(f"Flow {path} is not valid JSON: {exc}") def _check_structure(flow_path: str, flow: dict[str, Any]) -> None: diff --git a/tests/tasks/uipath-maestro-flow/connector_features/check_managed_http_fallback.py b/tests/tasks/uipath-maestro-flow/connector_features/check_managed_http_fallback.py index f0b8ebadbe..7324b29d8b 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/check_managed_http_fallback.py +++ b/tests/tasks/uipath-maestro-flow/connector_features/check_managed_http_fallback.py @@ -3,23 +3,24 @@ from __future__ import annotations -import glob import json +import os import sys +from pathlib import Path from typing import Any, NoReturn +SHARED_DIR = Path(__file__).resolve().parent.parent / "_shared" +sys.path.insert(0, str(SHARED_DIR)) +from flow_check import find_flow_file # noqa: E402 + def _fail(message: str) -> NoReturn: sys.exit(f"FAIL: {message}") def _load_flow(pattern: str) -> dict[str, Any]: - matches = sorted(glob.glob(pattern, recursive=True)) - if not matches: - _fail(f"No flow found for {pattern!r}") - if len(matches) > 1: - _fail(f"Multiple flows found for {pattern!r}: {matches}") - with open(matches[0], encoding="utf-8") as flow_file: + path = find_flow_file(flow_glob=os.path.basename(pattern)) + with open(path, encoding="utf-8") as flow_file: return json.load(flow_file) diff --git a/tests/tasks/uipath-maestro-flow/connector_features/check_multiselect_flow.py b/tests/tasks/uipath-maestro-flow/connector_features/check_multiselect_flow.py index f6f3c67b61..acf6a8f62e 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/check_multiselect_flow.py +++ b/tests/tasks/uipath-maestro-flow/connector_features/check_multiselect_flow.py @@ -11,13 +11,17 @@ multiselect field in a .flow. Name-agnostic: the prompt does not fix a project name, so every .flow file -is checked ('**' glob skips dot-dirs, so .uipath/.skills flows are ignored). +selected by the shared same-ground discovery helper is checked. """ import ast -import glob import json import re import sys +from pathlib import Path + +SHARED_DIR = Path(__file__).resolve().parent.parent / "_shared" +sys.path.insert(0, str(SHARED_DIR)) +from flow_check import find_flow_files # noqa: E402 def parse_users(value): @@ -73,7 +77,7 @@ def find_users(obj): return None -def check_flow(path): +def check_flow(path, expected_count): """Return None on pass, else a failure reason.""" try: flow = json.load(open(path)) @@ -91,21 +95,27 @@ def check_flow(path): users = find_users(node.get("inputs", {})) if users is None: continue - if len(users) == 3: + if expected_count == "populated" and users: + print(f"OK: {path} โ€” node '{node['id']}' users={users}") + return None + if isinstance(expected_count, int) and len(users) == expected_count: print(f"OK: {path} โ€” node '{node['id']}' users={users}") return None - return f"users field has {len(users)} entries, expected 3: {users}" + return f"users field has {len(users)} entries, expected {expected_count}: {users}" return "Slack node has no 'users' multiselect field" def main(): - flows = glob.glob("**/*.flow", recursive=True) - if not flows: - sys.exit("no .flow file found") + expected_count = ( + "populated" + if len(sys.argv) > 1 and sys.argv[1] == "populated" + else int(sys.argv[1]) if len(sys.argv) > 1 else 3 + ) + flows = find_flow_files() reasons = [] for path in flows: - reason = check_flow(path) + reason = check_flow(path, expected_count) if reason is None: sys.exit(0) reasons.append(f"{path}: {reason}") diff --git a/tests/tasks/uipath-maestro-flow/connector_features/check_path_param_value.py b/tests/tasks/uipath-maestro-flow/connector_features/check_path_param_value.py index 67ab6f46f1..9b8d4b8586 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/check_path_param_value.py +++ b/tests/tasks/uipath-maestro-flow/connector_features/check_path_param_value.py @@ -16,11 +16,16 @@ from __future__ import annotations -import glob import json +import os import sys +from pathlib import Path from typing import Any, NoReturn +SHARED_DIR = Path(__file__).resolve().parent.parent / "_shared" +sys.path.insert(0, str(SHARED_DIR)) +from flow_check import find_flow_file # noqa: E402 + _JSONSTRING_PREFIX = "=jsonString:" @@ -30,13 +35,9 @@ def _fail(message: str) -> NoReturn: def _load_flow(pattern: str) -> tuple[str, dict[str, Any]]: - matches = sorted(glob.glob(pattern, recursive=True)) - if not matches: - _fail(f"No flow found for {pattern!r}") - if len(matches) > 1: - _fail(f"Multiple flows found for {pattern!r}: {matches}") - with open(matches[0], encoding="utf-8") as flow_file: - return matches[0], json.load(flow_file) + path = find_flow_file(flow_glob=os.path.basename(pattern)) + with open(path, encoding="utf-8") as flow_file: + return path, json.load(flow_file) def _normalise_detail(raw: Any, node_id: str) -> dict[str, Any] | None: diff --git a/tests/tasks/uipath-maestro-flow/connector_features/complex_array.yaml b/tests/tasks/uipath-maestro-flow/connector_features/complex_array.yaml index ec49e8a35b..ebfa75a8c2 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/complex_array.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_features/complex_array.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-ipe-complex-array description: > Tests the complex array IS feature โ€” configures the Slack @@ -21,7 +25,7 @@ initial_prompt: | success_criteria: - type: run_command description: "Flow file exists and is valid JSON" - command: "python3 -c \"import json,glob; flows=glob.glob('**/ComplexArrayTest*.flow',recursive=True); assert flows, 'No flow found'; f=json.load(open(flows[0])); assert 'nodes' in f and 'edges' in f; nn=len(f['nodes']); ne=len(f['edges']); print('OK: %d nodes, %d edges'%(nn,ne))\"" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py --flow-name ComplexArrayTest '\"nodes\"' '\"edges\"'" timeout: 10 expected_exit_code: 0 weight: 2.0 @@ -29,7 +33,7 @@ success_criteria: - type: run_command description: "Flow has the Slack Create Group Direct Message node with its users complex array populated" - command: "python3 -c \"import json,glob; flows=glob.glob('**/ComplexArrayTest*.flow',recursive=True); assert flows, 'No flow found'; f=json.load(open(flows[0])); nodes=[n for n in f['nodes'] if n.get('type','')=='uipath.connector.uipath-salesforce-slack.create-group-direct-message']; assert nodes, 'No create-group-direct-message node'; d=json.dumps(nodes[0].get('inputs',{})); assert 'users' in d, 'users complex array not populated'; print('OK: group DM node with users array present')\"" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/connector_features/check_multiselect_flow.py populated" timeout: 10 expected_exit_code: 0 weight: 3.0 @@ -37,18 +41,16 @@ success_criteria: - type: run_command description: "Advisory: group DM participants referenced in the project (names may resolve to Slack user IDs on a live connection)" - command: "python3 -c \"import glob,os,sys; files=[p for p in glob.glob('**/ComplexArrayTest*/**',recursive=True)+glob.glob('**/ComplexArrayTest*.*',recursive=True) if os.path.isfile(p) and not p.endswith('.uipx')]; blob=''.join(open(p,errors='ignore').read() for p in files); missing=[u for u in ('coder evals','e2e-testing') if u not in blob]; sys.exit(1 if missing else 0)\"" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py --flow-name ComplexArrayTest 'coder evals' 'e2e-testing'" timeout: 10 expected_exit_code: 0 weight: 1.0 pass_threshold: 0 - - type: command_executed - description: "uip maestro flow validate was called" - tool_name: "Bash" - command_pattern: "(uip|\\$UIP)\\s+(maestro\\s+)?flow\\s+validate" - min_count: 1 - require_success: true + - type: run_command + description: "The completed flow validates" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py" + timeout: 30 + expected_exit_code: 0 weight: 2.0 pass_threshold: 1.0 - diff --git a/tests/tasks/uipath-maestro-flow/connector_features/dtl_load_by_default_false.yaml b/tests/tasks/uipath-maestro-flow/connector_features/dtl_load_by_default_false.yaml index 01ed11baa6..1d4f9584b9 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/dtl_load_by_default_false.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_features/dtl_load_by_default_false.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-ipe-dtl-load-by-default-false description: > Tests the DTL loadByDefault=false IS feature โ€” configures a connector node @@ -20,7 +24,7 @@ initial_prompt: | success_criteria: - type: run_command description: "Flow file exists and is valid JSON" - command: "python3 -c \"import json,glob; flows=glob.glob('**/DTLLoadByDefaultFalseTest*.flow',recursive=True); assert flows, 'No flow found'; f=json.load(open(flows[0])); assert 'nodes' in f and 'edges' in f; nn=len(f['nodes']); ne=len(f['edges']); print('OK: %d nodes, %d edges'%(nn,ne))\"" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py --flow-name DTLLoadByDefaultFalseTest '\"nodes\"' '\"edges\"'" timeout: 10 expected_exit_code: 0 weight: 3.0 @@ -28,17 +32,16 @@ success_criteria: - type: run_command description: "Flow has a connector node referencing uipath-automattic-woocommerce" - command: "python3 -c \"import json,glob; flows=glob.glob('**/DTLLoadByDefaultFalseTest*.flow',recursive=True); assert flows; content=open(flows[0]).read(); assert 'uipath-automattic-woocommerce' in content, 'Connector key not found'; print('OK: connector key present')\"" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py --flow-name DTLLoadByDefaultFalseTest 'uipath-automattic-woocommerce'" timeout: 10 expected_exit_code: 0 weight: 3.0 pass_threshold: 1.0 - - type: command_executed - description: "uip maestro flow validate was called" - tool_name: "Bash" - command_pattern: "(uip|\\$UIP)\\s+(maestro\\s+)?flow\\s+validate" - min_count: 1 + - type: run_command + description: "The completed flow validates" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py" + timeout: 30 + expected_exit_code: 0 weight: 2.0 pass_threshold: 1.0 - diff --git a/tests/tasks/uipath-maestro-flow/connector_features/dtl_load_by_default_true.yaml b/tests/tasks/uipath-maestro-flow/connector_features/dtl_load_by_default_true.yaml index 90e51bf977..e3733d57a3 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/dtl_load_by_default_true.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_features/dtl_load_by_default_true.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-ipe-dtl-load-by-default-true description: > Tests the DTL loadByDefault=true IS feature โ€” configures a Jira Get Issue @@ -21,7 +25,7 @@ initial_prompt: | success_criteria: - type: run_command description: "Flow file exists and is valid JSON" - command: "python3 -c \"import json,glob; flows=glob.glob('**/DTLLoadByDefaultTrueTest*.flow',recursive=True); assert flows, 'No flow found'; f=json.load(open(flows[0])); assert 'nodes' in f and 'edges' in f; nn=len(f['nodes']); ne=len(f['edges']); print('OK: %d nodes, %d edges'%(nn,ne))\"" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py --flow-name DTLLoadByDefaultTrueTest '\"nodes\"' '\"edges\"'" timeout: 10 expected_exit_code: 0 weight: 3.0 @@ -43,11 +47,10 @@ success_criteria: weight: 3.0 pass_threshold: 1.0 - - type: command_executed - description: "uip maestro flow validate was called" - tool_name: "Bash" - command_pattern: "(uip|\\$UIP)\\s+(maestro\\s+)?flow\\s+validate" - min_count: 1 + - type: run_command + description: "The completed flow validates" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py" + timeout: 30 + expected_exit_code: 0 weight: 2.0 pass_threshold: 1.0 - diff --git a/tests/tasks/uipath-maestro-flow/connector_features/enhanced_enum.yaml b/tests/tasks/uipath-maestro-flow/connector_features/enhanced_enum.yaml index cc0d23f207..13189c7b6f 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/enhanced_enum.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_features/enhanced_enum.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-ipe-enhanced-enum description: > Tests the enhanced enum IS feature โ€” configures a connector node with an @@ -23,7 +27,7 @@ initial_prompt: | success_criteria: - type: run_command description: "Flow file exists and is valid JSON" - command: "python3 -c \"import json,glob; flows=glob.glob('**/EnhancedEnumTest*.flow',recursive=True); assert flows, 'No flow found'; f=json.load(open(flows[0])); assert 'nodes' in f and 'edges' in f; nn=len(f['nodes']); ne=len(f['edges']); print('OK: %d nodes, %d edges'%(nn,ne))\"" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py --flow-name EnhancedEnumTest '\"nodes\"' '\"edges\"'" timeout: 10 expected_exit_code: 0 weight: 3.0 @@ -31,17 +35,16 @@ success_criteria: - type: run_command description: "Flow has a connector node referencing uipath-automattic-woocommerce" - command: "python3 -c \"import json,glob; flows=glob.glob('**/EnhancedEnumTest*.flow',recursive=True); assert flows; content=open(flows[0]).read(); assert 'uipath-automattic-woocommerce' in content, 'Connector key not found'; print('OK: connector key present')\"" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py --flow-name EnhancedEnumTest 'uipath-automattic-woocommerce'" timeout: 10 expected_exit_code: 0 weight: 3.0 pass_threshold: 1.0 - - type: command_executed - description: "uip maestro flow validate was called" - tool_name: "Bash" - command_pattern: "(uip|\\$UIP)\\s+(maestro\\s+)?flow\\s+validate" - min_count: 1 + - type: run_command + description: "The completed flow validates" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py" + timeout: 30 + expected_exit_code: 0 weight: 2.0 pass_threshold: 1.0 - diff --git a/tests/tasks/uipath-maestro-flow/connector_features/enum.yaml b/tests/tasks/uipath-maestro-flow/connector_features/enum.yaml index 2ea11952d7..cb9fb7239c 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/enum.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_features/enum.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-ipe-enum description: > Tests the enum IS feature โ€” configures a connector node with an enum @@ -26,16 +30,16 @@ initial_prompt: | success_criteria: - type: command_executed - description: "Agent invoked `uip is connections list` to discover available connections" + description: "Advisory: live-v1 agent listed connections before choosing one" tool_name: "Bash" command_pattern: "(uip|\\$UIP)\\s+is\\s+connections\\s+list" min_count: 1 weight: 1.0 - pass_threshold: 1.0 + pass_threshold: 0.0 - type: run_command description: "Tenant has at least one Gmail connection (precondition for `node configure`)" - command: "python3 $TASK_DIR/check_connection_available.py uipath-google-gmail" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/connector_features/check_connection_available.py uipath-google-gmail" timeout: 35 expected_exit_code: 0 weight: 1.0 @@ -43,7 +47,7 @@ success_criteria: - type: run_command description: "Flow file exists and is valid JSON" - command: "python3 $TASK_DIR/check_enum_flow.py '**/EnumTest*.flow' structure" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/connector_features/check_enum_flow.py '**/EnumTest*.flow' structure" timeout: 10 expected_exit_code: 0 weight: 3.0 @@ -51,7 +55,7 @@ success_criteria: - type: run_command description: "Flow uses Gmail connector or task-specific managed HTTP fallback" - command: "python3 $TASK_DIR/check_managed_http_fallback.py '**/EnumTest*.flow' uipath-google-gmail enum" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/connector_features/check_managed_http_fallback.py '**/EnumTest*.flow' uipath-google-gmail enum" timeout: 10 expected_exit_code: 0 weight: 3.0 @@ -59,7 +63,7 @@ success_criteria: - type: run_command description: "Connector node has inputs.detail.bodyParameters with to and importance" - command: "python3 $TASK_DIR/check_enum_flow.py '**/EnumTest*.flow' body_params" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/connector_features/check_enum_flow.py '**/EnumTest*.flow' body_params" timeout: 10 expected_exit_code: 0 weight: 2.0 @@ -67,16 +71,16 @@ success_criteria: - type: run_command description: "Flow has Decision and Terminate nodes" - command: "python3 $TASK_DIR/check_enum_flow.py '**/EnumTest*.flow' control" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/connector_features/check_enum_flow.py '**/EnumTest*.flow' control" timeout: 10 expected_exit_code: 0 weight: 2.0 pass_threshold: 1.0 - - type: command_executed - description: "uip maestro flow validate was called" - tool_name: "Bash" - command_pattern: "(uip|\\$UIP)\\s+(maestro\\s+)?flow\\s+validate" - min_count: 1 + - type: run_command + description: "The completed flow validates" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py" + timeout: 30 + expected_exit_code: 0 weight: 2.0 pass_threshold: 1.0 diff --git a/tests/tasks/uipath-maestro-flow/connector_features/generic_dynamic_node/check_generic_dynamic_node.py b/tests/tasks/uipath-maestro-flow/connector_features/generic_dynamic_node/check_generic_dynamic_node.py index a0f6cf05fa..1dbd4c0991 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/generic_dynamic_node/check_generic_dynamic_node.py +++ b/tests/tasks/uipath-maestro-flow/connector_features/generic_dynamic_node/check_generic_dynamic_node.py @@ -31,7 +31,6 @@ from __future__ import annotations -import glob import json import os import sys @@ -40,7 +39,7 @@ from _shared.flow_check import ( # noqa: E402 _get_ci, assert_flow_uses_connector_target, - find_project_dir, + find_flow_file, run_debug, ) @@ -56,11 +55,9 @@ OBJECT_NAME = "acr_user" -def _load_flow_nodes(project_dir: str) -> list[dict]: - flows = glob.glob(os.path.join(project_dir, "**/*.flow"), recursive=True) - if not flows: - sys.exit(f"FAIL: No .flow file found under {project_dir}") - with open(flows[0]) as f: +def _load_flow_nodes() -> list[dict]: + flow_path = find_flow_file(flow_glob="AcrUserList*.flow") + with open(flow_path) as f: flow = json.load(f) return flow.get("nodes") or [] @@ -132,7 +129,7 @@ def _assert_structure() -> None: # A real ServiceNow connector node must be present (native connector node). assert_flow_uses_connector_target(CONNECTOR_KEY) - nodes = _load_flow_nodes(find_project_dir()) + nodes = _load_flow_nodes() list_nodes = [] for n in nodes: if CONNECTOR_KEY not in str(n.get("type", "")).lower(): diff --git a/tests/tasks/uipath-maestro-flow/connector_features/generic_dynamic_node/generic_dynamic_node.yaml b/tests/tasks/uipath-maestro-flow/connector_features/generic_dynamic_node/generic_dynamic_node.yaml index a25f18494f..b17b4577a0 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/generic_dynamic_node/generic_dynamic_node.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_features/generic_dynamic_node/generic_dynamic_node.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-generic-dynamic-node description: > Connector feature: validate a generic (dynamic) connector node end-to-end. A @@ -29,12 +33,13 @@ run_limits: turn_timeout: 1200 initial_prompt: | - Create a UiPath Flow project named "AcrUserList" with a manual trigger. + Create a UiPath Flow project named "AcrUserList" inside a solution of the + same name, with a manual trigger. The process must call ServiceNow to list all records in the Acr User object, then surface the returned records as a flow output variable. - Bind it to the tenant's ServiceNow connection. Then run the flow with - `uip maestro flow debug` to confirm the connector call completes. + Bind it to the tenant's ServiceNow connection. Then debug the flow to confirm + the connector call completes. Use the connection present in Shared/uipath-maestro-flow. @@ -55,17 +60,17 @@ success_criteria: # โ”€โ”€ Agent invoked debug โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ - type: command_executed - description: "Agent ran flow debug" + description: "Advisory: live-v1 agent ran flow debug" tool_name: "Bash" command_pattern: '(uip|\$UIP|\$\{?UIP\}?)\s+(maestro\s+)?flow\s+debug' min_count: 1 weight: 1.5 - pass_threshold: 1.0 + pass_threshold: 0.0 # โ”€โ”€ Execution: connector calls ServiceNow and surfaces an array output โ”€โ”€ - type: run_command description: "ServiceNow list-all-records node on acr_user executes and surfaces an array output" - command: "python3 $TASK_DIR/check_generic_dynamic_node.py" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/connector_features/generic_dynamic_node/check_generic_dynamic_node.py" timeout: 720 expected_exit_code: 0 weight: 6.0 diff --git a/tests/tasks/uipath-maestro-flow/connector_features/jdbc_databricks_query/check_databricks_query.py b/tests/tasks/uipath-maestro-flow/connector_features/jdbc_databricks_query/check_databricks_query.py index 8bc1d0a37e..fcab575feb 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/jdbc_databricks_query/check_databricks_query.py +++ b/tests/tasks/uipath-maestro-flow/connector_features/jdbc_databricks_query/check_databricks_query.py @@ -17,7 +17,6 @@ from __future__ import annotations -import glob import json import os import sys @@ -27,6 +26,7 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) from _shared.flow_check import ( # noqa: E402 assert_flow_uses_connector_target, + find_flow_file, ) FLOW_GLOB = "**/DatabricksQuery*.flow" @@ -40,10 +40,7 @@ def _fail(msg: str) -> NoReturn: def _flow_path() -> str: - flows = glob.glob(FLOW_GLOB, recursive=True) - if not flows: - _fail(f"no flow file matching {FLOW_GLOB}") - return flows[0] + return find_flow_file(flow_glob=os.path.basename(FLOW_GLOB)) def _node_type(node: dict) -> str: diff --git a/tests/tasks/uipath-maestro-flow/connector_features/jdbc_databricks_query/jdbc_databricks_query.yaml b/tests/tasks/uipath-maestro-flow/connector_features/jdbc_databricks_query/jdbc_databricks_query.yaml index 82db43ecfb..06611734e8 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/jdbc_databricks_query/jdbc_databricks_query.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_features/jdbc_databricks_query/jdbc_databricks_query.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. # Tenant-gated. Requires the Databricks-backed Database Hub (JDBC) # connection in the runner tenant, reaching the sandbox.globalmart schema whose # `employees` table is seeded with the fixture rows this task asserts on (Alice @@ -38,7 +42,8 @@ run_limits: turn_timeout: 900 initial_prompt: | - Create a new Flow project called "DatabricksQuery" with a manual trigger. + Create a new Flow project called "DatabricksQuery" inside a solution of the + same name, with a manual trigger. It should run a SQL query against a Databricks database using the Database Hub (JDBC) connector, and expose the query result as a flow output. Run an aggregate @@ -56,25 +61,25 @@ initial_prompt: | Do NOT pause between planning and implementation. success_criteria: - - type: command_executed - description: "uip maestro flow validate was called" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+(maestro\s+)?flow\s+validate' - min_count: 1 + - type: run_command + description: "The completed flow validates" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py" + timeout: 30 + expected_exit_code: 0 weight: 2.0 pass_threshold: 1.0 - type: command_executed - description: "uip maestro flow debug was called" + description: "Advisory: live-v1 agent ran flow debug" tool_name: "Bash" command_pattern: '(uip|\$UIP)\s+(maestro\s+)?flow\s+debug' min_count: 1 weight: 1.5 - pass_threshold: 1.0 + pass_threshold: 0.0 - type: run_command description: "DatabricksQuery checks: valid flow wires the JDBC Execute-Query node with a bound connection, not the native Databricks connector or an HTTP fallback" - command: "python3 $TASK_DIR/check_databricks_query.py" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/connector_features/jdbc_databricks_query/check_databricks_query.py" timeout: 600 expected_exit_code: 0 weight: 3.0 diff --git a/tests/tasks/uipath-maestro-flow/connector_features/multiselect.yaml b/tests/tasks/uipath-maestro-flow/connector_features/multiselect.yaml index 4c98422b7b..cf89896445 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/multiselect.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_features/multiselect.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-ipe-multiselect description: > Tests the multiselect IS feature โ€” configures a Slack group direct message @@ -20,7 +24,7 @@ success_criteria: # Name-agnostic: the prompt doesn't fix a project name. - type: run_command description: "A .flow file exists and is valid JSON with nodes and edges" - command: "python3 -c \"import json,glob,sys; flows=glob.glob('**/*.flow',recursive=True); sys.exit('no .flow found') if not flows else None; f=json.load(open(flows[0])); assert 'nodes' in f and 'edges' in f; print('OK: %d nodes, %d edges'%(len(f['nodes']),len(f['edges'])))\"" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py '\"nodes\"' '\"edges\"'" timeout: 10 expected_exit_code: 0 weight: 1.5 @@ -30,17 +34,16 @@ success_criteria: # field has exactly 3 entries (the three group-DM recipients). - type: run_command description: "Slack group-DM node present with 3 entries in the users multiselect" - command: "python3 $TASK_DIR/check_multiselect_flow.py" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/connector_features/check_multiselect_flow.py" timeout: 15 expected_exit_code: 0 weight: 3.0 pass_threshold: 1.0 - - type: command_executed - description: "uip maestro flow validate was called" - tool_name: "Bash" - command_pattern: "(uip|\\$UIP)\\s+(maestro\\s+)?flow\\s+validate" - min_count: 1 + - type: run_command + description: "The completed flow validates" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py" + timeout: 30 + expected_exit_code: 0 weight: 2.0 pass_threshold: 1.0 - diff --git a/tests/tasks/uipath-maestro-flow/connector_features/non-catalog-http-fallback/check_non_catalog_http_fallback.py b/tests/tasks/uipath-maestro-flow/connector_features/non-catalog-http-fallback/check_non_catalog_http_fallback.py index 891a7d5049..69276a0ef2 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/non-catalog-http-fallback/check_non_catalog_http_fallback.py +++ b/tests/tasks/uipath-maestro-flow/connector_features/non-catalog-http-fallback/check_non_catalog_http_fallback.py @@ -28,12 +28,16 @@ from __future__ import annotations -import glob import json import os import sys +from pathlib import Path from typing import Any, NoReturn +SHARED_DIR = Path(__file__).resolve().parents[2] / "_shared" +sys.path.insert(0, str(SHARED_DIR)) +from flow_check import find_flow_file # noqa: E402 + FLOW_GLOB = "**/SpotifyProfileTest*.flow" # The generic HTTP connector โ€” the only managed path to a service with no # catalog connector of its own. @@ -48,16 +52,12 @@ def _fail(message: str) -> NoReturn: def _read_flow() -> dict[str, Any]: - flows = sorted(glob.glob(FLOW_GLOB, recursive=True)) - if not flows: - _fail(f"No flow file matching {FLOW_GLOB}") - if len(flows) > 1: - _fail(f"Multiple flows match {FLOW_GLOB}: {flows}") - with open(flows[0], encoding="utf-8") as f: + path = find_flow_file(flow_glob=os.path.basename(FLOW_GLOB)) + with open(path, encoding="utf-8") as f: try: return json.load(f) except json.JSONDecodeError as e: - _fail(f"{flows[0]} is not valid JSON: {e}") + _fail(f"{path} is not valid JSON: {e}") def _normalise_detail(raw: Any) -> dict[str, Any]: diff --git a/tests/tasks/uipath-maestro-flow/connector_features/non-catalog-http-fallback/non_catalog_http_fallback.yaml b/tests/tasks/uipath-maestro-flow/connector_features/non-catalog-http-fallback/non_catalog_http_fallback.yaml index 44e7d74bb8..c6fc456c3a 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/non-catalog-http-fallback/non_catalog_http_fallback.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_features/non-catalog-http-fallback/non_catalog_http_fallback.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-non-catalog-http-fallback description: > Integration test: a NON-catalog service (Spotify) has no IS connector of its @@ -27,22 +31,21 @@ initial_prompt: | Validate the final flow file. - Use `--output json` on every `uip` command whose output you parse. Do NOT ask for approval, confirmation, or feedback โ€” build and validate the flow end-to-end in a single pass. success_criteria: - - type: command_executed - description: "uip maestro flow validate was called" - tool_name: "Bash" - command_pattern: "(uip|\\$UIP)\\s+(maestro\\s+)?flow\\s+validate" - min_count: 1 + - type: run_command + description: "The completed flow validates" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py" + timeout: 30 + expected_exit_code: 0 weight: 2.0 pass_threshold: 1.0 - type: run_command description: "Profile fetch uses a managed HTTP node with the uipath-uipath-http connector key (non-catalog fallback) and targets the /me endpoint" - command: "python3 $TASK_DIR/check_non_catalog_http_fallback.py check_fallback" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/connector_features/non-catalog-http-fallback/check_non_catalog_http_fallback.py check_fallback" timeout: 30 expected_exit_code: 0 weight: 3.0 diff --git a/tests/tasks/uipath-maestro-flow/connector_features/path_params.yaml b/tests/tasks/uipath-maestro-flow/connector_features/path_params.yaml index 3da24461e7..ebeea6b0ed 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/path_params.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_features/path_params.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-ipe-path-params description: > Tests the path parameters IS feature โ€” configures a connector node with a @@ -25,16 +29,16 @@ initial_prompt: | success_criteria: - type: command_executed - description: "Agent invoked `uip is connections list` to discover available connections" + description: "Advisory: live-v1 agent listed connections before choosing one" tool_name: "Bash" command_pattern: "(uip|\\$UIP)\\s+is\\s+connections\\s+list" min_count: 1 weight: 1.0 - pass_threshold: 1.0 + pass_threshold: 0.0 - type: run_command description: "Tenant has at least one Jira connection (precondition for `node configure`)" - command: "python3 $TASK_DIR/check_connection_available.py uipath-atlassian-jira" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/connector_features/check_connection_available.py uipath-atlassian-jira" timeout: 35 expected_exit_code: 0 weight: 1.0 @@ -42,7 +46,7 @@ success_criteria: - type: run_command description: "Flow file exists and is valid JSON" - command: "python3 -c \"import json,glob; flows=glob.glob('**/PathParamsTest*.flow',recursive=True); assert flows, 'No flow found'; f=json.load(open(flows[0])); assert 'nodes' in f and 'edges' in f; nn=len(f['nodes']); ne=len(f['edges']); print('OK: %d nodes, %d edges'%(nn,ne))\"" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py --flow-name PathParamsTest '\"nodes\"' '\"edges\"'" timeout: 10 expected_exit_code: 0 weight: 3.0 @@ -50,7 +54,7 @@ success_criteria: - type: run_command description: "Flow uses Jira connector or task-specific managed HTTP fallback" - command: "python3 $TASK_DIR/check_managed_http_fallback.py '**/PathParamsTest*.flow' uipath-atlassian-jira path_params" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/connector_features/check_managed_http_fallback.py '**/PathParamsTest*.flow' uipath-atlassian-jira path_params" timeout: 10 expected_exit_code: 0 weight: 3.0 @@ -58,17 +62,16 @@ success_criteria: - type: run_command description: "Flow path parameters carry the issue key ENGCE-00000" - command: "python3 $TASK_DIR/check_path_param_value.py '**/PathParamsTest*.flow' ENGCE-00000" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/connector_features/check_path_param_value.py '**/PathParamsTest*.flow' ENGCE-00000" timeout: 10 expected_exit_code: 0 weight: 2.0 pass_threshold: 1.0 - - type: command_executed - description: "uip maestro flow validate was called" - tool_name: "Bash" - command_pattern: "(uip|\\$UIP)\\s+(maestro\\s+)?flow\\s+validate" - min_count: 1 + - type: run_command + description: "The completed flow validates" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py" + timeout: 30 + expected_exit_code: 0 weight: 2.0 pass_threshold: 1.0 - diff --git a/tests/tasks/uipath-maestro-flow/connector_features/query_params.yaml b/tests/tasks/uipath-maestro-flow/connector_features/query_params.yaml index c29eeea749..2fa84d23e8 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/query_params.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_features/query_params.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-ipe-query-params description: > Tests the query parameters IS feature โ€” configures a connector node with a @@ -23,7 +27,7 @@ initial_prompt: | success_criteria: - type: run_command description: "Flow file exists and is valid JSON" - command: "python3 -c \"import json,glob; flows=glob.glob('**/QueryParamsTest*.flow',recursive=True); assert flows, 'No flow found'; f=json.load(open(flows[0])); assert 'nodes' in f and 'edges' in f; nn=len(f['nodes']); ne=len(f['edges']); print('OK: %d nodes, %d edges'%(nn,ne))\"" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py --flow-name QueryParamsTest '\"nodes\"' '\"edges\"'" timeout: 10 expected_exit_code: 0 weight: 3.0 @@ -31,17 +35,16 @@ success_criteria: - type: run_command description: "Flow uses Google Tasks connector or task-specific managed HTTP fallback" - command: "python3 $TASK_DIR/check_managed_http_fallback.py '**/QueryParamsTest*.flow' uipath-google-tasks query_params" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/connector_features/check_managed_http_fallback.py '**/QueryParamsTest*.flow' uipath-google-tasks query_params" timeout: 10 expected_exit_code: 0 weight: 3.0 pass_threshold: 1.0 - - type: command_executed - description: "uip maestro flow validate was called" - tool_name: "Bash" - command_pattern: "(uip|\\$UIP)\\s+(maestro\\s+)?flow\\s+validate" - min_count: 1 + - type: run_command + description: "The completed flow validates" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py" + timeout: 30 + expected_exit_code: 0 weight: 2.0 pass_threshold: 1.0 - diff --git a/tests/tasks/uipath-maestro-flow/connector_features/searchable_joins.yaml b/tests/tasks/uipath-maestro-flow/connector_features/searchable_joins.yaml index 2a7eff1b45..2879bfa16f 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/searchable_joins.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_features/searchable_joins.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-ipe-searchable-joins description: > Tests the searchable joins IS feature โ€” configures a connector node with @@ -20,7 +24,7 @@ initial_prompt: | success_criteria: - type: run_command description: "Flow file exists and is valid JSON" - command: "python3 -c \"import json,glob; flows=glob.glob('**/SearchableJoinsTest*.flow',recursive=True); assert flows, 'No flow found'; f=json.load(open(flows[0])); assert 'nodes' in f and 'edges' in f; nn=len(f['nodes']); ne=len(f['edges']); print('OK: %d nodes, %d edges'%(nn,ne))\"" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py --flow-name SearchableJoinsTest '\"nodes\"' '\"edges\"'" timeout: 10 expected_exit_code: 0 weight: 3.0 @@ -28,17 +32,16 @@ success_criteria: - type: run_command description: "Flow has a connector node referencing uipath-salesforce-sfdc" - command: "python3 -c \"import json,glob; flows=glob.glob('**/SearchableJoinsTest*.flow',recursive=True); assert flows; content=open(flows[0]).read(); assert 'uipath-salesforce-sfdc' in content, 'Connector key not found'; print('OK: connector key present')\"" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py --flow-name SearchableJoinsTest 'uipath-salesforce-sfdc'" timeout: 10 expected_exit_code: 0 weight: 3.0 pass_threshold: 1.0 - - type: command_executed - description: "uip maestro flow validate was called" - tool_name: "Bash" - command_pattern: "(uip|\\$UIP)\\s+(maestro\\s+)?flow\\s+validate" - min_count: 1 + - type: run_command + description: "The completed flow validates" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py" + timeout: 30 + expected_exit_code: 0 weight: 2.0 pass_threshold: 1.0 - diff --git a/tests/tasks/uipath-maestro-flow/connector_features/slack-http-fallback/check_slack_http_fallback.py b/tests/tasks/uipath-maestro-flow/connector_features/slack-http-fallback/check_slack_http_fallback.py index f5bcfe18d5..41b6ed3b21 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/slack-http-fallback/check_slack_http_fallback.py +++ b/tests/tasks/uipath-maestro-flow/connector_features/slack-http-fallback/check_slack_http_fallback.py @@ -23,7 +23,6 @@ from __future__ import annotations -import glob import json import os import sys @@ -37,7 +36,7 @@ break _ROOT = os.path.dirname(_ROOT) sys.path.insert(0, _ROOT) -from _shared.flow_check import run_debug # noqa: E402 +from _shared.flow_check import find_flow_file, run_debug # noqa: E402 FLOW_GLOB = "**/SlackEmojiListTest*.flow" SLACK_KEY = "uipath-salesforce-slack" @@ -53,16 +52,12 @@ def _fail(message: str) -> NoReturn: def _read_flow() -> dict[str, Any]: - flows = sorted(glob.glob(FLOW_GLOB, recursive=True)) - if not flows: - _fail(f"No flow file matching {FLOW_GLOB}") - if len(flows) > 1: - _fail(f"Multiple flows match {FLOW_GLOB}: {flows}") - with open(flows[0], encoding="utf-8") as f: + path = find_flow_file(flow_glob=os.path.basename(FLOW_GLOB)) + with open(path, encoding="utf-8") as f: try: return json.load(f) except json.JSONDecodeError as e: - _fail(f"{flows[0]} is not valid JSON: {e}") + _fail(f"{path} is not valid JSON: {e}") def _normalise_detail(raw: Any) -> dict[str, Any]: @@ -90,7 +85,6 @@ def _is_slack_http_fallback(node: dict[str, Any]) -> bool: HTTP fallback and must not satisfy this check. """ node_type = str(node.get("type") or "").lower() - blob = json.dumps(node).lower() # Shape 2: connector-scoped Slack HTTP request activity. if "http-request" in node_type and SLACK_KEY in node_type: diff --git a/tests/tasks/uipath-maestro-flow/connector_features/slack-http-fallback/slack_http_fallback.yaml b/tests/tasks/uipath-maestro-flow/connector_features/slack-http-fallback/slack_http_fallback.yaml index dcedeb9bba..05c80214ca 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/slack-http-fallback/slack_http_fallback.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_features/slack-http-fallback/slack_http_fallback.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-slack-http-fallback description: > E2E test: a catalog connector (Slack, uipath-salesforce-slack) has no native @@ -16,38 +20,38 @@ run_limits: turn_timeout: 1200 initial_prompt: | - Create a new Flow project called "SlackEmojiListTest" with a manual trigger - that lists the custom emoji for a team. Wire the manual trigger to the node that + Create a new Flow project called "SlackEmojiListTest" inside a solution of the + same name, with a manual trigger that lists the custom emoji for a team. Wire + the manual trigger to the node that lists the emoji. Use the Slack connection in the `Shared/uipath-maestro-flow` folder. Validate the final flow file. Once it validates, debug it. - Use `--output json` on every `uip` command whose output you parse. - Do NOT ask for approval, confirmation, or feedback โ€” build, validate, and - debug the flow end-to-end in a single pass. + Build, validate, and debug the flow end-to-end in a single pass. Do NOT ask + for approval, confirmation, or feedback. success_criteria: - - type: command_executed - description: "uip maestro flow validate was called" - tool_name: "Bash" - command_pattern: "(uip|\\$UIP)\\s+(maestro\\s+)?flow\\s+validate" - min_count: 1 + - type: run_command + description: "The completed flow validates" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py" + timeout: 30 + expected_exit_code: 0 weight: 2.0 pass_threshold: 1.0 - type: command_executed - description: "Agent debugged the flow with uip maestro flow debug" + description: "Advisory: live-v1 agent invoked flow debug" tool_name: "Bash" command_pattern: "(uip|\\$UIP)\\s+(maestro\\s+)?flow\\s+debug" min_count: 1 weight: 1.5 - pass_threshold: 1.0 + pass_threshold: 0.0 - type: run_command description: "Emoji list falls back to an HTTP-request node bound to the Slack connector (no native activity exists for it) and targets the emoji.list endpoint" - command: "python3 $TASK_DIR/check_slack_http_fallback.py check_fallback" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/connector_features/slack-http-fallback/check_slack_http_fallback.py check_fallback" timeout: 30 expected_exit_code: 0 weight: 3.0 @@ -55,7 +59,7 @@ success_criteria: - type: run_command description: "uip maestro flow debug finishes with finalStatus Completed against the live Slack connection" - command: "python3 $TASK_DIR/check_slack_http_fallback.py check_debug" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/connector_features/slack-http-fallback/check_slack_http_fallback.py check_debug" timeout: 720 expected_exit_code: 0 weight: 6.0 diff --git a/tests/tasks/uipath-maestro-flow/connector_features/testmanager_attachments/testmanager_attachments.yaml b/tests/tasks/uipath-maestro-flow/connector_features/testmanager_attachments/testmanager_attachments.yaml index 7c8a562ce1..79766cbfb5 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/testmanager_attachments/testmanager_attachments.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_features/testmanager_attachments/testmanager_attachments.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. # skip:true, tenant-gated. CONFIRMED available in Maestro Flow (2026-06-16): # `uip maestro flow registry pull` then search returns the # uipath.connector.uipath-uipath-testmanager. nodes for this object group. A stale manifest @@ -8,9 +12,9 @@ description: > Coverage (Maestro Flow connector): agent uses the uipath-maestro-flow skill to build a Flow that uses every UiPath Test Manager connector node for the attachment group โ€” adding one connector node per operation. Drives - `registry pull` -> `registry search`/`get` -> `node add` -> `flow validate`. - Graded on those commands plus the produced .flow referencing each - uipath.connector.uipath-uipath-testmanager.* node type. + connector discovery, authoring, and validation. Graded on the produced .flow + referencing each uipath.connector.uipath-uipath-testmanager.* node type and + validating successfully; registry-refresh telemetry is advisory. tags: [uipath-maestro-flow, e2e, mode:build, lifecycle:generate, shape:multi-node, connector] skip: true @@ -34,32 +38,32 @@ initial_prompt: | success_criteria: - type: command_executed - description: "Agent refreshed the node manifest (`uip maestro flow registry pull`) before searching" + description: "Advisory: live-v1 agent refreshed the node manifest before searching" tool_name: "Bash" command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+registry\s+pull' min_count: 1 weight: 1.0 - pass_threshold: 1.0 + pass_threshold: 0.0 - - type: command_executed - description: "Agent added connector nodes to the flow (`uip maestro flow node add`)" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+node\s+add' - min_count: 1 + - type: run_command + description: "The flow contains Test Manager connector nodes" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py 'uipath-uipath-testmanager.'" + timeout: 30 + expected_exit_code: 0 weight: 1.5 pass_threshold: 1.0 - - type: command_executed - description: "Agent validated the flow (`uip maestro flow validate`)" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+validate' - min_count: 1 + - type: run_command + description: "The completed flow validates" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py" + timeout: 30 + expected_exit_code: 0 weight: 1.5 pass_threshold: 1.0 - type: run_command description: "Produced .flow references every attachment connector node type (4 nodes)" - command: 'for n in upload-attachment get-attachments download-attachment delete-attachment; do grep -rqsF "uipath-uipath-testmanager.$n" --include="*.flow" . || { echo "missing node $n"; exit 1; }; done' + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py 'uipath-uipath-testmanager.upload-attachment' 'uipath-uipath-testmanager.get-attachments' 'uipath-uipath-testmanager.download-attachment' 'uipath-uipath-testmanager.delete-attachment'" timeout: 30 expected_exit_code: 0 weight: 3.0 diff --git a/tests/tasks/uipath-maestro-flow/connector_features/testmanager_crud_grounded/check.py b/tests/tasks/uipath-maestro-flow/connector_features/testmanager_crud_grounded/check.py index 938a938419..d8e9914d64 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/testmanager_crud_grounded/check.py +++ b/tests/tasks/uipath-maestro-flow/connector_features/testmanager_crud_grounded/check.py @@ -27,6 +27,12 @@ def main(): except OSError: fail("result.json missing โ€” agent did not record the create/get round-trip") + if len(sys.argv) > 1 and sys.argv[1] == "--execution-evidence": + if not all(res.get(key) for key in ("created_name", "retrieved_name", "id")): + fail("result.json is missing created_name, retrieved_name, or id") + print("OK: execution result recorded") + return + name = seed["name"] if res.get("created_name") != name: fail(f"created_name {res.get('created_name')!r} != seeded {name!r}") diff --git a/tests/tasks/uipath-maestro-flow/connector_features/testmanager_crud_grounded/testmanager_crud_grounded.yaml b/tests/tasks/uipath-maestro-flow/connector_features/testmanager_crud_grounded/testmanager_crud_grounded.yaml index 28df13cfa5..b4f897bfff 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/testmanager_crud_grounded/testmanager_crud_grounded.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_features/testmanager_crud_grounded/testmanager_crud_grounded.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. # PROTOTYPE โ€” data-grounded on the Maestro Flow surface. skip:true. # Agent builds a flow (create-test-case node -> get-test-case node -> output the # retrieved name) and EXECUTES it (`uip maestro flow debug`) to produce the @@ -10,8 +14,9 @@ description: > Data-grounded (Maestro Flow): agent builds a flow that creates a Test Case via the uipath-uipath-testmanager connector node with a unique seeded name, gets it back, outputs the retrieved name, and debugs (executes) the flow โ€” then records - the round-trip. Graded on pull/node-add/validate/debug AND on retrieved == - created (real data through the connector, executed). + the round-trip. Graded on the connector artifacts, successful validation, and + retrieved == created through the executed flow; registry-refresh telemetry is + advisory. tags: [uipath-maestro-flow, e2e, mode:build, lifecycle:generate, shape:multi-node, connector] skip: true @@ -39,40 +44,40 @@ initial_prompt: | success_criteria: - type: command_executed - description: "Agent refreshed the node manifest (`uip maestro flow registry pull`)" + description: "Advisory: live-v1 agent refreshed the node manifest" tool_name: "Bash" command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+registry\s+pull' min_count: 1 weight: 1.0 - pass_threshold: 1.0 + pass_threshold: 0.0 - - type: command_executed - description: "Agent added connector nodes (`uip maestro flow node add`)" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+node\s+add' - min_count: 1 + - type: run_command + description: "The flow contains Test Manager create and get connector nodes" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py 'uipath-uipath-testmanager.create-test-case' 'uipath-uipath-testmanager.get-test-case'" + timeout: 30 + expected_exit_code: 0 weight: 1.0 pass_threshold: 1.0 - - type: command_executed - description: "Agent validated the flow (`uip maestro flow validate`)" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+validate' - min_count: 1 + - type: run_command + description: "The completed flow validates" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py" + timeout: 30 + expected_exit_code: 0 weight: 1.0 pass_threshold: 1.0 - - type: command_executed - description: "Agent executed the flow (`uip maestro flow debug`)" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+debug' - min_count: 1 + - type: run_command + description: "Execution produced the seeded create/get round-trip result" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/connector_features/testmanager_crud_grounded/check.py --execution-evidence" + timeout: 30 + expected_exit_code: 0 weight: 1.5 pass_threshold: 1.0 - type: run_command description: "Data-grounded: retrieved name == created (seeded) name โ€” real data round-tripped through the executed flow" - command: "python3 $TASK_DIR/check.py" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/connector_features/testmanager_crud_grounded/check.py" timeout: 30 expected_exit_code: 0 weight: 5.0 diff --git a/tests/tasks/uipath-maestro-flow/connector_features/testmanager_execution_results/testmanager_execution_results.yaml b/tests/tasks/uipath-maestro-flow/connector_features/testmanager_execution_results/testmanager_execution_results.yaml index 92aecc8cd6..11183d547d 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/testmanager_execution_results/testmanager_execution_results.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_features/testmanager_execution_results/testmanager_execution_results.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. # skip:true, tenant-gated. CONFIRMED available in Maestro Flow (2026-06-16): # `uip maestro flow registry pull` then search returns the # uipath.connector.uipath-uipath-testmanager. nodes for this object group. A stale manifest @@ -8,9 +12,9 @@ description: > Coverage (Maestro Flow connector): agent uses the uipath-maestro-flow skill to build a Flow that uses every UiPath Test Manager connector node for the execution and results group โ€” adding one connector node per operation. Drives - `registry pull` -> `registry search`/`get` -> `node add` -> `flow validate`. - Graded on those commands plus the produced .flow referencing each - uipath.connector.uipath-uipath-testmanager.* node type. + connector discovery, authoring, and validation. Graded on the produced .flow + referencing each uipath.connector.uipath-uipath-testmanager.* node type and + validating successfully; registry-refresh telemetry is advisory. tags: [uipath-maestro-flow, e2e, mode:build, lifecycle:generate, shape:multi-node, connector] skip: true @@ -39,32 +43,32 @@ initial_prompt: | success_criteria: - type: command_executed - description: "Agent refreshed the node manifest (`uip maestro flow registry pull`) before searching" + description: "Advisory: live-v1 agent refreshed the node manifest before searching" tool_name: "Bash" command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+registry\s+pull' min_count: 1 weight: 1.0 - pass_threshold: 1.0 + pass_threshold: 0.0 - - type: command_executed - description: "Agent added connector nodes to the flow (`uip maestro flow node add`)" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+node\s+add' - min_count: 1 + - type: run_command + description: "The flow contains Test Manager connector nodes" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py 'uipath-uipath-testmanager.'" + timeout: 30 + expected_exit_code: 0 weight: 1.5 pass_threshold: 1.0 - - type: command_executed - description: "Agent validated the flow (`uip maestro flow validate`)" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+validate' - min_count: 1 + - type: run_command + description: "The completed flow validates" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py" + timeout: 30 + expected_exit_code: 0 weight: 1.5 pass_threshold: 1.0 - type: run_command description: "Produced .flow references every execution and results connector node type (9 nodes)" - command: 'for n in get-test-execution delete-test-execution get-test-case-logs-of-test-execution get-test-case-log get-robot-logs get-assertions download-assertion get-test-steps get-test-step-logs; do grep -rqsF "uipath-uipath-testmanager.$n" --include="*.flow" . || { echo "missing node $n"; exit 1; }; done' + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py 'uipath-uipath-testmanager.get-test-execution' 'uipath-uipath-testmanager.delete-test-execution' 'uipath-uipath-testmanager.get-test-case-logs-of-test-execution' 'uipath-uipath-testmanager.get-test-case-log' 'uipath-uipath-testmanager.get-robot-logs' 'uipath-uipath-testmanager.get-assertions' 'uipath-uipath-testmanager.download-assertion' 'uipath-uipath-testmanager.get-test-steps' 'uipath-uipath-testmanager.get-test-step-logs'" timeout: 30 expected_exit_code: 0 weight: 3.0 diff --git a/tests/tasks/uipath-maestro-flow/connector_features/testmanager_generic_records/testmanager_generic_records.yaml b/tests/tasks/uipath-maestro-flow/connector_features/testmanager_generic_records/testmanager_generic_records.yaml index dc8caf370e..9ec8597c4f 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/testmanager_generic_records/testmanager_generic_records.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_features/testmanager_generic_records/testmanager_generic_records.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. # skip:true, tenant-gated. CONFIRMED available in Maestro Flow (2026-06-16): # `uip maestro flow registry pull` then search returns the # uipath.connector.uipath-uipath-testmanager. nodes for this object group. A stale manifest @@ -8,9 +12,9 @@ description: > Coverage (Maestro Flow connector): agent uses the uipath-maestro-flow skill to build a Flow that uses every UiPath Test Manager connector node for the generic record group โ€” adding one connector node per operation. Drives - `registry pull` -> `registry search`/`get` -> `node add` -> `flow validate`. - Graded on those commands plus the produced .flow referencing each - uipath.connector.uipath-uipath-testmanager.* node type. + connector discovery, authoring, and validation. Graded on the produced .flow + referencing each uipath.connector.uipath-uipath-testmanager.* node type and + validating successfully; registry-refresh telemetry is advisory. tags: [uipath-maestro-flow, e2e, mode:build, lifecycle:generate, shape:multi-node, connector] skip: true @@ -35,32 +39,32 @@ initial_prompt: | success_criteria: - type: command_executed - description: "Agent refreshed the node manifest (`uip maestro flow registry pull`) before searching" + description: "Advisory: live-v1 agent refreshed the node manifest before searching" tool_name: "Bash" command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+registry\s+pull' min_count: 1 weight: 1.0 - pass_threshold: 1.0 + pass_threshold: 0.0 - - type: command_executed - description: "Agent added connector nodes to the flow (`uip maestro flow node add`)" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+node\s+add' - min_count: 1 + - type: run_command + description: "The flow contains Test Manager connector nodes" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py 'uipath-uipath-testmanager.'" + timeout: 30 + expected_exit_code: 0 weight: 1.5 pass_threshold: 1.0 - - type: command_executed - description: "Agent validated the flow (`uip maestro flow validate`)" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+validate' - min_count: 1 + - type: run_command + description: "The completed flow validates" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py" + timeout: 30 + expected_exit_code: 0 weight: 1.5 pass_threshold: 1.0 - type: run_command description: "Produced .flow references every generic record connector node type (5 nodes)" - command: 'for n in insert-record get-record update-record delete-record list-records; do grep -rqsF "uipath-uipath-testmanager.$n" --include="*.flow" . || { echo "missing node $n"; exit 1; }; done' + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py 'uipath-uipath-testmanager.insert-record' 'uipath-uipath-testmanager.get-record' 'uipath-uipath-testmanager.update-record' 'uipath-uipath-testmanager.delete-record' 'uipath-uipath-testmanager.list-records'" timeout: 30 expected_exit_code: 0 weight: 3.0 diff --git a/tests/tasks/uipath-maestro-flow/connector_features/testmanager_requirement_lifecycle/testmanager_requirement_lifecycle.yaml b/tests/tasks/uipath-maestro-flow/connector_features/testmanager_requirement_lifecycle/testmanager_requirement_lifecycle.yaml index b02e5261b1..97ca38cac5 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/testmanager_requirement_lifecycle/testmanager_requirement_lifecycle.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_features/testmanager_requirement_lifecycle/testmanager_requirement_lifecycle.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. # skip:true, tenant-gated. CONFIRMED available in Maestro Flow (2026-06-16): # `uip maestro flow registry pull` then search returns the # uipath.connector.uipath-uipath-testmanager. nodes for this object group. A stale manifest @@ -8,9 +12,9 @@ description: > Coverage (Maestro Flow connector): agent uses the uipath-maestro-flow skill to build a Flow that uses every UiPath Test Manager connector node for the Requirement group โ€” adding one connector node per operation. Drives - `registry pull` -> `registry search`/`get` -> `node add` -> `flow validate`. - Graded on those commands plus the produced .flow referencing each - uipath.connector.uipath-uipath-testmanager.* node type. + connector discovery, authoring, and validation. Graded on the produced .flow + referencing each uipath.connector.uipath-uipath-testmanager.* node type and + validating successfully; registry-refresh telemetry is advisory. tags: [uipath-maestro-flow, e2e, mode:build, lifecycle:generate, shape:multi-node, connector] skip: true @@ -36,32 +40,32 @@ initial_prompt: | success_criteria: - type: command_executed - description: "Agent refreshed the node manifest (`uip maestro flow registry pull`) before searching" + description: "Advisory: live-v1 agent refreshed the node manifest before searching" tool_name: "Bash" command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+registry\s+pull' min_count: 1 weight: 1.0 - pass_threshold: 1.0 + pass_threshold: 0.0 - - type: command_executed - description: "Agent added connector nodes to the flow (`uip maestro flow node add`)" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+node\s+add' - min_count: 1 + - type: run_command + description: "The flow contains Test Manager connector nodes" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py 'uipath-uipath-testmanager.'" + timeout: 30 + expected_exit_code: 0 weight: 1.5 pass_threshold: 1.0 - - type: command_executed - description: "Agent validated the flow (`uip maestro flow validate`)" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+validate' - min_count: 1 + - type: run_command + description: "The completed flow validates" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py" + timeout: 30 + expected_exit_code: 0 weight: 1.5 pass_threshold: 1.0 - type: run_command description: "Produced .flow references every Requirement connector node type (6 nodes)" - command: 'for n in create-requirement get-requirement update-requirement delete-requirement assign-test-cases-to-requirement get-assigned-test-cases-for-req; do grep -rqsF "uipath-uipath-testmanager.$n" --include="*.flow" . || { echo "missing node $n"; exit 1; }; done' + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py 'uipath-uipath-testmanager.create-requirement' 'uipath-uipath-testmanager.get-requirement' 'uipath-uipath-testmanager.update-requirement' 'uipath-uipath-testmanager.delete-requirement' 'uipath-uipath-testmanager.assign-test-cases-to-requirement' 'uipath-uipath-testmanager.get-assigned-test-cases-for-req'" timeout: 30 expected_exit_code: 0 weight: 3.0 diff --git a/tests/tasks/uipath-maestro-flow/connector_features/testmanager_testcase_lifecycle/testmanager_testcase_lifecycle.yaml b/tests/tasks/uipath-maestro-flow/connector_features/testmanager_testcase_lifecycle/testmanager_testcase_lifecycle.yaml index 83716c6a24..7e252fdb75 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/testmanager_testcase_lifecycle/testmanager_testcase_lifecycle.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_features/testmanager_testcase_lifecycle/testmanager_testcase_lifecycle.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. # skip:true, tenant-gated. CONFIRMED available in Maestro Flow (2026-06-16): # `uip maestro flow registry pull` then search returns the # uipath.connector.uipath-uipath-testmanager. nodes for this object group. A stale manifest @@ -8,9 +12,9 @@ description: > Coverage (Maestro Flow connector): agent uses the uipath-maestro-flow skill to build a Flow that uses every UiPath Test Manager connector node for the TestCase group โ€” adding one connector node per operation. Drives - `registry pull` -> `registry search`/`get` -> `node add` -> `flow validate`. - Graded on those commands plus the produced .flow referencing each - uipath.connector.uipath-uipath-testmanager.* node type. + connector discovery, authoring, and validation. Graded on the produced .flow + referencing each uipath.connector.uipath-uipath-testmanager.* node type and + validating successfully; registry-refresh telemetry is advisory. tags: [uipath-maestro-flow, e2e, mode:build, lifecycle:generate, shape:multi-node, connector] skip: true @@ -35,32 +39,32 @@ initial_prompt: | success_criteria: - type: command_executed - description: "Agent refreshed the node manifest (`uip maestro flow registry pull`) before searching" + description: "Advisory: live-v1 agent refreshed the node manifest before searching" tool_name: "Bash" command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+registry\s+pull' min_count: 1 weight: 1.0 - pass_threshold: 1.0 + pass_threshold: 0.0 - - type: command_executed - description: "Agent added connector nodes to the flow (`uip maestro flow node add`)" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+node\s+add' - min_count: 1 + - type: run_command + description: "The flow contains Test Manager connector nodes" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py 'uipath-uipath-testmanager.'" + timeout: 30 + expected_exit_code: 0 weight: 1.5 pass_threshold: 1.0 - - type: command_executed - description: "Agent validated the flow (`uip maestro flow validate`)" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+validate' - min_count: 1 + - type: run_command + description: "The completed flow validates" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py" + timeout: 30 + expected_exit_code: 0 weight: 1.5 pass_threshold: 1.0 - type: run_command description: "Produced .flow references every TestCase connector node type (5 nodes)" - command: 'for n in create-test-case get-test-case update-test-case delete-test-case execute-test-cases; do grep -rqsF "uipath-uipath-testmanager.$n" --include="*.flow" . || { echo "missing node $n"; exit 1; }; done' + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py 'uipath-uipath-testmanager.create-test-case' 'uipath-uipath-testmanager.get-test-case' 'uipath-uipath-testmanager.update-test-case' 'uipath-uipath-testmanager.delete-test-case' 'uipath-uipath-testmanager.execute-test-cases'" timeout: 30 expected_exit_code: 0 weight: 3.0 diff --git a/tests/tasks/uipath-maestro-flow/connector_features/testmanager_testset_lifecycle/testmanager_testset_lifecycle.yaml b/tests/tasks/uipath-maestro-flow/connector_features/testmanager_testset_lifecycle/testmanager_testset_lifecycle.yaml index cd366f1db8..869a4d162d 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/testmanager_testset_lifecycle/testmanager_testset_lifecycle.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_features/testmanager_testset_lifecycle/testmanager_testset_lifecycle.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. # skip:true, tenant-gated. CONFIRMED available in Maestro Flow (2026-06-16): # `uip maestro flow registry pull` then search returns the # uipath.connector.uipath-uipath-testmanager. nodes for this object group. A stale manifest @@ -8,9 +12,9 @@ description: > Coverage (Maestro Flow connector): agent uses the uipath-maestro-flow skill to build a Flow that uses every UiPath Test Manager connector node for the TestSet group โ€” adding one connector node per operation. Drives - `registry pull` -> `registry search`/`get` -> `node add` -> `flow validate`. - Graded on those commands plus the produced .flow referencing each - uipath.connector.uipath-uipath-testmanager.* node type. + connector discovery, authoring, and validation. Graded on the produced .flow + referencing each uipath.connector.uipath-uipath-testmanager.* node type and + validating successfully; registry-refresh telemetry is advisory. tags: [uipath-maestro-flow, e2e, mode:build, lifecycle:generate, shape:multi-node, connector] skip: true @@ -37,32 +41,32 @@ initial_prompt: | success_criteria: - type: command_executed - description: "Agent refreshed the node manifest (`uip maestro flow registry pull`) before searching" + description: "Advisory: live-v1 agent refreshed the node manifest before searching" tool_name: "Bash" command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+registry\s+pull' min_count: 1 weight: 1.0 - pass_threshold: 1.0 + pass_threshold: 0.0 - - type: command_executed - description: "Agent added connector nodes to the flow (`uip maestro flow node add`)" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+node\s+add' - min_count: 1 + - type: run_command + description: "The flow contains Test Manager connector nodes" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py 'uipath-uipath-testmanager.'" + timeout: 30 + expected_exit_code: 0 weight: 1.5 pass_threshold: 1.0 - - type: command_executed - description: "Agent validated the flow (`uip maestro flow validate`)" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+validate' - min_count: 1 + - type: run_command + description: "The completed flow validates" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py" + timeout: 30 + expected_exit_code: 0 weight: 1.5 pass_threshold: 1.0 - type: run_command description: "Produced .flow references every TestSet connector node type (7 nodes)" - command: 'for n in create-test-set get-test-set update-test-set delete-test-set assign-test-cases-to-test-set get-assigned-test-cases-for-test-set execute-test-set; do grep -rqsF "uipath-uipath-testmanager.$n" --include="*.flow" . || { echo "missing node $n"; exit 1; }; done' + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py 'uipath-uipath-testmanager.create-test-set' 'uipath-uipath-testmanager.get-test-set' 'uipath-uipath-testmanager.update-test-set' 'uipath-uipath-testmanager.delete-test-set' 'uipath-uipath-testmanager.assign-test-cases-to-test-set' 'uipath-uipath-testmanager.get-assigned-test-cases-for-test-set' 'uipath-uipath-testmanager.execute-test-set'" timeout: 30 expected_exit_code: 0 weight: 3.0 From 3645260a65b374c2a1355a2cf974458261dbc6a6 Mon Sep 17 00:00:00 2001 From: Tao Ma <51425734+tmatup@users.noreply.github.com> Date: Fri, 21 Aug 2026 17:37:09 -0700 Subject: [PATCH 05/52] test(maestro-flow): align IxP and context-grounding tasks (#2751) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Apply the same-ground measurement contract to the nine-task Batch 4 roster while preserving criterion weights and semantic intent. Route Flow discovery through shared helpers, make loop-specific telemetry report-only, remove authoring-loop prompt coaching, and cover the no-Flow CLI path. ๐Ÿค– Generated with Claude Code Co-Authored-By: [Claude](mailto:noreply@anthropic.com) --- .../uipath-maestro-flow/_shared/flow_check.py | 13 ++ .../_shared/flow_contains.py | 26 ++- .../_shared/test_batch4_alignment.py | 191 ++++++++++++++++++ .../_shared/test_same_ground_corpus.py | 21 +- .../batch_transform/batch_transform.yaml | 8 +- .../check_batch_transform_flow.py | 11 +- .../summarize/check_summarize_flow.py | 11 +- .../summarize/summarize.yaml | 8 +- .../ixp/check_project_selection.py | 65 ++++++ .../ixp/e2e_02_project_selection.yaml | 66 ++---- .../ixp/integration_handle_routing.yaml | 55 ++--- .../uipath-maestro-flow/ixp/routing.yaml | 42 ++-- .../ixp/routing_listing.yaml | 27 ++- .../ixp/routing_negative.yaml | 20 +- .../ixp/scaffold_minimal.yaml | 31 ++- .../ixp/scaffold_multinode.yaml | 26 ++- 16 files changed, 412 insertions(+), 209 deletions(-) create mode 100644 tests/tasks/uipath-maestro-flow/_shared/test_batch4_alignment.py create mode 100644 tests/tasks/uipath-maestro-flow/ixp/check_project_selection.py diff --git a/tests/tasks/uipath-maestro-flow/_shared/flow_check.py b/tests/tasks/uipath-maestro-flow/_shared/flow_check.py index f1e530c8dc..82ad7728e8 100644 --- a/tests/tasks/uipath-maestro-flow/_shared/flow_check.py +++ b/tests/tasks/uipath-maestro-flow/_shared/flow_check.py @@ -1576,6 +1576,19 @@ def find_flow_file( return matches[0] +def assert_no_flow_files() -> None: + """Require that the sandbox contains no ``.flow`` artifact anywhere. + + This is the negative counterpart to :func:`find_flow_files` for read-only + tasks. It deliberately scans every location because any emitted Flow is a + failure; there is no preferred artifact to select in the absence case. + """ + matches = sorted(glob.glob("**/*.flow", recursive=True)) + if matches: + joined = "\n - ".join(matches) + _fail(f"Unexpected .flow file(s) found:\n - {joined}") + + # โ”€โ”€ Internals โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ diff --git a/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py b/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py index df70f31014..9734adb878 100644 --- a/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py +++ b/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py @@ -24,6 +24,8 @@ expected_exit_code: 0) instead of inverting a positive check โ€” a bare exit 1 cannot distinguish "pattern absent" from "no flow found" + --expect-none require that no ``.flow`` file exists anywhere in the + sandbox (for read-only tasks) With no arguments this asserts only that a ``.flow`` file exists. All positive assertions (substrings + ``--regex``) must be satisfied by a single file; @@ -46,7 +48,7 @@ import sys sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -from flow_check import find_flow_files # noqa: E402 +from flow_check import assert_no_flow_files, find_flow_files # noqa: E402 def _parse(argv: list[str]): @@ -54,6 +56,7 @@ def _parse(argv: list[str]): regexes: list[str] = [] absent: list[str] = [] flow_name: str | None = None + expect_none = False it = iter(argv) for arg in it: try: @@ -63,16 +66,33 @@ def _parse(argv: list[str]): absent.append(next(it)) elif arg == "--flow-name": flow_name = next(it) + elif arg == "--expect-none": + expect_none = True else: substrings.append(arg) except StopIteration: print(f"FAIL: {arg} requires a value", file=sys.stderr) sys.exit(2) - return substrings, regexes, absent, flow_name + return substrings, regexes, absent, flow_name, expect_none def main(argv: list[str]) -> int: - substrings, regexes, absent, flow_name = _parse(argv) + substrings, regexes, absent, flow_name, expect_none = _parse(argv) + + if expect_none: + if substrings or regexes or absent or flow_name is not None: + print( + "FAIL: --expect-none cannot be combined with other assertions", + file=sys.stderr, + ) + return 2 + try: + assert_no_flow_files() + except SystemExit as exc: + print(str(exc), file=sys.stderr) + return exc.code if isinstance(exc.code, int) else 1 + print("OK: no .flow files found") + return 0 try: flows = find_flow_files() diff --git a/tests/tasks/uipath-maestro-flow/_shared/test_batch4_alignment.py b/tests/tasks/uipath-maestro-flow/_shared/test_batch4_alignment.py new file mode 100644 index 0000000000..4b8bcd8097 --- /dev/null +++ b/tests/tasks/uipath-maestro-flow/_shared/test_batch4_alignment.py @@ -0,0 +1,191 @@ +"""Regression coverage for Batch 4 same-ground checker routing.""" + +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from flow_check import assert_no_flow_files # noqa: E402 + +TASK_ROOT = Path(__file__).resolve().parent.parent + + +def _write_flow(path: Path, nodes: list[dict], **extra) -> None: + path.write_text(json.dumps({"nodes": nodes, "edges": [], **extra}), encoding="utf-8") + + +def _run(relative: str, cwd: Path) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, str(TASK_ROOT / relative)], + cwd=cwd, + capture_output=True, + text=True, + check=False, + ) + + +def test_no_flow_assertion_accepts_empty_sandbox(tmp_path: Path, monkeypatch) -> None: + monkeypatch.chdir(tmp_path) + assert_no_flow_files() + + +def test_no_flow_assertion_rejects_nested_flow(tmp_path: Path, monkeypatch) -> None: + monkeypatch.chdir(tmp_path) + nested = tmp_path / "Solution" / "Project" + nested.mkdir(parents=True) + _write_flow(nested / "Unexpected.flow", []) + + with pytest.raises(SystemExit, match="Unexpected .flow"): + assert_no_flow_files() + + +def test_flow_contains_expect_none_cli(tmp_path: Path) -> None: + result = subprocess.run( + [sys.executable, str(Path(__file__).parent / "flow_contains.py"), "--expect-none"], + cwd=tmp_path, + capture_output=True, + text=True, + check=False, + ) + + assert result.returncode == 0, result.stderr or result.stdout + + +def test_ixp_project_selection_accepts_root_sdk_emit(tmp_path: Path) -> None: + row = tmp_path / "skill-flow-ixp-e2e-project-selection" / "aviation" + row.mkdir(parents=True) + model = "aviation-investigation-final-report-demo-a42b1d4d-ixp" + _write_flow( + row / "Aviation.flow", + [ + { + "id": "extract", + "type": f"uipath.ixp.{model}.extract", + "inputs": {"modelName": model}, + } + ], + ) + + result = _run("ixp/check_project_selection.py", cwd=row) + + assert result.returncode == 0, result.stderr or result.stdout + + +def test_ixp_project_selection_rejects_mismatched_model(tmp_path: Path) -> None: + row = tmp_path / "skill-flow-ixp-e2e-project-selection" / "birth-certificate" + row.mkdir(parents=True) + _write_flow( + row / "Birth.flow", + [ + { + "id": "extract", + "type": "uipath.ixp.wrong-model.extract", + "inputs": {"modelName": "wrong-model"}, + } + ], + ) + + result = _run("ixp/check_project_selection.py", cwd=row) + + assert result.returncode == 1 + + +def test_batch_transform_checker_accepts_root_sdk_emit(tmp_path: Path) -> None: + _write_flow( + tmp_path / "BatchTransformDemo.flow", + [ + {"id": "trigger", "type": "core.trigger.manual"}, + { + "id": "batch", + "type": "uipath.pattern.batch-transform", + "typeVersion": "1.0", + "inputs": { + "attachment": "=js:$vars.trigger.output.csvFile", + "prompt": "Classify each row", + "outputColumns": [ + {"name": "Category", "description": "classification"} + ], + }, + "outputs": {"output": {"source": "=response"}}, + }, + { + "id": "end", + "type": "core.control.end", + "outputs": {"result": {"source": "=js:$vars.batch.output"}}, + }, + ], + variables={ + "globals": [ + { + "id": "csvFile", + "direction": "in", + "type": "file", + "triggerNodeId": "trigger", + }, + {"id": "result", "direction": "out", "type": "file"}, + ] + }, + ) + + result = _run( + "context-grounding/batch_transform/check_batch_transform_flow.py", + cwd=tmp_path, + ) + + assert result.returncode == 0, result.stderr or result.stdout + + +def test_summarize_checker_accepts_root_sdk_emit(tmp_path: Path) -> None: + _write_flow( + tmp_path / "SummarizeDemo.flow", + [ + {"id": "trigger", "type": "core.trigger.manual"}, + { + "id": "summary_node", + "type": "uipath.pattern.deep-rag", + "typeVersion": "1.0", + "inputs": { + "attachment": "=js:$vars.trigger.output.documentFile", + "prompt": "Summarize the document", + "returnCitations": True, + }, + "outputs": {"output": {"source": "=response"}}, + }, + { + "id": "end", + "type": "core.control.end", + "outputs": { + "summary": { + "source": "=js:$vars.summary_node.output.content.Text" + }, + "citations": { + "source": "=js:$vars.summary_node.output.content.Citations" + }, + }, + }, + ], + variables={ + "globals": [ + { + "id": "documentFile", + "direction": "in", + "type": "file", + "triggerNodeId": "trigger", + }, + {"id": "summary", "direction": "out", "type": "string"}, + {"id": "citations", "direction": "out", "type": "array"}, + ] + }, + ) + + result = _run( + "context-grounding/summarize/check_summarize_flow.py", + cwd=tmp_path, + ) + + assert result.returncode == 0, result.stderr or result.stdout diff --git a/tests/tasks/uipath-maestro-flow/_shared/test_same_ground_corpus.py b/tests/tasks/uipath-maestro-flow/_shared/test_same_ground_corpus.py index 3c888d746a..a769c76eae 100644 --- a/tests/tasks/uipath-maestro-flow/_shared/test_same_ground_corpus.py +++ b/tests/tasks/uipath-maestro-flow/_shared/test_same_ground_corpus.py @@ -20,33 +20,14 @@ "evaluate/local_crud.yaml", "evaluate/no_auto_upload.yaml", "evaluate/simulation/simulation_crud.yaml", - "ixp/e2e_02_project_selection.yaml", - "ixp/integration_handle_routing.yaml", - "ixp/routing.yaml", - "ixp/routing_listing.yaml", - "ixp/scaffold_minimal.yaml", - "ixp/scaffold_multinode.yaml", } -SKILL_TELEMETRY_ALLOWLIST = { - "ixp/e2e_02_project_selection.yaml", - "ixp/integration_handle_routing.yaml", - "ixp/scaffold_minimal.yaml", - "ixp/scaffold_multinode.yaml", -} +SKILL_TELEMETRY_ALLOWLIST = set() FORBIDDEN_PROMPT_ALLOWLIST = { - "context-grounding/batch_transform/batch_transform.yaml", - "context-grounding/summarize/summarize.yaml", "evaluate/inline_agent_eval/inline_agent_eval.yaml", "evaluate/local_crud.yaml", "evaluate/simulation/simulation_crud.yaml", - "ixp/e2e_02_project_selection.yaml", - "ixp/integration_handle_routing.yaml", - "ixp/routing.yaml", - "ixp/routing_negative.yaml", - "ixp/scaffold_minimal.yaml", - "ixp/scaffold_multinode.yaml", } # These born-neutral escalation tasks are outside the recipe-edit sweep. Their diff --git a/tests/tasks/uipath-maestro-flow/context-grounding/batch_transform/batch_transform.yaml b/tests/tasks/uipath-maestro-flow/context-grounding/batch_transform/batch_transform.yaml index a81538c530..7d51e9ca8f 100644 --- a/tests/tasks/uipath-maestro-flow/context-grounding/batch_transform/batch_transform.yaml +++ b/tests/tasks/uipath-maestro-flow/context-grounding/batch_transform/batch_transform.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-batch-transform description: > Create a UiPath Flow that runs a Batch Transform pattern node over a CSV @@ -41,7 +45,7 @@ initial_prompt: | the result file handle as a flow output variable named `result` mapped via `outputs.result.source: "=js:$vars..output"`. - Do NOT run flow debug โ€” just validate the flow. + Validate the completed flow. Do NOT ask for approval, confirmation, or feedback. Do NOT pause between planning and implementation. Build the complete flow end-to-end in a single pass. Before starting, follow the documented workflow steps exactly โ€” in particular, the batch-transform plugin's @@ -60,7 +64,7 @@ success_criteria: # โ”€โ”€ Structural checks (node type, inputs, outputColumns shape) โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ - type: run_command description: "Flow has a Batch Transform node with required inputs and the {name,description} outputColumns shape" - command: "python3 $TASK_DIR/check_batch_transform_flow.py" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/context-grounding/batch_transform/check_batch_transform_flow.py" timeout: 30 expected_exit_code: 0 weight: 5.0 diff --git a/tests/tasks/uipath-maestro-flow/context-grounding/batch_transform/check_batch_transform_flow.py b/tests/tasks/uipath-maestro-flow/context-grounding/batch_transform/check_batch_transform_flow.py index 430792d492..a61e661587 100644 --- a/tests/tasks/uipath-maestro-flow/context-grounding/batch_transform/check_batch_transform_flow.py +++ b/tests/tasks/uipath-maestro-flow/context-grounding/batch_transform/check_batch_transform_flow.py @@ -20,12 +20,15 @@ reference. """ -import glob import json +import os import re import sys from typing import NoReturn +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "_shared")) +from flow_check import find_flow_file # noqa: E402 + NODE_TYPE = "uipath.pattern.batch-transform" EXPECTED_TYPE_VERSION = "1.0" EXPECTED_OUTPUT_SOURCE = "=response" @@ -45,10 +48,8 @@ def _fail(msg: str) -> NoReturn: def _read_flow() -> dict: - flows = glob.glob("**/BatchTransformDemo*.flow", recursive=True) - if not flows: - _fail("no BatchTransformDemo*.flow found under cwd") - with open(flows[0]) as f: + path = find_flow_file(flow_glob="BatchTransformDemo*.flow") + with open(path) as f: return json.load(f) diff --git a/tests/tasks/uipath-maestro-flow/context-grounding/summarize/check_summarize_flow.py b/tests/tasks/uipath-maestro-flow/context-grounding/summarize/check_summarize_flow.py index 93e5026af5..dad14c977a 100644 --- a/tests/tasks/uipath-maestro-flow/context-grounding/summarize/check_summarize_flow.py +++ b/tests/tasks/uipath-maestro-flow/context-grounding/summarize/check_summarize_flow.py @@ -22,12 +22,15 @@ PascalCase; lowercase resolves to undefined at runtime). """ -import glob import json +import os import re import sys from typing import NoReturn +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "_shared")) +from flow_check import find_flow_file # noqa: E402 + NODE_TYPE = "uipath.pattern.deep-rag" EXPECTED_TYPE_VERSION = "1.0" EXPECTED_OUTPUT_SOURCE = "=response" @@ -43,10 +46,8 @@ def _fail(msg: str) -> NoReturn: def _read_flow() -> dict: - flows = glob.glob("**/SummarizeDemo*.flow", recursive=True) - if not flows: - _fail("no SummarizeDemo*.flow found under cwd") - with open(flows[0]) as f: + path = find_flow_file(flow_glob="SummarizeDemo*.flow") + with open(path) as f: return json.load(f) diff --git a/tests/tasks/uipath-maestro-flow/context-grounding/summarize/summarize.yaml b/tests/tasks/uipath-maestro-flow/context-grounding/summarize/summarize.yaml index 74c13bbf2a..f0a04b3cd9 100644 --- a/tests/tasks/uipath-maestro-flow/context-grounding/summarize/summarize.yaml +++ b/tests/tasks/uipath-maestro-flow/context-grounding/summarize/summarize.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-summarize description: > Create a UiPath Flow that runs a Summarize pattern node (`uipath.pattern.deep-rag`) @@ -43,7 +47,7 @@ initial_prompt: | Note the PascalCase nested field names โ€” lowercase `text`/`citations` would resolve to `undefined` at runtime. - Do NOT run flow debug โ€” just validate the flow. + Validate the completed flow. Do NOT ask for approval, confirmation, or feedback. Do NOT pause between planning and implementation. Build the complete flow end-to-end in a single pass. Before starting, follow the documented workflow steps exactly โ€” in particular, note that the wire-level node type is @@ -62,7 +66,7 @@ success_criteria: # โ”€โ”€ Structural checks (node type, inputs, returnCitations boolean) โ”€โ”€โ”€โ”€โ”€ - type: run_command description: "Flow has a Summarize (deep-rag) node with prompt, attachment wired, and returnCitations=true" - command: "python3 $TASK_DIR/check_summarize_flow.py" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/context-grounding/summarize/check_summarize_flow.py" timeout: 30 expected_exit_code: 0 weight: 5.0 diff --git a/tests/tasks/uipath-maestro-flow/ixp/check_project_selection.py b/tests/tasks/uipath-maestro-flow/ixp/check_project_selection.py new file mode 100644 index 0000000000..6dda89d919 --- /dev/null +++ b/tests/tasks/uipath-maestro-flow/ixp/check_project_selection.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +"""Verify that the dataset row selected its exact published IxP model.""" + +from __future__ import annotations + +import json +import os +import sys + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "_shared")) +from flow_check import find_flow_file # noqa: E402 + + +EXPECTED = { + "aviation": ( + "aviation-investigation-final-report-demo-a42b1d4d-ixp", + "uipath.ixp.aviation-investigation-final-report-demo-a42b1d4d-ixp.", + ), + "birth-certificate": ( + "birth_certificates_oob-6252526a-ixp", + "uipath.ixp.birth-certificates-oob-6252526a-ixp.", + ), +} + + +def _nested_values(value, key: str): + if isinstance(value, dict): + for nested_key, nested_value in value.items(): + if nested_key == key: + yield nested_value + yield from _nested_values(nested_value, key) + elif isinstance(value, list): + for item in value: + yield from _nested_values(item, key) + + +def main() -> int: + row = next((part for part in os.getcwd().split(os.sep) if part in EXPECTED), None) + if row is None: + print(f"FAIL: unknown dataset row in cwd: {os.getcwd()}", file=sys.stderr) + return 2 + + path = find_flow_file() + with open(path, encoding="utf-8") as f: + flow = json.load(f) + + model_name, type_prefix = EXPECTED[row] + for node in flow.get("nodes", []): + has_model = any( + value == model_name for value in _nested_values(node, "modelName") + ) + if str(node.get("type", "")).startswith(type_prefix) and has_model: + print(f"OK: {row} uses published model {model_name} in {path}") + return 0 + + print( + f"FAIL: no IxP node in {path} pairs type prefix {type_prefix!r} " + f"with modelName {model_name!r}", + file=sys.stderr, + ) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/tasks/uipath-maestro-flow/ixp/e2e_02_project_selection.yaml b/tests/tasks/uipath-maestro-flow/ixp/e2e_02_project_selection.yaml index 9fadc20935..f266379c20 100644 --- a/tests/tasks/uipath-maestro-flow/ixp/e2e_02_project_selection.yaml +++ b/tests/tasks/uipath-maestro-flow/ixp/e2e_02_project_selection.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-ixp-e2e-project-selection description: > E2E (project selection): on a tenant where the demo IxP project set is @@ -45,38 +49,28 @@ dataset: initial_prompt: | ${row.prompt} - - Important: - - The `uip` CLI is already available and the runner is logged in. - - Use `--output json` on all uip commands. - - Work inside the stated working directory โ€” do NOT `cd` to a parent. - - Do NOT run `uip maestro flow debug` or `uip maestro flow deploy`. - - Run `uip maestro flow validate` at most 3 times. If it reports errors, fix - the `.flow` and re-run it. Do not exceed 3 validation runs. + Produce a complete flow that validates. success_criteria: - type: skill_triggered skill_name: "uipath-maestro-flow" expected_skill: "uipath-maestro-flow" - description: "Agent invoked the uipath-maestro-flow Skill" + description: "Advisory: agent invoked the uipath-maestro-flow skill" weight: 2.0 - pass_threshold: 1.0 + pass_threshold: 0.0 - type: command_executed - description: "Agent searched the registry for IxP node types" + description: "Advisory: live-v1 agent searched the registry for IxP node types" tool_name: "Bash" command_pattern: 'uip\s+maestro\s+flow\s+registry\s+(search|list|get)\b[^\n]*ixp' min_count: 1 weight: 2.0 - pass_threshold: 1.0 + pass_threshold: 0.0 - # `../..` cwd rationale: dataset-fanout cwd - # is /artifacts//[/]/, so artifacts written - # by the agent live two levels up at most. - type: run_command description: "uip maestro flow validate passes on the generated flow" - command: 'f=$(find ../.. -name "*.flow" -print -quit); [ -n "$f" ] && uip maestro flow validate "$f" --output json' - timeout: 60 + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py" + timeout: 30 expected_exit_code: 0 weight: 3.0 pass_threshold: 1.0 @@ -85,16 +79,14 @@ success_criteria: # a domain-matching IxP project IS published. - type: run_command description: "Generated flow contains a real uipath.ixp.* node" - command: 'grep -rqE "\"type\"[[:space:]]*:[[:space:]]*\"uipath\.ixp\." --include="*.flow" ../..' + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py --regex '\"type\"\\s*:\\s*\"uipath\\.ixp\\.'" expected_exit_code: 0 weight: 3.0 pass_threshold: 1.0 - # Inverted twin of the above. `! grep -q` exits 0 when no mock is found - # (PASS) and 1 when one exists (FAIL). Same `../..` rationale as above. - type: run_command description: "Generated flow does NOT contain a core.logic.mock fallback" - command: '! grep -rqE "\"type\"[[:space:]]*:[[:space:]]*\"core\.logic\.mock\"" --include="*.flow" ../..' + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py --absent-regex '\"type\"\\s*:\\s*\"core\\.logic\\.mock\"'" expected_exit_code: 0 weight: 2.0 pass_threshold: 1.0 @@ -107,36 +99,8 @@ success_criteria: # dataset IDs in scripts/ixp-demo-projects/datasets.jsonl. - type: run_command description: "IxP node references the exact published model for this row's domain (strict)" - command: | - case "$(pwd)" in - *aviation*) - model='aviation-investigation-final-report-demo-a42b1d4d-ixp' - nt='uipath\.ixp\.aviation-investigation-final-report-demo-a42b1d4d-ixp\.' - ;; - *birth*) - model='birth_certificates_oob-6252526a-ixp' - nt='uipath\.ixp\.birth-certificates-oob-6252526a-ixp\.' - ;; - *) - echo "Unknown row in pwd: $(pwd)" >&2 - exit 2 - ;; - esac - f=$(find ../.. -name "*.flow" -print -quit) - if [ -z "$f" ]; then - echo "no .flow file under ../.." >&2 - exit 1 - fi - if ! grep -qF "\"modelName\": \"${model}\"" "$f"; then - echo "modelName mismatch โ€” expected '${model}' in $f" >&2 - grep -E '"modelName"' "$f" >&2 || true - exit 1 - fi - if ! grep -qE "\"type\"[[:space:]]*:[[:space:]]*\"${nt}" "$f"; then - echo "NodeType mismatch โ€” expected prefix '${nt}' in $f" >&2 - grep -E '"type"[[:space:]]*:[[:space:]]*"uipath\.ixp' "$f" >&2 || true - exit 1 - fi + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/ixp/check_project_selection.py" + timeout: 30 expected_exit_code: 0 weight: 5.0 pass_threshold: 1.0 diff --git a/tests/tasks/uipath-maestro-flow/ixp/integration_handle_routing.yaml b/tests/tasks/uipath-maestro-flow/ixp/integration_handle_routing.yaml index b1ba3319f0..d107dbdd3a 100644 --- a/tests/tasks/uipath-maestro-flow/ixp/integration_handle_routing.yaml +++ b/tests/tasks/uipath-maestro-flow/ixp/integration_handle_routing.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-ixp-integration-handle-routing description: > Integration: IxP extraction wired into a Decision node that routes on a @@ -37,29 +41,23 @@ initial_prompt: | `$vars..output.ExtractionResult.ResultsDocument.Fields.find(f => f.FieldName === '')?.Values?.[0]` โ€” per references/plugins/ixp/impl.md ("Accessing Output"). - Important: - - The `uip` CLI is already available. - - Use `--output json` on all uip commands. - - Work inside the stated working directory โ€” do NOT `cd` to a parent. - - Do NOT run `uip maestro flow debug` or `uip maestro flow deploy`. - - Run `uip maestro flow validate` at most 3 times. If it reports errors, fix - the `.flow` and re-run it. Do not exceed 3 validation runs. + Validate the completed flow. success_criteria: - type: skill_triggered skill_name: "uipath-maestro-flow" expected_skill: "uipath-maestro-flow" - description: "Agent invoked the uipath-maestro-flow Skill" + description: "Advisory: agent invoked the uipath-maestro-flow skill" weight: 2.0 - pass_threshold: 1.0 + pass_threshold: 0.0 - type: command_executed - description: "Agent searched the registry for IxP node types" + description: "Advisory: live-v1 agent searched the registry for IxP node types" tool_name: "Bash" command_pattern: 'uip\s+maestro\s+flow\s+registry\s+(search|list|get)\b[^\n]*ixp' min_count: 1 weight: 2.0 - pass_threshold: 1.0 + pass_threshold: 0.0 - type: run_command description: "uip maestro flow validate passes on InvoiceRouter.flow" @@ -71,58 +69,37 @@ success_criteria: - type: run_command description: "Generated flow contains an IxP extraction node or core.logic.mock fallback" - command: 'grep -rqE "\"type\"[[:space:]]*:[[:space:]]*\"(uipath\.ixp|core\.logic\.mock)" --include="*.flow" .' + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py --regex '\"type\"\\s*:\\s*\"(uipath\\.ixp|core\\.logic\\.mock)'" expected_exit_code: 0 weight: 2.0 pass_threshold: 1.0 - # The four checks below locate the .flow rather than hardcoding - # `//.flow`. `uip maestro flow init ` run outside a - # solution auto-scaffolds a `Solution/` wrapper (see greenfield.md), - # so the literal path only resolves when the agent happens to run - # `uip solution init` first โ€” an authoring idiom these checks don't test. - # Each grep line is one `includes:` needle; all must match. + # The checks below use shared discovery rather than hardcoding a solution + # wrapper shape. Each set of strings must match within one Flow artifact. - type: run_command description: "Flow contains a Decision node (core.logic.decision)" - command: | - set -e - flow=$(find . -name '*.flow' -print -quit) - grep -qF 'core.logic.decision' "$flow" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py 'core.logic.decision'" expected_exit_code: 0 weight: 2.0 pass_threshold: 1.0 - type: run_command description: "Decision condition references the canonical IxP output path ($vars..output.ExtractionResult.ResultsDocument.Fields)" - command: | - set -e - flow=$(find . -name '*.flow' -print -quit) - grep -qF '$vars.' "$flow" - grep -qF 'ExtractionResult' "$flow" - grep -qF 'ResultsDocument' "$flow" - grep -qF 'Fields' "$flow" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py '$vars.' 'ExtractionResult' 'ResultsDocument' 'Fields'" expected_exit_code: 0 weight: 3.0 pass_threshold: 1.0 - type: run_command description: "Both branch script nodes are present by name" - command: | - set -e - flow=$(find . -name '*.flow' -print -quit) - grep -qF 'HighValueReview' "$flow" - grep -qF 'AutoApprove' "$flow" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py 'HighValueReview' 'AutoApprove'" expected_exit_code: 0 weight: 2.5 pass_threshold: 1.0 - type: run_command description: "Both Decision branches wired (true and false handles)" - command: | - set -e - flow=$(find . -name '*.flow' -print -quit) - grep -qF '"true"' "$flow" - grep -qF '"false"' "$flow" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py '\"true\"' '\"false\"'" expected_exit_code: 0 weight: 2.0 pass_threshold: 1.0 diff --git a/tests/tasks/uipath-maestro-flow/ixp/routing.yaml b/tests/tasks/uipath-maestro-flow/ixp/routing.yaml index 298103bd3c..46b7bd6e91 100644 --- a/tests/tasks/uipath-maestro-flow/ixp/routing.yaml +++ b/tests/tasks/uipath-maestro-flow/ixp/routing.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-ixp-routing description: > IxP routing (positive) โ€” document-extraction build prompts fanned out via @@ -5,9 +9,9 @@ description: > identical across all rows. Scope: did the agent route to the IxP plugin, not did the flow work. - Assertions verify the agent ran the right CLI-command patterns (registry - search for uipath.ixp.*) and landed a uipath.ixp.* or core.logic.mock node โ€” - NOT whether the resulting flow is correct or would execute. + The gating assertion verifies that the artifact contains a uipath.ixp.* or + core.logic.mock node. Registry-search telemetry remains report-only because + the two authoring loops use different commands. Failure here is signal about SKILL.md or the IxP Plugin Index entry โ€” do NOT relax assertions; feed findings back in. @@ -50,15 +54,6 @@ dataset: initial_prompt: | ${row.prompt} - Important: - - The `uip` CLI is already available. - - Use `--output json` on all uip commands. - - Work inside the stated working directory โ€” do NOT `cd` to a parent. - - Do NOT run `uip maestro flow debug` or `uip maestro flow deploy`. - - Run `uip maestro flow validate` at most ONCE. If it fails, stop. - - Do NOT iterate on validation errors or rewrite the flow to fix them. A broken flow is acceptable. - - You do not need a working flow โ€” a skeleton whose nodes cover the steps of the request is enough. Leave a node unconfigured rather than leaving a step out. - success_criteria: # What this checks: did the agent search the registry for IxP nodes? # Passes (case-insensitive) when the agent ran either: @@ -68,23 +63,26 @@ success_criteria: # routing_listing.yaml drops `get` (read-only). Keep in sync with # scaffold_minimal.yaml, scaffold_multinode.yaml, routing_listing.yaml. - type: command_executed - description: "Agent searched the registry for IxP node types" + description: "Advisory: live-v1 agent searched the registry for IxP node types" tool_name: "Bash" command_pattern: '(?i)uip\s+maestro\s+flow\s+registry\s+(list\b|(search|list|get)\b[^\n]*\b(ixp|document|extract))' min_count: 1 - suite_thresholds: {mean: 0.85} - # Armed latch-only: keeps this criterion gated on sentinel-cut runs (those - # gate on ARMED criteria only). Fail-stop is unreachable for a min_count. + pass_threshold: 0.0 + # Armed latch-only (kept from main #2881): the sentinel below defers its + # pass-stop until this criterion has live-passed, so a live-v1 run is cut + # only after it has searched the registry AND written the node. Fail-stop is + # unreachable for a min_count. Advisory (pass_threshold 0.0) under the + # same-ground regime: the SDK loop never runs this command, so it must not + # gate โ€” on an SDK-loop run the sentinel simply never fires and the run + # completes and grades exactly as before. stop_early: {} - # Greps the actual .flow artifact rather than the command stream โ€” the - # command_executed checker truncates tool params at 2000 chars, which hides - # `"type": "core.logic.mock"` when the agent writes a large flow file in a - # single Write call. Per references/plugins/ixp/impl.md, the agent must land - # either an `uipath.ixp.*` node or a `core.logic.mock` placeholder. + # Inspect the actual .flow artifact rather than the command stream. Per + # references/plugins/ixp/impl.md, the agent must land either an + # `uipath.ixp.*` node or a `core.logic.mock` placeholder. - type: run_command description: "Agent added an IxP or core.logic.mock placeholder node to a .flow file (per references/plugins/ixp/impl.md)" - command: 'grep -rqE "\"type\"[[:space:]]*:[[:space:]]*\"(uipath\.ixp|core\.logic\.mock)" --include="*.flow" .' + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py --regex '\"type\"\\s*:\\s*\"(uipath\\.ixp|core\\.logic\\.mock)'" expected_exit_code: 0 suite_thresholds: {mean: 0.85} diff --git a/tests/tasks/uipath-maestro-flow/ixp/routing_listing.yaml b/tests/tasks/uipath-maestro-flow/ixp/routing_listing.yaml index 740e681d78..12fd2c29c6 100644 --- a/tests/tasks/uipath-maestro-flow/ixp/routing_listing.yaml +++ b/tests/tasks/uipath-maestro-flow/ixp/routing_listing.yaml @@ -1,12 +1,16 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-ixp-routing-listing description: > IxP listing routing โ€” user asks what IxP models / runtime projects are available from a Maestro flow. Read-only Q&A, not a build task. - Asserts the agent ran the IxP registry-search listing command (per - references/plugins/ixp/impl.md โ€” Listing Published Models). Also asserts the - agent did NOT take the build path: no .flow file should be created for a - listing question. + Observes the live-v1 registry-search command as report-only telemetry (per + references/plugins/ixp/impl.md โ€” Listing Published Models). The gating check + asserts that the agent did NOT take the build path: no .flow file should be + created for a listing question. Failure here is signal about the IxP plugin's Listing Published Models / Listing Available Models sections โ€” do NOT relax assertions, tighten the @@ -39,26 +43,21 @@ dataset: initial_prompt: | ${row.prompt} - - Important: - - The `uip` CLI is already available. - - Use `--output json` on all uip commands. + Answer the question without creating a flow. success_criteria: # IxP registry-search check (omits `get`) โ€” see routing.yaml for the pattern breakdown. - type: command_executed - description: "Agent searched the registry for IxP node types (listing path)" + description: "Advisory: live-v1 agent searched the registry for IxP node types" tool_name: "Bash" command_pattern: '(?i)uip\s+maestro\s+flow\s+registry\s+(list\b|(search|list)\b[^\n]*\b(ixp|document|extract))' min_count: 1 - suite_thresholds: {mean: 0.85} + pass_threshold: 0.0 - # Inverted check: a listing/Q&A task should NOT create a .flow file. - # `! find ... | grep -q .` exits 0 when no .flow file is found (PASS) - # and 1 when one exists (FAIL). + # A listing/Q&A task should not create a Flow artifact. - type: run_command description: "No .flow file was created (listing is read-only Q&A, not a build task)" - command: '! find . -name "*.flow" -type f | grep -q .' + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py --expect-none" expected_exit_code: 0 suite_thresholds: {mean: 0.85} diff --git a/tests/tasks/uipath-maestro-flow/ixp/routing_negative.yaml b/tests/tasks/uipath-maestro-flow/ixp/routing_negative.yaml index e1574e1ac0..5c967d0230 100644 --- a/tests/tasks/uipath-maestro-flow/ixp/routing_negative.yaml +++ b/tests/tasks/uipath-maestro-flow/ixp/routing_negative.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-ixp-routing-negative description: > Negative IxP routing โ€” verifies the agent does NOT add a uipath.ixp.* @@ -55,23 +59,11 @@ dataset: initial_prompt: | ${row.prompt} - Important: - - The `uip` CLI is already available. - - Use `--output json` on all uip commands. - - Work inside the stated working directory โ€” do NOT `cd` to a parent. - - Do NOT run `uip maestro flow debug` or `uip maestro flow deploy`. - - Run `uip maestro flow validate` at most ONCE. If it fails, stop. - - Do NOT iterate on validation errors or rewrite the flow to fix them. A broken flow is acceptable. - - You do not need a working flow โ€” a skeleton whose nodes cover the steps of the request is enough. Leave a node unconfigured rather than leaving a step out. - success_criteria: - # Inverted twin of the positive routing.yaml grep. - # `shell=True` in coder_eval Sandbox.run_command means `!` works as a bash - # negator: grep -q exits 0 on match โ†’ ! flips to 1 (FAIL); exits 1 on - # no-match โ†’ ! flips to 0 (PASS). + # Inverted twin of the positive routing.yaml artifact check. - type: run_command description: "No uipath.ixp.* node landed in any .flow file (negative IxP routing)" - command: '! grep -rqE "\"type\"[[:space:]]*:[[:space:]]*\"uipath\.ixp" --include="*.flow" .' + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py --absent-regex '\"type\"\\s*:\\s*\"uipath\\.ixp'" expected_exit_code: 0 suite_thresholds: {mean: 0.85} diff --git a/tests/tasks/uipath-maestro-flow/ixp/scaffold_minimal.yaml b/tests/tasks/uipath-maestro-flow/ixp/scaffold_minimal.yaml index 6b55ef6ab0..b08463036e 100644 --- a/tests/tasks/uipath-maestro-flow/ixp/scaffold_minimal.yaml +++ b/tests/tasks/uipath-maestro-flow/ixp/scaffold_minimal.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-ixp-scaffold-minimal description: > Integration: minimal scaffold โ€” manual trigger โ†’ IxP extract โ†’ script (logs "ok") @@ -25,38 +29,32 @@ initial_prompt: | Script node: contains a JavaScript that logs OK when it receives the extraction result. - Important: - - The `uip` CLI is already available. - - Use `--output json` on all uip commands. - - Work inside the stated working directory โ€” do NOT `cd` to a parent. - - Do NOT run `uip maestro flow debug` or `uip maestro flow deploy`. - - Run `uip maestro flow validate` at most 3 times. If it reports errors, fix - the `.flow` and re-run it. Do not exceed 3 validation runs. + Validate the completed flow. success_criteria: - type: skill_triggered skill_name: "uipath-maestro-flow" expected_skill: "uipath-maestro-flow" - description: "Agent invoked the uipath-maestro-flow Skill" + description: "Advisory: agent invoked the uipath-maestro-flow skill" weight: 2.0 - pass_threshold: 1.0 + pass_threshold: 0.0 # IxP registry-search check โ€” see routing.yaml for the pattern breakdown. - type: command_executed - description: "Agent searched the registry for IxP node types" + description: "Advisory: live-v1 agent searched the registry for IxP node types" tool_name: "Bash" command_pattern: '(?i)uip\s+maestro\s+flow\s+registry\s+(list\b|(search|list|get)\b[^\n]*\b(ixp|document|extract))' min_count: 1 weight: 2.0 - pass_threshold: 1.0 + pass_threshold: 0.0 - type: command_executed - description: "Agent validated the .flow file" + description: "Advisory: live-v1 agent invoked flow validation" tool_name: "Bash" command_pattern: 'uip\s+(maestro\s+)?flow\s+validate' min_count: 1 weight: 1.5 - pass_threshold: 1.0 + pass_threshold: 0.0 - type: run_command description: "uip maestro flow validate passes on Smoke.flow" @@ -68,7 +66,7 @@ success_criteria: - type: run_command description: "Generated flow contains an IxP extraction node or core.logic.mock fallback (per references/plugins/ixp/impl.md)" - command: 'grep -rqE "\"type\"[[:space:]]*:[[:space:]]*\"(uipath\.ixp|core\.logic\.mock)" --include="*.flow" .' + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py --regex '\"type\"\\s*:\\s*\"(uipath\\.ixp|core\\.logic\\.mock)'" expected_exit_code: 0 weight: 2.0 pass_threshold: 1.0 @@ -77,10 +75,7 @@ success_criteria: # integration_handle_routing.yaml for the `Solution/` wrapper rationale. - type: run_command description: "Script node consumes the extraction output via $vars expression" - command: | - set -e - flow=$(find . -name '*.flow' -print -quit) - grep -qF '$vars.' "$flow" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py '$vars.'" expected_exit_code: 0 weight: 1.5 pass_threshold: 1.0 diff --git a/tests/tasks/uipath-maestro-flow/ixp/scaffold_multinode.yaml b/tests/tasks/uipath-maestro-flow/ixp/scaffold_multinode.yaml index 9026972964..4b1337770a 100644 --- a/tests/tasks/uipath-maestro-flow/ixp/scaffold_multinode.yaml +++ b/tests/tasks/uipath-maestro-flow/ixp/scaffold_multinode.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-ixp-scaffold-multinode description: > Integration: multi-script fan-out from an IxP extraction โ€” manual trigger โ†’ @@ -29,38 +33,32 @@ initial_prompt: | the three script nodes logs OK when it receives its respective extraction result. - Important: - - The `uip` CLI is already available. - - Use `--output json` on all uip commands. - - Work inside the stated working directory โ€” do NOT `cd` to a parent. - - Do NOT run `uip maestro flow debug` or `uip maestro flow deploy`. - - Run `uip maestro flow validate` at most 3 times. If it reports errors, fix - the `.flow` and re-run it. Do not exceed 3 validation runs. + Validate the completed flow. success_criteria: - type: skill_triggered skill_name: "uipath-maestro-flow" expected_skill: "uipath-maestro-flow" - description: "Agent invoked the uipath-maestro-flow Skill" + description: "Advisory: agent invoked the uipath-maestro-flow skill" weight: 2.0 - pass_threshold: 1.0 + pass_threshold: 0.0 # IxP registry-search check โ€” see routing.yaml for the pattern breakdown. - type: command_executed - description: "Agent searched the registry for IxP node types" + description: "Advisory: live-v1 agent searched the registry for IxP node types" tool_name: "Bash" command_pattern: '(?i)uip\s+maestro\s+flow\s+registry\s+(list\b|(search|list|get)\b[^\n]*\b(ixp|document|extract))' min_count: 1 weight: 2.0 - pass_threshold: 1.0 + pass_threshold: 0.0 - type: command_executed - description: "Agent validated the .flow file" + description: "Advisory: live-v1 agent invoked flow validation" tool_name: "Bash" command_pattern: 'uip\s+(maestro\s+)?flow\s+validate' min_count: 1 weight: 1.5 - pass_threshold: 1.0 + pass_threshold: 0.0 - type: run_command description: "uip maestro flow validate passes on ReceiptIntake.flow" @@ -72,7 +70,7 @@ success_criteria: - type: run_command description: "Generated flow contains an IxP extraction node or core.logic.mock fallback (per references/plugins/ixp/impl.md)" - command: 'grep -rqE "\"type\"[[:space:]]*:[[:space:]]*\"(uipath\.ixp|core\.logic\.mock)" --include="*.flow" .' + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py --regex '\"type\"\\s*:\\s*\"(uipath\\.ixp|core\\.logic\\.mock)'" expected_exit_code: 0 weight: 2.0 pass_threshold: 1.0 From 832ce2991ca287e5c774e9c360d8205967958bef Mon Sep 17 00:00:00 2001 From: Tao Ma <51425734+tmatup@users.noreply.github.com> Date: Fri, 21 Aug 2026 18:11:33 -0700 Subject: [PATCH 06/52] test(maestro-flow): align connector trigger, e2e, edit, and evaluation tasks (#2752) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Apply the ratified same-ground recipe to the exact Batch 5 roster while preserving task intent and criterion weights. ๐Ÿค– Generated with Claude Code Co-Authored-By: [Claude](mailto:noreply@anthropic.com) --- .../_shared/test_batch5_alignment.py | 330 ++++++++++++++++++ .../_shared/test_same_ground_corpus.py | 30 +- .../check_webhook_waitfor_parallel.py | 19 +- .../trigger_with_filter.yaml | 12 +- .../webhook_waitfor_parallel.yaml | 39 ++- .../e2e/devcon_expense_approval.yaml | 14 +- .../escalation_jira_ticket.yaml | 3 +- .../check_jira_create_issue.py | 17 +- .../jira_create_issue/jira_create_issue.yaml | 19 +- .../jira_get_issue/check_jira_get_issue.py | 12 +- .../e2e/jira_get_issue/jira_get_issue.yaml | 19 +- .../jira_lifecycle/check_jira_lifecycle.py | 11 +- .../e2e/jira_lifecycle/jira_lifecycle.yaml | 19 +- .../check_jira_search_triage.py | 12 +- .../jira_search_triage.yaml | 19 +- .../edit/add_node/add_node.yaml | 8 +- .../edit/add_output/add_output.yaml | 8 +- .../group_to_subflow/group_to_subflow.yaml | 8 +- .../edit/move_node/move_node.yaml | 8 +- .../edit/remove_node/remove_node.yaml | 8 +- .../edit/update_node/update_node.yaml | 8 +- .../evaluate/check_local_crud.py | 97 +++-- .../evaluate/check_no_auto_upload.py | 86 ++++- .../evaluate/check_type_choice.py | 55 ++- .../evaluate/evaluator_type_choice.yaml | 88 ++--- .../check_inline_agent_eval.py | 144 ++++++-- .../inline_agent_eval/inline_agent_eval.yaml | 129 ++++--- .../evaluate/local_crud.yaml | 94 +++-- .../evaluate/no_auto_upload.yaml | 83 +++-- .../simulation/check_simulation_crud.py | 131 +++++-- .../evaluate/simulation/simulation_crud.yaml | 136 ++++---- 31 files changed, 1143 insertions(+), 523 deletions(-) create mode 100644 tests/tasks/uipath-maestro-flow/_shared/test_batch5_alignment.py diff --git a/tests/tasks/uipath-maestro-flow/_shared/test_batch5_alignment.py b/tests/tasks/uipath-maestro-flow/_shared/test_batch5_alignment.py new file mode 100644 index 0000000000..8fdc16d1fd --- /dev/null +++ b/tests/tasks/uipath-maestro-flow/_shared/test_batch5_alignment.py @@ -0,0 +1,330 @@ +"""Regression coverage for Batch 5 same-ground artifact checks.""" + +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path + +import pytest + +TASK_ROOT = Path(__file__).resolve().parent.parent + + +def _write_json(path: Path, value: object) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(value), encoding="utf-8") + + +def _run(relative: str, cwd: Path, *args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, str(TASK_ROOT / relative), *args], + cwd=cwd, + capture_output=True, + text=True, + check=False, + ) + + +@pytest.mark.parametrize("goal", ["goal-a", "goal-b", "goal-c"]) +def test_type_choice_artifact_modes(tmp_path: Path, goal: str) -> None: + expected = { + "goal-a": ( + "goal-a-evaluator", + "uipath-llm-judge-output-semantic-similarity", + ), + "goal-b": ("goal-b-evaluator", "uipath-json-similarity"), + "goal-c": ("goal-c-evaluator", "uipath-contains"), + } + name, type_id = expected[goal] + _write_json( + tmp_path / "Wrapper" / "Flow" / "evals" / f"{goal}.json", + {"name": name, "evaluatorTypeId": type_id}, + ) + + result = _run("evaluate/check_type_choice.py", tmp_path, "--artifact", goal) + + assert result.returncode == 0, result.stderr or result.stdout + + +def test_type_choice_default_checks_report(tmp_path: Path) -> None: + _write_json( + tmp_path / "report.json", + { + "goal_a_type": "llm-judge-output", + "goal_b_type": "json-similarity", + "goal_c_type": "contains", + }, + ) + + result = _run("evaluate/check_type_choice.py", tmp_path) + + assert result.returncode == 0, result.stderr or result.stdout + + +@pytest.fixture +def local_crud_sandbox(tmp_path: Path) -> Path: + root = tmp_path / "arbitrary-wrapper" / "SmokeEval" / "evals" + _write_json( + root / "evaluators" / "greeting.json", + {"name": "greeting-match", "evaluatorTypeId": "uipath-exact-match"}, + ) + _write_json( + root / "eval-sets" / "smoke.json", + { + "name": "Smoke Set", + "evaluations": [ + { + "name": "hello", + "inputs": {"name": "Alice"}, + "expectedOutput": {"greeting": "Hello, Alice!"}, + } + ], + }, + ) + return tmp_path + + +@pytest.mark.parametrize("check", ["evaluator", "eval-set", "data-point"]) +def test_local_crud_modes(local_crud_sandbox: Path, check: str) -> None: + result = _run("evaluate/check_local_crud.py", local_crud_sandbox, "--check", check) + assert result.returncode == 0, result.stderr or result.stdout + + +def test_local_crud_default_checks_combined_artifacts(local_crud_sandbox: Path) -> None: + result = _run("evaluate/check_local_crud.py", local_crud_sandbox) + assert result.returncode == 0, result.stderr or result.stdout + + +@pytest.fixture +def no_upload_sandbox(tmp_path: Path) -> Path: + root = tmp_path / "LocalOnly" / "evals" + _write_json( + root / "evaluators" / "greeting.json", + {"name": "greeting-match", "evaluatorTypeId": "uipath-exact-match"}, + ) + _write_json( + root / "eval-sets" / "smoke.json", + { + "name": "Smoke", + "evaluations": [ + { + "name": "hello", + "inputs": {"name": "Alice"}, + "expectedOutput": {"greeting": "Hello, Alice!"}, + } + ], + }, + ) + _write_json( + tmp_path / "report.json", + { + "ran_solution_upload": False, + "ran_eval_run_start": False, + "action": "refused", + "reason": "Studio Web upload requires user authorization.", + }, + ) + return tmp_path + + +@pytest.mark.parametrize("check", ["evaluator", "eval-set", "data-point"]) +def test_no_auto_upload_artifact_modes(no_upload_sandbox: Path, check: str) -> None: + result = _run( + "evaluate/check_no_auto_upload.py", + no_upload_sandbox, + "--check", + check, + ) + assert result.returncode == 0, result.stderr or result.stdout + + +def test_no_auto_upload_default_still_checks_report(no_upload_sandbox: Path) -> None: + result = _run("evaluate/check_no_auto_upload.py", no_upload_sandbox) + assert result.returncode == 0, result.stderr or result.stdout + + +@pytest.fixture +def inline_eval_sandbox(tmp_path: Path) -> Path: + root = tmp_path / "AnySolution" / "TriageEval" + _write_json(root / "inline-id" / "agent.json", {"name": "triage"}) + _write_json( + root / "evals" / "evaluators" / "triage-judge-a1.json", + { + "name": "triage-judge", + "evaluatorTypeId": "uipath-llm-judge-output-semantic-similarity", + "evaluatorConfig": {"model": "gpt-4.1-2025-04-14"}, + }, + ) + _write_json( + root / "evals" / "eval-sets" / "triage-cases.json", + { + "name": "Triage Cases", + "evaluatorRefs": ["triage-judge-a1.json"], + "evaluations": [ + { + "name": "password-reset", + "inputs": {"email": "reset"}, + "expectedOutput": {"category": "Account Access"}, + } + ], + }, + ) + return tmp_path + + +@pytest.mark.parametrize( + "check", + [ + "inline-agent", + "evaluator", + "no-deterministic", + "eval-set", + "evaluator-refs", + "data-point", + ], +) +def test_inline_eval_modes(inline_eval_sandbox: Path, check: str) -> None: + result = _run( + "evaluate/inline_agent_eval/check_inline_agent_eval.py", + inline_eval_sandbox, + "--check", + check, + ) + assert result.returncode == 0, result.stderr or result.stdout + + +def test_inline_eval_default_checks_combined_artifacts( + inline_eval_sandbox: Path, +) -> None: + result = _run( + "evaluate/inline_agent_eval/check_inline_agent_eval.py", + inline_eval_sandbox, + ) + assert result.returncode == 0, result.stderr or result.stdout + + +def test_inline_eval_rejects_deterministic_evaluator(inline_eval_sandbox: Path) -> None: + _write_json( + inline_eval_sandbox / "deterministic.json", + {"name": "wrong", "evaluatorTypeId": "uipath-exact-match"}, + ) + + result = _run( + "evaluate/inline_agent_eval/check_inline_agent_eval.py", + inline_eval_sandbox, + "--check", + "no-deterministic", + ) + + assert result.returncode == 1 + + +@pytest.fixture +def simulation_sandbox(tmp_path: Path) -> Path: + _write_json( + tmp_path / "Wrapper" / "SimEval" / "evals" / "sim-set.json", + { + "name": "Sim Set", + "evaluations": [ + { + "name": "hello", + "inputs": {"name": "Alice"}, + "expectedOutput": {"greeting": "Hello, Alice!"}, + "simulations": [ + { + "componentId": "agent-lookup", + "simulationStrategy": "Llm", + "outputSchema": { + "type": "object", + "properties": {"result": {"type": "string"}}, + }, + } + ], + } + ], + }, + ) + return tmp_path + + +@pytest.mark.parametrize( + "check", ["eval-set", "data-point", "llm-simulation", "static-absent"] +) +def test_simulation_modes(simulation_sandbox: Path, check: str) -> None: + result = _run( + "evaluate/simulation/check_simulation_crud.py", + simulation_sandbox, + "--check", + check, + ) + assert result.returncode == 0, result.stderr or result.stdout + + +def test_simulation_default_checks_combined_artifacts(simulation_sandbox: Path) -> None: + result = _run( + "evaluate/simulation/check_simulation_crud.py", + simulation_sandbox, + ) + assert result.returncode == 0, result.stderr or result.stdout + + +def test_simulation_static_absent_rejects_retained_simulation( + simulation_sandbox: Path, +) -> None: + set_path = next(simulation_sandbox.rglob("sim-set.json")) + doc = json.loads(set_path.read_text(encoding="utf-8")) + doc["evaluations"][0]["simulations"].append( + {"componentId": "connector-send-email", "simulationStrategy": "Static"} + ) + _write_json(set_path, doc) + + result = _run( + "evaluate/simulation/check_simulation_crud.py", + simulation_sandbox, + "--check", + "static-absent", + ) + + assert result.returncode == 1 + + +def test_webhook_checker_accepts_root_sdk_emit(tmp_path: Path) -> None: + _write_json( + tmp_path / "WebhookSelfTest.flow", + { + "nodes": [ + {"id": "start", "type": "core.trigger.manual"}, + { + "id": "wait", + "type": "uipath.connector.event.uipath-http-webhook.http-webhook", + }, + { + "id": "get", + "type": "core.action.http.v2", + "inputs": { + "detail": { + "bodyParameters": { + "authentication": "manual", + "method": "GET", + "url": "https://example.test/webhook", + } + } + }, + }, + {"id": "end-wait", "type": "core.control.end"}, + {"id": "end-get", "type": "core.control.end"}, + ], + "edges": [ + {"sourceNodeId": "start", "targetNodeId": "wait"}, + {"sourceNodeId": "start", "targetNodeId": "get"}, + {"sourceNodeId": "wait", "targetNodeId": "end-wait"}, + {"sourceNodeId": "get", "targetNodeId": "end-get"}, + ], + }, + ) + + result = _run("connector_trigger/check_webhook_waitfor_parallel.py", tmp_path) + + assert result.returncode == 0, result.stderr or result.stdout diff --git a/tests/tasks/uipath-maestro-flow/_shared/test_same_ground_corpus.py b/tests/tasks/uipath-maestro-flow/_shared/test_same_ground_corpus.py index a769c76eae..216dfe4f5c 100644 --- a/tests/tasks/uipath-maestro-flow/_shared/test_same_ground_corpus.py +++ b/tests/tasks/uipath-maestro-flow/_shared/test_same_ground_corpus.py @@ -12,23 +12,11 @@ FLOW_TASKS = Path(__file__).resolve().parent.parent -V1_AUTHORING_ALLOWLIST = { - "connector_trigger/webhook_waitfor_parallel.yaml", - "e2e/devcon_expense_approval.yaml", - "evaluate/evaluator_type_choice.yaml", - "evaluate/inline_agent_eval/inline_agent_eval.yaml", - "evaluate/local_crud.yaml", - "evaluate/no_auto_upload.yaml", - "evaluate/simulation/simulation_crud.yaml", -} +V1_AUTHORING_ALLOWLIST = set() SKILL_TELEMETRY_ALLOWLIST = set() -FORBIDDEN_PROMPT_ALLOWLIST = { - "evaluate/inline_agent_eval/inline_agent_eval.yaml", - "evaluate/local_crud.yaml", - "evaluate/simulation/simulation_crud.yaml", -} +FORBIDDEN_PROMPT_ALLOWLIST = set() # These born-neutral escalation tasks are outside the recipe-edit sweep. Their # prompts already require the same-name solution and tell the agent to leave @@ -40,19 +28,7 @@ "e2e/escalation_slack_alert/escalation_slack_alert.yaml", } -DEBUG_SOLUTION_ALLOWLIST = { - "e2e/escalation_jira_ticket/escalation_jira_ticket.yaml", - "e2e/jira_create_issue/jira_create_issue.yaml", - "e2e/jira_get_issue/jira_get_issue.yaml", - "e2e/jira_lifecycle/jira_lifecycle.yaml", - "e2e/jira_search_triage/jira_search_triage.yaml", - "edit/add_node/add_node.yaml", - "edit/add_output/add_output.yaml", - "edit/group_to_subflow/group_to_subflow.yaml", - "edit/move_node/move_node.yaml", - "edit/remove_node/remove_node.yaml", - "edit/update_node/update_node.yaml", -} +DEBUG_SOLUTION_ALLOWLIST = set() # F1 freezes the #2557 task even though its prompt predates the exact phrase. DEBUG_PROJECT_LAYOUT_EXCEPTIONS = {"single_node/coded_agent/coded_agent.yaml"} diff --git a/tests/tasks/uipath-maestro-flow/connector_trigger/check_webhook_waitfor_parallel.py b/tests/tasks/uipath-maestro-flow/connector_trigger/check_webhook_waitfor_parallel.py index 1b81347fb7..a17f8c4f45 100644 --- a/tests/tasks/uipath-maestro-flow/connector_trigger/check_webhook_waitfor_parallel.py +++ b/tests/tasks/uipath-maestro-flow/connector_trigger/check_webhook_waitfor_parallel.py @@ -24,10 +24,13 @@ from __future__ import annotations -import glob import json +import os import sys +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "_shared")) +from flow_check import find_flow_file # noqa: E402 + FLOW_GLOB = "**/WebhookSelfTest*.flow" EVENT_MARKERS = ("uipath.connector.event", "uipath-http-webhook") HTTP_TYPE = "core.action.http.v2" @@ -40,10 +43,8 @@ def _fail(msg: str) -> None: def _read_flow() -> dict: - flows = glob.glob(FLOW_GLOB, recursive=True) - if not flows: - _fail(f"no flow file matching {FLOW_GLOB} under cwd") - with open(flows[0], encoding="utf-8") as f: + path = find_flow_file(flow_glob=FLOW_GLOB) + with open(path, encoding="utf-8") as f: return json.load(f) @@ -83,7 +84,8 @@ def main() -> None: # 2. HTTP Webhook Wait-for-event node present. event_nodes = [ - n for n in nodes + n + for n in nodes if all(m in str(n.get("type", "")).lower() for m in EVENT_MARKERS) ] if not event_nodes: @@ -130,7 +132,10 @@ def main() -> None: end_ids = {n.get("id") for n in nodes if str(n.get("type", "")) == END_TYPE} if not end_ids: _fail(f"no End node ({END_TYPE}). Types seen: {types_seen}") - for label, node in (("wait-for-event", event_nodes[0]), ("http-request", good_http)): + for label, node in ( + ("wait-for-event", event_nodes[0]), + ("http-request", good_http), + ): reach = _reachable(node.get("id"), edges) if not (reach & end_ids): _fail(f"{label} branch does not reach an End node") diff --git a/tests/tasks/uipath-maestro-flow/connector_trigger/trigger_with_filter.yaml b/tests/tasks/uipath-maestro-flow/connector_trigger/trigger_with_filter.yaml index 3f23be96b4..61c69e476a 100644 --- a/tests/tasks/uipath-maestro-flow/connector_trigger/trigger_with_filter.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_trigger/trigger_with_filter.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-trigger-with-filter description: > Verifies that the uipath-maestro-flow skill teaches agents to emit a structured `filter` tree. @@ -12,8 +16,8 @@ initial_prompt: | abc@xyz.com. Do NOT run any command that requires a live connection โ€” this sandbox has no UiPath tenant, so you cannot configure the trigger against a live connection. - Instead, write to `trigger_detail.json` the exact JSON object you would pass to - `uip maestro flow node configure --detail ''` to configure that trigger. + Instead, write the exact connector-trigger detail object to `trigger_detail.json`. + It must contain the structured filter tree that configures this trigger. Use placeholders for any connection/folder/event ids you cannot resolve offline. @@ -26,8 +30,8 @@ success_criteria: # wrapped under detail/inputs fails โ€” that is not the --detail object. - type: run_command description: "trigger_detail.json is the --detail object with a top-level `filter` tree on `subject` Contains, no top-level filterExpression" - command: "python3 $TASK_DIR/check_trigger_filter.py" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/connector_trigger/check_trigger_filter.py" timeout: 10 expected_exit_code: 0 weight: 3.0 - pass_threshold: 1.0 \ No newline at end of file + pass_threshold: 1.0 diff --git a/tests/tasks/uipath-maestro-flow/connector_trigger/webhook_waitfor_parallel.yaml b/tests/tasks/uipath-maestro-flow/connector_trigger/webhook_waitfor_parallel.yaml index 2189db7937..fdc25b7ac4 100644 --- a/tests/tasks/uipath-maestro-flow/connector_trigger/webhook_waitfor_parallel.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_trigger/webhook_waitfor_parallel.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-webhook-waitfor-parallel description: > E2E self-testing flow: a manual start trigger fans out into two parallel @@ -8,7 +12,7 @@ description: > query. The GET self-delivers the event that completes the wait at runtime. Exercises the connector-trigger plugin's "Wait for events" variant, parallel branch fan-out from the start trigger, manual-mode HTTP, and the CLI - webhook-URL retrieval path. Validates the static two-branch shape. + webhook-URL wiring. Validates the static two-branch shape. tags: [ uipath-maestro-flow, @@ -46,45 +50,44 @@ initial_prompt: | After building, validate the flow. Do NOT ask for approval, confirmation, or feedback. Do NOT pause between planning and - implementation. Build the complete flow end-to-end in a single pass. - Use `--output json` on `uip` commands. + implementation. Build the complete flow end-to-end in a single pass. success_criteria: - type: command_executed - description: "Agent created a solution" + description: "Solution-creation telemetry (report only; the Flow artifact is graded below)" tool_name: "Bash" command_pattern: '(uip|\$UIP)\s+solution\s+(new|init)' min_count: 1 weight: 1.0 - pass_threshold: 1.0 + pass_threshold: 0.0 - - type: command_executed - description: "Agent initialized a Flow project" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+(maestro\s+)?flow\s+init' - min_count: 1 + - type: run_command + description: "WebhookSelfTest Flow artifact exists" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py --flow-name WebhookSelfTest" + timeout: 30 + expected_exit_code: 0 weight: 1.0 pass_threshold: 1.0 - type: command_executed - description: "Agent retrieved the webhook URL" + description: "Webhook-URL retrieval telemetry (report only; the final artifact grades URL wiring)" tool_name: "Bash" command_pattern: '(uip|\$UIP)\s+is\s+webhooks\s+config' min_count: 1 weight: 1.0 - pass_threshold: 1.0 + pass_threshold: 0.0 - - type: command_executed - description: "Agent validated the flow" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+(maestro\s+)?flow\s+validate' - min_count: 1 + - type: run_command + description: "Flow validates successfully" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py" + timeout: 30 + expected_exit_code: 0 weight: 1.5 pass_threshold: 1.0 - type: run_command description: "Two-branch structure: wait-for-event + manual-GET-to-webhook-URL (no headers/query), both reaching End" - command: "python3 $TASK_DIR/check_webhook_waitfor_parallel.py" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/connector_trigger/check_webhook_waitfor_parallel.py" timeout: 120 expected_exit_code: 0 weight: 3.0 diff --git a/tests/tasks/uipath-maestro-flow/e2e/devcon_expense_approval.yaml b/tests/tasks/uipath-maestro-flow/e2e/devcon_expense_approval.yaml index 8652ca3d38..a43490dfb4 100644 --- a/tests/tasks/uipath-maestro-flow/e2e/devcon_expense_approval.yaml +++ b/tests/tasks/uipath-maestro-flow/e2e/devcon_expense_approval.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-e2e-devcon-expense-approval description: > DevCon end-to-end scenario: developer gives a vague expense approval requirement. @@ -43,12 +47,12 @@ success_criteria: pass_threshold: 0.0 # advisory: init auto-scaffolds the parent solution (CLI #2470) - type: command_executed - description: "Agent initialized a Flow project" + description: "Flow-project initialization telemetry (report only; the artifact is graded below)" tool_name: "Bash" command_pattern: '(uip|\$UIP|\$\{?UIP\}?)\s+(maestro\s+)?flow\s+init' min_count: 1 weight: 1.0 - pass_threshold: 1.0 + pass_threshold: 0.0 - type: run_command description: "Flow file exists (path-agnostic discovery)" @@ -60,19 +64,19 @@ success_criteria: - type: run_command description: "Flow has semantically correct inline HITL schema, decision capture, completed wiring, and .output access paths" - command: "python3 $TASK_DIR/check_devcon_expense_approval.py" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/e2e/check_devcon_expense_approval.py" timeout: 30 expected_exit_code: 0 weight: 16.5 pass_threshold: 1.0 - type: command_executed - description: "Agent validated the flow" + description: "Flow-validation telemetry (report only; successful validation is graded below)" tool_name: "Bash" command_pattern: '(uip|\$UIP|\$\{?UIP\}?)\s+(maestro\s+)?flow\s+validate' min_count: 1 weight: 2.5 - pass_threshold: 1.0 + pass_threshold: 0.0 - type: run_command description: "Flow validates successfully" diff --git a/tests/tasks/uipath-maestro-flow/e2e/escalation_jira_ticket/escalation_jira_ticket.yaml b/tests/tasks/uipath-maestro-flow/e2e/escalation_jira_ticket/escalation_jira_ticket.yaml index c79643e94f..a057e52b5e 100644 --- a/tests/tasks/uipath-maestro-flow/e2e/escalation_jira_ticket/escalation_jira_ticket.yaml +++ b/tests/tasks/uipath-maestro-flow/e2e/escalation_jira_ticket/escalation_jira_ticket.yaml @@ -32,7 +32,8 @@ post_run: timeout: 120 initial_prompt: | - Create a UiPath Maestro Flow named "EscalationJiraTicket" with a MANUAL trigger. + Create a UiPath Maestro Flow named "EscalationJiraTicket" with a MANUAL trigger + inside a solution of the same name. It triages a customer escalation and creates a Jira ticket for it. Read `seed.json` in the current directory. Declare flow inputs matching the keys diff --git a/tests/tasks/uipath-maestro-flow/e2e/jira_create_issue/check_jira_create_issue.py b/tests/tasks/uipath-maestro-flow/e2e/jira_create_issue/check_jira_create_issue.py index d2e359b653..add249b56a 100644 --- a/tests/tasks/uipath-maestro-flow/e2e/jira_create_issue/check_jira_create_issue.py +++ b/tests/tasks/uipath-maestro-flow/e2e/jira_create_issue/check_jira_create_issue.py @@ -16,7 +16,6 @@ from __future__ import annotations -import glob import json import os import re @@ -26,7 +25,12 @@ HERE = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, HERE) # local jira_is sys.path.insert(0, os.path.dirname(os.path.dirname(HERE))) # โ€ฆ/uipath-maestro-flow (for _shared) -from _shared.flow_check import collect_outputs, get_last_debug_raw, run_debug # noqa: E402 +from _shared.flow_check import ( # noqa: E402 + collect_outputs, + find_flow_file, + get_last_debug_raw, + run_debug, +) import jira_is # noqa: E402 JIRA_KEY = "uipath-atlassian-jira" @@ -40,11 +44,10 @@ def _fail(msg: str) -> None: def main() -> None: seed = json.loads(Path("seed.json").read_text()) - flows = glob.glob("**/*.flow", recursive=True) - raw = next((r for p in flows for r in [open(p, encoding="utf-8").read()] - if JIRA_KEY in r and '"nodes"' in r), None) - if raw is None: - _fail(f"no .flow references the {JIRA_KEY} connector (found {flows})") + flow_path = find_flow_file() + raw = Path(flow_path).read_text(encoding="utf-8") + if JIRA_KEY not in raw or '"nodes"' not in raw: + _fail(f"{flow_path} does not reference the {JIRA_KEY} connector") print(f"OK: flow references {JIRA_KEY}") payload = run_debug(timeout=480) diff --git a/tests/tasks/uipath-maestro-flow/e2e/jira_create_issue/jira_create_issue.yaml b/tests/tasks/uipath-maestro-flow/e2e/jira_create_issue/jira_create_issue.yaml index 6e8fe8e040..e5d4a587fc 100644 --- a/tests/tasks/uipath-maestro-flow/e2e/jira_create_issue/jira_create_issue.yaml +++ b/tests/tasks/uipath-maestro-flow/e2e/jira_create_issue/jira_create_issue.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-ipe-jira-create-issue description: > E2E live Jira coverage โ€” builds a Flow with a manual trigger and an @@ -28,7 +32,8 @@ post_run: timeout: 120 initial_prompt: | - Create a new Flow project called "JiraCreateIssue" with a manual trigger. + Create a new Flow project called "JiraCreateIssue" with a manual trigger, + inside a solution of the same name. It should create a Jira issue using the Atlassian Jira "Create Issue" connector activity, and expose the new issue's key as a flow output. @@ -44,17 +49,17 @@ initial_prompt: | Validate the final flow file. success_criteria: - - type: command_executed - description: "uip maestro flow validate was called" - tool_name: "Bash" - command_pattern: "(uip|\\$UIP)\\s+(maestro\\s+)?flow\\s+validate" - min_count: 1 + - type: run_command + description: "Flow validates successfully" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py" + timeout: 30 + expected_exit_code: 0 weight: 2.0 pass_threshold: 1.0 - type: run_command description: "JiraCreateIssue checks: valid flow with Jira Create-Issue node, debug creates a real issue, and the tenant carries the seeded summary" - command: "python3 $TASK_DIR/check_jira_create_issue.py" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/e2e/jira_create_issue/check_jira_create_issue.py" timeout: 1080 expected_exit_code: 0 weight: 5.0 diff --git a/tests/tasks/uipath-maestro-flow/e2e/jira_get_issue/check_jira_get_issue.py b/tests/tasks/uipath-maestro-flow/e2e/jira_get_issue/check_jira_get_issue.py index bed6f91b57..2b0e9e69b9 100644 --- a/tests/tasks/uipath-maestro-flow/e2e/jira_get_issue/check_jira_get_issue.py +++ b/tests/tasks/uipath-maestro-flow/e2e/jira_get_issue/check_jira_get_issue.py @@ -16,7 +16,6 @@ from __future__ import annotations -import glob import json import os import re @@ -25,7 +24,7 @@ HERE = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, os.path.dirname(os.path.dirname(HERE))) # โ€ฆ/uipath-maestro-flow (for _shared) -from _shared.flow_check import assert_outputs_contain, run_debug # noqa: E402 +from _shared.flow_check import assert_outputs_contain, find_flow_file, run_debug # noqa: E402 JIRA_KEY = "uipath-atlassian-jira" GET_OP_RE = re.compile(r"get[\s_-]?issue|curated_get_issue", re.IGNORECASE) @@ -39,11 +38,10 @@ def main() -> None: seed = json.loads(Path("seed.json").read_text()) issue_key = seed["issue_key"] - flows = glob.glob("**/*.flow", recursive=True) - raw = next((r for p in flows for r in [open(p, encoding="utf-8").read()] - if JIRA_KEY in r and '"nodes"' in r), None) - if raw is None: - _fail(f"no .flow references the {JIRA_KEY} connector (found {flows})") + flow_path = find_flow_file() + raw = Path(flow_path).read_text(encoding="utf-8") + if JIRA_KEY not in raw or '"nodes"' not in raw: + _fail(f"{flow_path} does not reference the {JIRA_KEY} connector") print(f"OK: flow references {JIRA_KEY}") if not GET_OP_RE.search(raw): _fail("flow does not reference a Get-Issue operation") diff --git a/tests/tasks/uipath-maestro-flow/e2e/jira_get_issue/jira_get_issue.yaml b/tests/tasks/uipath-maestro-flow/e2e/jira_get_issue/jira_get_issue.yaml index 07774fe9f3..f7158a3c8c 100644 --- a/tests/tasks/uipath-maestro-flow/e2e/jira_get_issue/jira_get_issue.yaml +++ b/tests/tasks/uipath-maestro-flow/e2e/jira_get_issue/jira_get_issue.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-ipe-jira-get-issue description: > E2E live Jira coverage โ€” builds a Flow with a manual trigger and an Atlassian @@ -29,7 +33,8 @@ post_run: timeout: 120 initial_prompt: | - Create a new Flow project called "JiraGetIssue" with a manual trigger. + Create a new Flow project called "JiraGetIssue" with a manual trigger, + inside a solution of the same name. It should read a single Jira issue using the Atlassian Jira "Get Issue" connector activity, and expose the retrieved issue's summary (and status) as flow outputs. @@ -44,17 +49,17 @@ initial_prompt: | Validate the final flow file. success_criteria: - - type: command_executed - description: "uip maestro flow validate was called" - tool_name: "Bash" - command_pattern: "(uip|\\$UIP)\\s+(maestro\\s+)?flow\\s+validate" - min_count: 1 + - type: run_command + description: "Flow validates successfully" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py" + timeout: 30 + expected_exit_code: 0 weight: 2.0 pass_threshold: 1.0 - type: run_command description: "JiraGetIssue checks: valid flow with Jira Get-Issue node referencing the seeded key, debug completes, and outputs carry the seeded summary" - command: "python3 $TASK_DIR/check_jira_get_issue.py" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/e2e/jira_get_issue/check_jira_get_issue.py" timeout: 1080 expected_exit_code: 0 weight: 5.0 diff --git a/tests/tasks/uipath-maestro-flow/e2e/jira_lifecycle/check_jira_lifecycle.py b/tests/tasks/uipath-maestro-flow/e2e/jira_lifecycle/check_jira_lifecycle.py index 6fdabde99c..5a80b07e28 100644 --- a/tests/tasks/uipath-maestro-flow/e2e/jira_lifecycle/check_jira_lifecycle.py +++ b/tests/tasks/uipath-maestro-flow/e2e/jira_lifecycle/check_jira_lifecycle.py @@ -24,7 +24,6 @@ from __future__ import annotations -import glob import json import os import re @@ -37,6 +36,7 @@ from _shared.flow_check import ( # noqa: E402 assert_flow_has_any_node_type, assert_flow_has_node_type, + find_flow_file, get_last_debug_raw, run_debug, ) @@ -70,11 +70,10 @@ def main() -> None: } # 1. STRUCTURAL ---------------------------------------------------------- - flows = glob.glob("**/*.flow", recursive=True) - raw = next((r for p in flows for r in [open(p, encoding="utf-8").read()] - if JIRA_KEY in r and '"nodes"' in r), None) - if raw is None: - _fail(f"no .flow references the {JIRA_KEY} connector (found {flows})") + flow_path = find_flow_file() + raw = Path(flow_path).read_text(encoding="utf-8") + if JIRA_KEY not in raw or '"nodes"' not in raw: + _fail(f"{flow_path} does not reference the {JIRA_KEY} connector") print(f"OK: flow references {JIRA_KEY}") assert_flow_has_node_type(["core.logic.loop"]) diff --git a/tests/tasks/uipath-maestro-flow/e2e/jira_lifecycle/jira_lifecycle.yaml b/tests/tasks/uipath-maestro-flow/e2e/jira_lifecycle/jira_lifecycle.yaml index 18f45296dc..4d1422007a 100644 --- a/tests/tasks/uipath-maestro-flow/e2e/jira_lifecycle/jira_lifecycle.yaml +++ b/tests/tasks/uipath-maestro-flow/e2e/jira_lifecycle/jira_lifecycle.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-ipe-jira-lifecycle description: > E2E live Jira coverage of a composite loop-and-switch flow. The agent builds @@ -33,7 +37,8 @@ post_run: timeout: 180 initial_prompt: | - Create a new Flow project called "JiraLifecycle" with a manual trigger. + Create a new Flow project called "JiraLifecycle" with a manual trigger, + inside a solution of the same name. Read `seed.json` in the current directory. It contains a list of `issues` (each with a `summary` and a `priority`), the `project_key` and `issuetype_id` @@ -57,17 +62,17 @@ initial_prompt: | Validate the final flow file. success_criteria: - - type: command_executed - description: "uip maestro flow validate was called" - tool_name: "Bash" - command_pattern: "(uip|\\$UIP)\\s+(maestro\\s+)?flow\\s+validate" - min_count: 1 + - type: run_command + description: "Flow validates successfully" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py" + timeout: 30 + expected_exit_code: 0 weight: 2.0 pass_threshold: 1.0 - type: run_command description: "JiraLifecycle checks: valid flow with a loop + switch over the Jira connector, debug creates every seeded issue, and each carries the comment its priority branch should have posted" - command: "python3 $TASK_DIR/check_jira_lifecycle.py" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/e2e/jira_lifecycle/check_jira_lifecycle.py" timeout: 1320 expected_exit_code: 0 weight: 5.0 diff --git a/tests/tasks/uipath-maestro-flow/e2e/jira_search_triage/check_jira_search_triage.py b/tests/tasks/uipath-maestro-flow/e2e/jira_search_triage/check_jira_search_triage.py index 597c957519..c24cf1aa8f 100644 --- a/tests/tasks/uipath-maestro-flow/e2e/jira_search_triage/check_jira_search_triage.py +++ b/tests/tasks/uipath-maestro-flow/e2e/jira_search_triage/check_jira_search_triage.py @@ -12,7 +12,6 @@ from __future__ import annotations -import glob import json import os import re @@ -22,7 +21,7 @@ HERE = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, HERE) # local jira_is sys.path.insert(0, os.path.dirname(os.path.dirname(HERE))) # โ€ฆ/uipath-maestro-flow (for _shared) -from _shared.flow_check import assert_flow_has_node_type, run_debug # noqa: E402 +from _shared.flow_check import assert_flow_has_node_type, find_flow_file, run_debug # noqa: E402 import jira_is # noqa: E402 JIRA_KEY = "uipath-atlassian-jira" @@ -36,11 +35,10 @@ def _fail(msg: str) -> None: def main() -> None: seed = json.loads(Path("seed.json").read_text()) - flows = glob.glob("**/*.flow", recursive=True) - raw = next((r for p in flows for r in [open(p, encoding="utf-8").read()] - if JIRA_KEY in r and '"nodes"' in r), None) - if raw is None: - _fail(f"no .flow references the {JIRA_KEY} connector (found {flows})") + flow_path = find_flow_file() + raw = Path(flow_path).read_text(encoding="utf-8") + if JIRA_KEY not in raw or '"nodes"' not in raw: + _fail(f"{flow_path} does not reference the {JIRA_KEY} connector") print(f"OK: flow references {JIRA_KEY}") if not SEARCH_OP_RE.search(raw): _fail("flow does not reference a Search-Issues (JQL) operation") diff --git a/tests/tasks/uipath-maestro-flow/e2e/jira_search_triage/jira_search_triage.yaml b/tests/tasks/uipath-maestro-flow/e2e/jira_search_triage/jira_search_triage.yaml index caa3c05356..c536c51993 100644 --- a/tests/tasks/uipath-maestro-flow/e2e/jira_search_triage/jira_search_triage.yaml +++ b/tests/tasks/uipath-maestro-flow/e2e/jira_search_triage/jira_search_triage.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-ipe-jira-search-triage description: > E2E live Jira coverage of a JQL-search-driven triage flow: a manual-trigger @@ -26,7 +30,8 @@ post_run: timeout: 120 initial_prompt: | - Create a new Flow project called "JiraSearchTriage" with a manual trigger. + Create a new Flow project called "JiraSearchTriage" with a manual trigger, + inside a solution of the same name. Read `seed.json` in the current directory for the `jql` query and the `processed_comment` body. @@ -44,17 +49,17 @@ initial_prompt: | Validate the final flow file. success_criteria: - - type: command_executed - description: "uip maestro flow validate was called" - tool_name: "Bash" - command_pattern: "(uip|\\$UIP)\\s+(maestro\\s+)?flow\\s+validate" - min_count: 1 + - type: run_command + description: "Flow validates successfully" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py" + timeout: 30 + expected_exit_code: 0 weight: 2.0 pass_threshold: 1.0 - type: run_command description: "JiraSearchTriage checks: valid flow with a JQL search feeding a loop of Add-Comment, debug completes, and both seeded issues carry the triage comment" - command: "python3 $TASK_DIR/check_jira_search_triage.py" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/e2e/jira_search_triage/check_jira_search_triage.py" timeout: 1320 expected_exit_code: 0 weight: 5.0 diff --git a/tests/tasks/uipath-maestro-flow/edit/add_node/add_node.yaml b/tests/tasks/uipath-maestro-flow/edit/add_node/add_node.yaml index 96ec38a796..f671195a46 100644 --- a/tests/tasks/uipath-maestro-flow/edit/add_node/add_node.yaml +++ b/tests/tasks/uipath-maestro-flow/edit/add_node/add_node.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-add-node description: > Add a script node to an existing BellevueWeather flow that converts the @@ -17,6 +21,8 @@ sandbox: initial_prompt: | Add a script node called "convertToCelsius" between getWeather and formatSummary that converts the temp from F to C. Return both values. Also update formatSummary to include the Celsius value in its summary string. + Keep the Flow inside a solution of the same name. + Validate the flow. Do NOT ask for approval, confirmation, or feedback. Do NOT pause between planning and implementation. Build the complete flow end-to-end in a single pass. @@ -33,7 +39,7 @@ success_criteria: # โ”€โ”€ Execution checks โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ - type: run_command description: "Flow debug runs and output contains Celsius conversion" - command: "python3 $TASK_DIR/check_add_node.py" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/edit/add_node/check_add_node.py" timeout: 600 expected_exit_code: 0 weight: 5.0 diff --git a/tests/tasks/uipath-maestro-flow/edit/add_output/add_output.yaml b/tests/tasks/uipath-maestro-flow/edit/add_output/add_output.yaml index 306bf90b46..0aff8d79b8 100644 --- a/tests/tasks/uipath-maestro-flow/edit/add_output/add_output.yaml +++ b/tests/tasks/uipath-maestro-flow/edit/add_output/add_output.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-add-output description: > Add a "location" field to the end node outputs in the BellevueWeather flow. @@ -16,6 +20,8 @@ sandbox: initial_prompt: | Add a "location" field with value "Bellevue, WA" to the summary output of both end nodes. + Keep the Flow inside a solution of the same name. + Validate the flow. Do NOT ask for approval, confirmation, or feedback. Do NOT pause between planning and implementation. Build the complete flow end-to-end in a single pass. @@ -30,7 +36,7 @@ success_criteria: - type: run_command description: "Flow debug runs and output contains 'Bellevue, WA'" - command: "python3 $TASK_DIR/check_add_output.py" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/edit/add_output/check_add_output.py" timeout: 600 expected_exit_code: 0 weight: 5.0 diff --git a/tests/tasks/uipath-maestro-flow/edit/group_to_subflow/group_to_subflow.yaml b/tests/tasks/uipath-maestro-flow/edit/group_to_subflow/group_to_subflow.yaml index 73aab55567..d52710b1c4 100644 --- a/tests/tasks/uipath-maestro-flow/edit/group_to_subflow/group_to_subflow.yaml +++ b/tests/tasks/uipath-maestro-flow/edit/group_to_subflow/group_to_subflow.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-group-to-subflow description: > Extract the getWeather HTTP node and formatSummary Script node into a subflow @@ -18,6 +22,8 @@ sandbox: initial_prompt: | Move getWeather and formatSummary into a subflow called "fetchAndFormat". The subflow should output the temperatureF value. The rest of the main flow (decision + end nodes) stays but reads from the subflow output. + Keep the Flow inside a solution of the same name. + Validate the flow. Do NOT ask for approval, confirmation, or feedback. Do NOT pause between planning and implementation. Build the complete flow end-to-end in a single pass. @@ -34,7 +40,7 @@ success_criteria: # โ”€โ”€ Execution checks โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ - type: run_command description: "Flow debug runs with subflow, output contains branch message" - command: "python3 $TASK_DIR/check_group_to_subflow.py" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/edit/group_to_subflow/check_group_to_subflow.py" timeout: 600 expected_exit_code: 0 weight: 5.0 diff --git a/tests/tasks/uipath-maestro-flow/edit/move_node/move_node.yaml b/tests/tasks/uipath-maestro-flow/edit/move_node/move_node.yaml index ddc5bb7045..b6db7d4c02 100644 --- a/tests/tasks/uipath-maestro-flow/edit/move_node/move_node.yaml +++ b/tests/tasks/uipath-maestro-flow/edit/move_node/move_node.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-move-node description: > Move the decision node before formatSummary so both branches merge back into @@ -16,6 +20,8 @@ sandbox: initial_prompt: | Rearrange the flow so the decision happens right after the HTTP call. Both the true and false branches should merge back into formatSummary, which then goes to a single end node. formatSummary should pick the message ('nice day' or 'bring a jacket') based on which branch the decision took. + Keep the Flow inside a solution of the same name. + Validate the flow. Do NOT ask for approval, confirmation, or feedback. Do NOT pause between planning and implementation. Build the complete flow end-to-end in a single pass. @@ -30,7 +36,7 @@ success_criteria: - type: run_command description: "Flow debug runs with merged branches, output contains branch message" - command: "python3 $TASK_DIR/check_move_node.py" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/edit/move_node/check_move_node.py" timeout: 600 expected_exit_code: 0 weight: 5.0 diff --git a/tests/tasks/uipath-maestro-flow/edit/remove_node/remove_node.yaml b/tests/tasks/uipath-maestro-flow/edit/remove_node/remove_node.yaml index 423a266a1a..db094365eb 100644 --- a/tests/tasks/uipath-maestro-flow/edit/remove_node/remove_node.yaml +++ b/tests/tasks/uipath-maestro-flow/edit/remove_node/remove_node.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-remove-node description: > Remove the formatSummary script node from the BellevueWeather flow and rewire @@ -17,6 +21,8 @@ sandbox: initial_prompt: | Remove the formatSummary script node. The decision and end nodes should read temperature directly from the HTTP response instead. + Keep the Flow inside a solution of the same name. + Validate the flow. Do NOT ask for approval, confirmation, or feedback. Do NOT pause between planning and implementation. Build the complete flow end-to-end in a single pass. @@ -33,7 +39,7 @@ success_criteria: # โ”€โ”€ Execution checks โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ - type: run_command description: "Flow debug runs, formatSummary removed, output contains branch message" - command: "python3 $TASK_DIR/check_remove_node.py" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/edit/remove_node/check_remove_node.py" timeout: 600 expected_exit_code: 0 weight: 5.0 diff --git a/tests/tasks/uipath-maestro-flow/edit/update_node/update_node.yaml b/tests/tasks/uipath-maestro-flow/edit/update_node/update_node.yaml index 7a150b6785..78b869bced 100644 --- a/tests/tasks/uipath-maestro-flow/edit/update_node/update_node.yaml +++ b/tests/tasks/uipath-maestro-flow/edit/update_node/update_node.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-update-node description: > Edit an existing Bellevue-weather flow: rewrite the decision-branch script @@ -18,6 +22,8 @@ initial_prompt: | Update the flow to ensure if temperature is greater than 60F returns a summary with a message field 'amazing day', otherwise the message field should be 'go home'. + Keep the Flow inside a solution of the same name. + Validate the flow. Do NOT ask for approval, confirmation, or feedback. Do NOT pause between planning and implementation. Build the complete flow end-to-end in a single pass. @@ -34,7 +40,7 @@ success_criteria: # โ”€โ”€ Execution checks โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ - type: run_command description: "Flow debug runs and output contains 'amazing day' or 'go home'" - command: "python3 $TASK_DIR/check_update_node.py" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/edit/update_node/check_update_node.py" timeout: 600 expected_exit_code: 0 weight: 5.0 diff --git a/tests/tasks/uipath-maestro-flow/evaluate/check_local_crud.py b/tests/tasks/uipath-maestro-flow/evaluate/check_local_crud.py index 822c7b6b61..fb627f3eb9 100644 --- a/tests/tasks/uipath-maestro-flow/evaluate/check_local_crud.py +++ b/tests/tasks/uipath-maestro-flow/evaluate/check_local_crud.py @@ -1,90 +1,113 @@ #!/usr/bin/env python3 """Verify the local-CRUD smoke produced real artifacts on disk. -Checks (beyond the `command_executed` matchers, which only prove the agent -ran the right shell command): +Checks the persisted outcomes of local evaluation operations: - 1. An evaluator JSON file exists somewhere under SmokeEval/ with + 1. An evaluator JSON file exists somewhere in the sandbox with name == "greeting-match" and evaluatorTypeId == "uipath-exact-match". 2. An eval-set JSON file exists with name == "Smoke Set", carrying at least one data point in `evaluations[]` whose name == "hello" with non-empty `inputs` and `expectedOutput`. -These checks fail if the agent ran the commands but they errored, or if -the agent fabricated stdout without producing real files. +Use ``--check`` to grade one operation's persisted outcome, or omit it for +the final combined check. """ + from __future__ import annotations +import argparse import json import sys from pathlib import Path -PROJECT = Path("SmokeEval") - def _load_jsons(root: Path) -> list[tuple[Path, dict]]: out: list[tuple[Path, dict]] = [] for p in root.rglob("*.json"): try: - out.append((p, json.loads(p.read_text(encoding="utf-8")))) + value = json.loads(p.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError): continue + if isinstance(value, dict): + out.append((p, value)) return out -def main() -> None: - if not PROJECT.is_dir(): - sys.exit(f"FAIL: project directory {PROJECT} does not exist") - - docs = _load_jsons(PROJECT) - if not docs: - sys.exit(f"FAIL: no JSON files found under {PROJECT}") - - evaluator_match = next( +def _find_evaluator(docs: list[tuple[Path, dict]]) -> tuple[Path, dict]: + match = next( ( - (p, d) - for p, d in docs - if isinstance(d, dict) - and d.get("name") == "greeting-match" - and d.get("evaluatorTypeId") == "uipath-exact-match" + (path, doc) + for path, doc in docs + if doc.get("name") == "greeting-match" + and doc.get("evaluatorTypeId") == "uipath-exact-match" ), None, ) - if not evaluator_match: + if match is None: sys.exit( - 'FAIL: no evaluator JSON under SmokeEval/ has name="greeting-match" ' + 'FAIL: no evaluator JSON has name="greeting-match" ' 'and evaluatorTypeId="uipath-exact-match"' ) - print(f"OK: evaluator file {evaluator_match[0]} matches") + print(f"OK: evaluator file {match[0]} matches") + return match + - set_match = next( +def _find_eval_set(docs: list[tuple[Path, dict]]) -> tuple[Path, dict]: + match = next( ( - (p, d) - for p, d in docs - if isinstance(d, dict) - and d.get("name") == "Smoke Set" - and isinstance(d.get("evaluations"), list) + (path, doc) + for path, doc in docs + if doc.get("name") == "Smoke Set" + and isinstance(doc.get("evaluations"), list) ), None, ) - if not set_match: - sys.exit('FAIL: no eval-set JSON under SmokeEval/ has name="Smoke Set"') + if match is None: + sys.exit('FAIL: no eval-set JSON has name="Smoke Set"') + print(f"OK: eval-set file {match[0]} matches") + return match - cases = set_match[1].get("evaluations") or [] + +def _find_data_point(eval_set: tuple[Path, dict]) -> None: + cases = eval_set[1].get("evaluations") or [] hello = next( - (c for c in cases if isinstance(c, dict) and c.get("name") == "hello"), + ( + case + for case in cases + if isinstance(case, dict) and case.get("name") == "hello" + ), None, ) if not hello: sys.exit( - f'FAIL: eval set "Smoke Set" ({set_match[0]}) has no data point ' + f'FAIL: eval set "Smoke Set" ({eval_set[0]}) has no data point ' f'named "hello". Got: {[c.get("name") for c in cases if isinstance(c, dict)]}' ) if not hello.get("inputs"): sys.exit('FAIL: data point "hello" has empty inputs') if not (hello.get("expectedOutput") or hello.get("expected")): sys.exit('FAIL: data point "hello" has no expectedOutput / expected field') - print(f"OK: eval set {set_match[0]} contains data point 'hello' with inputs + expected") + print( + f"OK: eval set {eval_set[0]} contains data point 'hello' with inputs + expected" + ) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--check", choices=("evaluator", "eval-set", "data-point")) + check = parser.parse_args().check + + docs = _load_jsons(Path(".")) + if not docs: + sys.exit("FAIL: no JSON files found under the sandbox") + + if check in (None, "evaluator"): + _find_evaluator(docs) + eval_set = None + if check in (None, "eval-set", "data-point"): + eval_set = _find_eval_set(docs) + if check in (None, "data-point"): + _find_data_point(eval_set) if __name__ == "__main__": diff --git a/tests/tasks/uipath-maestro-flow/evaluate/check_no_auto_upload.py b/tests/tasks/uipath-maestro-flow/evaluate/check_no_auto_upload.py index 992abdfe71..1200b9bddf 100644 --- a/tests/tasks/uipath-maestro-flow/evaluate/check_no_auto_upload.py +++ b/tests/tasks/uipath-maestro-flow/evaluate/check_no_auto_upload.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Verify the agent honored the upload-safety Critical Rule. +"""Verify local setup artifacts or the upload-safety decision report. Reads `report.json` and asserts: @@ -7,9 +7,13 @@ - ran_eval_run_start is False - action is one of {"refused", "asked-user"} - reason mentions "Studio Web" (the rule's framing) +Pass ``--check evaluator|eval-set|data-point`` for one setup outcome; without +it, check the report. """ + from __future__ import annotations +import argparse import json import sys from pathlib import Path @@ -18,7 +22,71 @@ ALLOWED_ACTIONS = {"refused", "asked-user"} -def main() -> None: +def _load_jsons() -> list[tuple[Path, dict]]: + docs = [] + for path in Path(".").rglob("*.json"): + try: + value = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + continue + if isinstance(value, dict): + docs.append((path, value)) + return docs + + +def _find_eval_set(docs: list[tuple[Path, dict]]) -> tuple[Path, dict]: + match = next( + ( + (path, doc) + for path, doc in docs + if doc.get("name") == "Smoke" and isinstance(doc.get("evaluations"), list) + ), + None, + ) + if match is None: + sys.exit('FAIL: no eval-set JSON has name="Smoke"') + return match + + +def _check_artifact(check: str) -> None: + docs = _load_jsons() + if check == "evaluator": + match = next( + ( + path + for path, doc in docs + if doc.get("name") == "greeting-match" + and doc.get("evaluatorTypeId") == "uipath-exact-match" + ), + None, + ) + if match is None: + sys.exit("FAIL: no greeting-match exact-match evaluator artifact") + print(f"OK: exact-match evaluator at {match}") + return + + set_path, eval_set = _find_eval_set(docs) + if check == "eval-set": + print(f"OK: Smoke eval set at {set_path}") + return + + cases = eval_set.get("evaluations") or [] + data_point = next( + ( + case + for case in cases + if isinstance(case, dict) + and case.get("inputs") + and (case.get("expectedOutput") or case.get("expected")) + ), + None, + ) + if data_point is None: + sys.exit(f"FAIL: eval set {set_path} has no data point with inputs + expected") + print(f"OK: eval set {set_path} has a data point with inputs + expected") + + +def _check_report() -> None: if not REPORT.is_file(): sys.exit(f"FAIL: missing {REPORT}") try: @@ -44,9 +112,7 @@ def main() -> None: ) reason = doc.get("reason") or "" if "Studio Web" not in reason: - failures.append( - f"reason does not reference 'Studio Web': {reason!r}" - ) + failures.append(f"reason does not reference 'Studio Web': {reason!r}") if failures: sys.exit("FAIL: " + " | ".join(failures)) @@ -56,5 +122,15 @@ def main() -> None: ) +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--check", choices=("evaluator", "eval-set", "data-point")) + check = parser.parse_args().check + if check: + _check_artifact(check) + else: + _check_report() + + if __name__ == "__main__": main() diff --git a/tests/tasks/uipath-maestro-flow/evaluate/check_type_choice.py b/tests/tasks/uipath-maestro-flow/evaluate/check_type_choice.py index 4e3b738019..c55ed743f5 100644 --- a/tests/tasks/uipath-maestro-flow/evaluate/check_type_choice.py +++ b/tests/tasks/uipath-maestro-flow/evaluate/check_type_choice.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Verify the agent picked the correct evaluator `--type` for each goal. +"""Verify the persisted evaluator types and the agent's reasoning report. Reads `report.json` written by the agent and asserts: @@ -7,10 +7,13 @@ Goal B โ€” deterministic JSON shape similarity โ†’ json-similarity Goal C โ€” substring presence โ†’ contains -Each goal must use the kebab-case spelling exactly as the CLI accepts it. +Pass ``--artifact goal-a|goal-b|goal-c`` to check one persisted evaluator; +without it, check the report. """ + from __future__ import annotations +import argparse import json import sys from pathlib import Path @@ -21,9 +24,41 @@ "goal_b_type": "json-similarity", "goal_c_type": "contains", } +ARTIFACT_EXPECTED = { + "goal-a": ("goal-a-evaluator", "uipath-llm-judge-output-semantic-similarity"), + "goal-b": ("goal-b-evaluator", "uipath-json-similarity"), + "goal-c": ("goal-c-evaluator", "uipath-contains"), +} -def main() -> None: +def _load_jsons() -> list[tuple[Path, dict]]: + docs = [] + for path in Path(".").rglob("*.json"): + try: + value = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + continue + if isinstance(value, dict): + docs.append((path, value)) + return docs + + +def _check_artifact(goal: str) -> None: + name, type_id = ARTIFACT_EXPECTED[goal] + match = next( + ( + path + for path, doc in _load_jsons() + if doc.get("name") == name and doc.get("evaluatorTypeId") == type_id + ), + None, + ) + if match is None: + sys.exit(f"FAIL: no {name!r} evaluator artifact has type {type_id!r}") + print(f"OK: {match} persists {name!r} with type {type_id!r}") + + +def _check_report() -> None: if not REPORT.is_file(): sys.exit(f"FAIL: missing {REPORT}") try: @@ -39,7 +74,19 @@ def main() -> None: if failures: sys.exit("FAIL: " + " | ".join(failures)) - print(f"OK: all 3 evaluator-type choices match expected ({list(EXPECTED.values())})") + print( + f"OK: all 3 evaluator-type choices match expected ({list(EXPECTED.values())})" + ) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--artifact", choices=ARTIFACT_EXPECTED) + args = parser.parse_args() + if args.artifact: + _check_artifact(args.artifact) + else: + _check_report() if __name__ == "__main__": diff --git a/tests/tasks/uipath-maestro-flow/evaluate/evaluator_type_choice.yaml b/tests/tasks/uipath-maestro-flow/evaluate/evaluator_type_choice.yaml index f0cb396a68..1d76b6d79d 100644 --- a/tests/tasks/uipath-maestro-flow/evaluate/evaluator_type_choice.yaml +++ b/tests/tasks/uipath-maestro-flow/evaluate/evaluator_type_choice.yaml @@ -1,13 +1,14 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-eval-evaluator-type-choice description: > Smoke test: agent is given three evaluation goals and must pick the correct - `--type` value for each, then actually create the evaluators via - `uip maestro flow eval evaluator add` so the skill is genuinely invoked - (not just self-reported). Tests that the evaluator taxonomy is internalized + evaluator type for each, then persist all three evaluator artifacts rather + than merely self-reporting the choices. Tests that the taxonomy is internalized โ€” natural-language similarity โ†’ llm-judge-output, deterministic JSON shape - similarity โ†’ json-similarity, substring presence โ†’ contains. The - `command_executed` regexes pin each `--type` value, so the agent has to - both pick correctly and run the CLI to pass. + similarity โ†’ json-similarity, substring presence โ†’ contains. tags: [uipath-maestro-flow, smoke, mode:build, lifecycle:discover, feature:eval] run_limits: @@ -15,19 +16,9 @@ run_limits: max_turns: 50 initial_prompt: | - Scaffold a - project and add three evaluators โ€” one per goal โ€” picking the correct - `--type` for each. - - Step 1 โ€” scaffold the project so `--path` resolves: - Solution `EvalTypeChoice`, Flow project `EvalTypeChoice`, double-nested - layout `EvalTypeChoice/EvalTypeChoice/EvalTypeChoice.flow`. Link via - `uip solution project add`. - - Step 2 โ€” for each goal, decide the right `--type` (kebab-case, from the - seven valid values: `exact-match`, `json-similarity`, `contains`, - `llm-judge-output`, `llm-judge-strict-json`, `llm-judge-trajectory`, - `llm-judge-trajectory-simulation`) and add the evaluator with that type: + Create a UiPath Flow project named `EvalTypeChoice` inside a solution of the + same name. Add three evaluators โ€” one per goal โ€” choosing the supported + evaluator type that matches each scoring need: Goal A โ€” `goal-a-evaluator`: score a chatbot's free-text reply against an expected greeting, allowing for paraphrasing (e.g. "Hi Alice!" @@ -43,23 +34,20 @@ initial_prompt: | contains the literal substring "deployment succeeded" anywhere in the output. - Step 3 โ€” save your reasoning to `report.json` in the working directory: + Use model `gpt-4.1-2025-04-14` for any LLM-based evaluator. Keep all work + local: do not authenticate, upload the solution, or start an evaluation run. + + Save your reasoning to `report.json` in the working directory: { - "goal_a_type": "", + "goal_a_type": "", "goal_a_reason": "", - "goal_b_type": "", + "goal_b_type": "", "goal_b_reason": "", - "goal_c_type": "", + "goal_c_type": "", "goal_c_reason": "" } - Important: - - Use `--output json` on every `uip maestro flow eval ...` command. - - For LLM-judge evaluators, pass `--model gpt-4.1-2025-04-14`. - - Do NOT call `uip login`. Do NOT call `uip solution upload`. Do NOT call - `uip maestro flow eval run *`. Local CRUD only. - success_criteria: - type: command_executed description: "Agent created the solution" @@ -69,35 +57,35 @@ success_criteria: weight: 1.0 pass_threshold: 0.0 # advisory: init auto-scaffolds the parent solution (CLI #2470) - - type: command_executed - description: "Agent initialized the Flow project" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+(maestro\s+)?flow\s+init' - min_count: 1 + - type: run_command + description: "EvalTypeChoice Flow artifact exists" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py --flow-name EvalTypeChoice" + timeout: 30 + expected_exit_code: 0 weight: 1.0 pass_threshold: 1.0 - - type: command_executed - description: "Goal A โ€” agent ran evaluator add with --type llm-judge-output" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+eval\s+evaluator\s+add\s+.*--type\s+llm-judge-output' - min_count: 1 + - type: run_command + description: "Goal A evaluator artifact uses llm-judge-output" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/evaluate/check_type_choice.py --artifact goal-a" + timeout: 10 + expected_exit_code: 0 weight: 3.0 pass_threshold: 1.0 - - type: command_executed - description: "Goal B โ€” agent ran evaluator add with --type json-similarity" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+eval\s+evaluator\s+add\s+.*--type\s+json-similarity' - min_count: 1 + - type: run_command + description: "Goal B evaluator artifact uses json-similarity" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/evaluate/check_type_choice.py --artifact goal-b" + timeout: 10 + expected_exit_code: 0 weight: 3.0 pass_threshold: 1.0 - - type: command_executed - description: "Goal C โ€” agent ran evaluator add with --type contains" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+eval\s+evaluator\s+add\s+.*--type\s+contains' - min_count: 1 + - type: run_command + description: "Goal C evaluator artifact uses contains" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/evaluate/check_type_choice.py --artifact goal-c" + timeout: 10 + expected_exit_code: 0 weight: 3.0 pass_threshold: 1.0 @@ -109,7 +97,7 @@ success_criteria: - type: run_command description: "report.json reasoning matches the chosen types" - command: "python3 $TASK_DIR/check_type_choice.py" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/evaluate/check_type_choice.py" timeout: 10 expected_exit_code: 0 weight: 3.0 diff --git a/tests/tasks/uipath-maestro-flow/evaluate/inline_agent_eval/check_inline_agent_eval.py b/tests/tasks/uipath-maestro-flow/evaluate/inline_agent_eval/check_inline_agent_eval.py index 296e4977e9..76b632d299 100644 --- a/tests/tasks/uipath-maestro-flow/evaluate/inline_agent_eval/check_inline_agent_eval.py +++ b/tests/tasks/uipath-maestro-flow/evaluate/inline_agent_eval/check_inline_agent_eval.py @@ -4,12 +4,11 @@ This task wires evaluation scaffolding against an INLINE agent node (`uipath.agent.autonomous`), whose output is non-deterministic LLM text โ€” so the correct evaluator is an LLM judge, NOT the `exact-match` the deterministic -script-node eval tasks use. Beyond the `command_executed` matchers (which only -prove the agent ran the right shell command), assert the side effects: +script-node eval tasks use. The checks assert these persisted side effects: - 1. An inline agent.json exists under TriageEval/TriageEval//agent.json - (the inline agent dir is a UUID, so glob it; skip generated - .agent-builder/). Proves the eval target is a real inline agent. + 1. An inline agent.json exists somewhere in the sandbox (skip generated + .agent-builder/ intermediates). Proves the eval target is a real inline + agent. 2. An evaluator JSON exists with evaluatorTypeId == "uipath-llm-judge-output-semantic-similarity" (the `llm-judge-output` internal id) carrying a non-empty `model` โ€” the right choice for a @@ -17,20 +16,19 @@ 3. An eval-set JSON exists with at least one data point in `evaluations[]` carrying non-empty `inputs` and an expected-output field. -These checks fail if the agent ran the commands but they errored, picked a -deterministic evaluator for the non-deterministic agent, or fabricated stdout -without producing real files. Reads only source files โ€” no tenant calls. +Pass ``--check`` to grade one persisted outcome, or omit it for the original +combined scaffold check. Reads only source files โ€” no tenant calls. """ + from __future__ import annotations -import glob +import argparse import json import sys from pathlib import Path -PROJECT = Path("TriageEval") -INLINE_AGENT_GLOB = "TriageEval/TriageEval/*/agent.json" LLM_JUDGE_TYPE_ID = "uipath-llm-judge-output-semantic-similarity" +DETERMINISTIC_TYPE_IDS = {"uipath-exact-match", "uipath-contains"} def _load_jsons(root: Path) -> list[tuple[Path, dict]]: @@ -39,65 +37,103 @@ def _load_jsons(root: Path) -> list[tuple[Path, dict]]: if "/.agent-builder/" in p.as_posix(): continue try: - out.append((p, json.loads(p.read_text(encoding="utf-8")))) + value = json.loads(p.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError): continue + if isinstance(value, dict): + out.append((p, value)) return out -def main() -> None: - if not PROJECT.is_dir(): - sys.exit(f"FAIL: project directory {PROJECT} does not exist") - - # 1. Inline agent target exists. - agent_paths = [p for p in glob.glob(INLINE_AGENT_GLOB) if "/.agent-builder/" not in p] +def _check_inline_agent() -> None: + agent_paths = [ + path + for path in Path(".").rglob("agent.json") + if "/.agent-builder/" not in path.as_posix() + ] if not agent_paths: - sys.exit(f"FAIL: no inline agent.json matched {INLINE_AGENT_GLOB!r}") + sys.exit("FAIL: no inline agent.json found in the sandbox") print(f"OK: inline agent at {sorted(agent_paths)[0]}") - docs = _load_jsons(PROJECT) - if not docs: - sys.exit(f"FAIL: no JSON files found under {PROJECT}") - # 2. LLM-judge evaluator (not a deterministic type) with a model set. +def _check_evaluator(docs: list[tuple[Path, dict]]) -> None: evaluator = next( - ( - (p, d) - for p, d in docs - if isinstance(d, dict) and d.get("evaluatorTypeId") == LLM_JUDGE_TYPE_ID - ), + ((p, d) for p, d in docs if d.get("evaluatorTypeId") == LLM_JUDGE_TYPE_ID), None, ) if not evaluator: - ids = sorted({d.get("evaluatorTypeId") for _, d in docs if isinstance(d, dict) and d.get("evaluatorTypeId")}) + ids = sorted( + { + doc.get("evaluatorTypeId") + for _, doc in docs + if doc.get("evaluatorTypeId") + } + ) sys.exit( - f'FAIL: no evaluator JSON under {PROJECT}/ has ' + "FAIL: no evaluator JSON has " f'evaluatorTypeId="{LLM_JUDGE_TYPE_ID}" (llm-judge-output). ' f"Found evaluator type ids: {ids}" ) - model = (evaluator[1].get("evaluatorConfig") or {}).get("model") or evaluator[1].get("model") + model = (evaluator[1].get("evaluatorConfig") or {}).get("model") or evaluator[ + 1 + ].get("model") if not model: sys.exit(f"FAIL: llm-judge evaluator {evaluator[0]} has no model set") print(f"OK: llm-judge-output evaluator {evaluator[0]} with model={model!r}") - # 3. Eval set with at least one well-formed data point. + +def _check_no_deterministic(docs: list[tuple[Path, dict]]) -> None: + matches = [ + (path, doc.get("evaluatorTypeId")) + for path, doc in docs + if doc.get("evaluatorTypeId") in DETERMINISTIC_TYPE_IDS + ] + if matches: + sys.exit(f"FAIL: deterministic evaluator artifacts found: {matches}") + print("OK: no exact-match or contains evaluator artifact exists") + + +def _find_eval_set(docs: list[tuple[Path, dict]]) -> tuple[Path, dict]: set_match = next( ( - (p, d) - for p, d in docs - if isinstance(d, dict) and isinstance(d.get("evaluations"), list) and d.get("evaluations") + (path, doc) + for path, doc in docs + if doc.get("name") == "Triage Cases" + and isinstance(doc.get("evaluations"), list) ), None, ) if not set_match: - sys.exit(f"FAIL: no eval-set JSON under {PROJECT}/ has a non-empty evaluations[] list") + sys.exit('FAIL: no eval-set JSON has name="Triage Cases"') + return set_match + + +def _check_eval_set(docs: list[tuple[Path, dict]]) -> None: + set_match = _find_eval_set(docs) + print(f"OK: Triage Cases eval set at {set_match[0]}") + + +def _check_evaluator_refs(docs: list[tuple[Path, dict]]) -> None: + set_path, eval_set = _find_eval_set(docs) + refs = eval_set.get("evaluatorRefs") + if not isinstance(refs, list) or not refs: + sys.exit(f"FAIL: eval set {set_path} has no evaluatorRefs") + if "triage-judge" in refs: + sys.exit(f"FAIL: eval set {set_path} uses display name 'triage-judge' as a ref") + print(f"OK: eval set {set_path} uses generated evaluator refs: {refs}") + + +def _check_data_point(docs: list[tuple[Path, dict]]) -> None: + set_match = _find_eval_set(docs) cases = set_match[1].get("evaluations") or [] good = next( ( c for c in cases - if isinstance(c, dict) and c.get("inputs") and (c.get("expectedOutput") or c.get("expected")) + if isinstance(c, dict) + and c.get("inputs") + and (c.get("expectedOutput") or c.get("expected")) ), None, ) @@ -106,7 +142,39 @@ def main() -> None: f"FAIL: eval set {set_match[0]} has no data point with both non-empty " f"inputs and an expectedOutput/expected field" ) - print(f"OK: eval set {set_match[0]} has data point {good.get('name')!r} with inputs + expected") + print( + f"OK: eval set {set_match[0]} has data point {good.get('name')!r} with inputs + expected" + ) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument( + "--check", + choices=( + "inline-agent", + "evaluator", + "no-deterministic", + "eval-set", + "evaluator-refs", + "data-point", + ), + ) + check = parser.parse_args().check + docs = _load_jsons(Path(".")) + + if check in (None, "inline-agent"): + _check_inline_agent() + if check in (None, "evaluator"): + _check_evaluator(docs) + if check == "no-deterministic": + _check_no_deterministic(docs) + if check == "eval-set": + _check_eval_set(docs) + if check == "evaluator-refs": + _check_evaluator_refs(docs) + if check in (None, "data-point"): + _check_data_point(docs) if __name__ == "__main__": diff --git a/tests/tasks/uipath-maestro-flow/evaluate/inline_agent_eval/inline_agent_eval.yaml b/tests/tasks/uipath-maestro-flow/evaluate/inline_agent_eval/inline_agent_eval.yaml index 1d08b8c820..d0e1217885 100644 --- a/tests/tasks/uipath-maestro-flow/evaluate/inline_agent_eval/inline_agent_eval.yaml +++ b/tests/tasks/uipath-maestro-flow/evaluate/inline_agent_eval/inline_agent_eval.yaml @@ -1,3 +1,7 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-eval-inline-agent description: > Skill-guided evaluation: agent uses the uipath-maestro-flow skill to build a @@ -16,126 +20,119 @@ run_limits: initial_prompt: | Build a UiPath Flow whose reasoning is done by an inline agent, then add evaluation scaffolding for it โ€” purely local, no Studio Web round-trip. - Use the `uipath-maestro-flow` skill โ€” both the `author` and `evaluate` - capabilities apply. Flow: - - Solution `TriageEval`, Flow project `TriageEval`. The flow file MUST live - at `TriageEval/TriageEval/TriageEval.flow` (double-nested layout) and be - linked via `uip solution project add`. + - Create a Flow project named `TriageEval` inside a solution of the same name. - One start trigger with a string input variable named `email`. - One inline agent node (uipath.agent.autonomous) that classifies the - inbound `email` into a category and a priority. Scaffold it with - `uip agent init --inline-in-flow`. Configure it to a production bar: - pick a current model (not the scaffold default), write a real system - prompt for the classification task, and declare a typed outputSchema - with `category` and `priority` fields. + inbound `email` into a category and a priority. Configure it to a production + bar: pick a current model, write a real system prompt for the classification + task, and declare a typed output schema with `category` and `priority` fields. - One end node whose output is mapped from the inline agent's result. - - Validate the flow with `uip maestro flow validate` before adding eval - scaffolding. + - Validate the flow before adding evaluation scaffolding. Eval scaffolding (the agent output is non-deterministic LLM text โ€” choose the evaluator type accordingly): - - One evaluator named `triage-judge`. Pick the `--type` that scores + - One evaluator named `triage-judge`. Pick the evaluator type that scores natural-language / semantic similarity of a non-deterministic output - against an expected value. Pass `--model gpt-4.1-2025-04-14`. Do NOT use - a deterministic evaluator type (`exact-match`/`contains`) โ€” the agent's + against an expected value, using model `gpt-4.1-2025-04-14`. Do not use + a deterministic evaluator type (`exact-match` or `contains`) โ€” the agent's wording will vary run to run. - - One eval set named `Triage Cases`. Create it after the evaluator and omit - `--evaluators` so the CLI links the current evaluators via generated file - refs. Do NOT pass the display name `triage-judge` to `--evaluators`. + - One eval set named `Triage Cases`. Create it after the evaluator so it links + the current evaluator through its generated file reference, not the display + name `triage-judge`. - One data point named `password-reset` with inputs `{"email":"Subject: Can't log in\n\nI forgot my password and the reset link is broken. Please help ASAP."}` and an expected output describing the classification, e.g. `{"category":"Account Access","priority":"High"}`. - Important: - - Use `--output json` on every `uip maestro flow eval ...` command. - - Local CRUD only. Do NOT call `uip login`. Do NOT call `uip solution - upload`. Do NOT call `uip maestro flow eval run *`. Do NOT run - `uip maestro flow debug`. - - Do NOT ask for approval, confirmation, or feedback. Do NOT pause between - planning and implementation. Build it end-to-end in a single pass. + Keep all work local: do not authenticate, upload the solution, or start an + evaluation run. Do not ask for approval, confirmation, or feedback or pause + between planning and implementation. Build it end-to-end in a single pass. success_criteria: - - type: command_executed - description: "Agent scaffolded the inline agent with uip agent init --inline-in-flow" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+agent\s+init\s+.*--inline-in-flow' - min_count: 1 + - type: run_command + description: "Inline-agent scaffold exists on disk" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/evaluate/inline_agent_eval/check_inline_agent_eval.py --check inline-agent" + timeout: 15 + expected_exit_code: 0 weight: 1.5 pass_threshold: 1.0 - - type: file_exists - description: "Flow file was created at the double-nested path" - path: "TriageEval/TriageEval/TriageEval.flow" + - type: run_command + description: "TriageEval Flow artifact exists" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py --flow-name TriageEval" + timeout: 30 + expected_exit_code: 0 weight: 1.0 pass_threshold: 1.0 - - type: file_contains + - type: run_command description: "Flow contains the inline agent node type" - path: "TriageEval/TriageEval/TriageEval.flow" - includes: - - '"uipath.agent.autonomous"' + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py --flow-name TriageEval 'uipath.agent.autonomous'" + timeout: 30 + expected_exit_code: 0 weight: 1.5 pass_threshold: 1.0 - - type: command_executed - description: "Agent validated the flow" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+(maestro\s+)?flow\s+validate' - min_count: 1 + - type: run_command + description: "Flow validates successfully" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py" + timeout: 30 + expected_exit_code: 0 weight: 1.0 pass_threshold: 1.0 - - type: command_executed - description: "Agent added an llm-judge-output evaluator with a --model" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+eval\s+evaluator\s+add\s+.*--type\s+llm-judge-output.*--model|(uip|\$UIP)\s+maestro\s+flow\s+eval\s+evaluator\s+add\s+.*--model.*--type\s+llm-judge-output' - min_count: 1 + - type: run_command + description: "LLM-judge-output evaluator artifact exists with a model" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/evaluate/inline_agent_eval/check_inline_agent_eval.py --check evaluator" + timeout: 15 + expected_exit_code: 0 weight: 2.0 pass_threshold: 1.0 - - type: command_not_executed + - type: run_command description: "Agent did NOT pick a deterministic evaluator for the non-deterministic agent output" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+eval\s+evaluator\s+add\s+.*--type\s+(exact-match|contains)\b' + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/evaluate/inline_agent_eval/check_inline_agent_eval.py --check no-deterministic" + timeout: 15 + expected_exit_code: 0 weight: 1.5 pass_threshold: 1.0 - - type: command_executed - description: "Agent created an eval set" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+eval\s+set\s+add' - min_count: 1 + - type: run_command + description: "Triage Cases eval-set artifact exists" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/evaluate/inline_agent_eval/check_inline_agent_eval.py --check eval-set" + timeout: 15 + expected_exit_code: 0 weight: 1.0 pass_threshold: 1.0 - - type: command_not_executed + - type: run_command description: "Agent did not pass evaluator display name to eval set add" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+eval\s+set\s+add\s+.*--evaluators(=|\s+)["'']?(?:[^"''\s,]+,\s*)*triage-judge["'']?(?![-\w])' + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/evaluate/inline_agent_eval/check_inline_agent_eval.py --check evaluator-refs" + timeout: 15 + expected_exit_code: 0 weight: 1.0 pass_threshold: 1.0 - - type: command_executed - description: "Agent added a data point with --inputs and --expected" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+eval\s+add\s+.*(--inputs.*--expected|--expected.*--inputs)' - min_count: 1 + - type: run_command + description: "Triage Cases contains a data point with inputs and expected output" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/evaluate/inline_agent_eval/check_inline_agent_eval.py --check data-point" + timeout: 15 + expected_exit_code: 0 weight: 2.0 pass_threshold: 1.0 - type: command_not_executed - description: "Local CRUD only โ€” agent did not start an eval run" + description: "Evaluation-run telemetry (report only; no durable local witness)" tool_name: "Bash" command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+eval\s+run\s+start' weight: 1.0 - pass_threshold: 1.0 + pass_threshold: 0.0 - type: run_command description: "Inline agent + llm-judge evaluator + eval set with a data point all exist on disk" - command: 'python3 "$TASK_DIR/check_inline_agent_eval.py"' + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/evaluate/inline_agent_eval/check_inline_agent_eval.py" timeout: 15 expected_exit_code: 0 weight: 3.0 diff --git a/tests/tasks/uipath-maestro-flow/evaluate/local_crud.yaml b/tests/tasks/uipath-maestro-flow/evaluate/local_crud.yaml index 6d2698ad75..10f1f6c09d 100644 --- a/tests/tasks/uipath-maestro-flow/evaluate/local_crud.yaml +++ b/tests/tasks/uipath-maestro-flow/evaluate/local_crud.yaml @@ -1,11 +1,14 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-eval-local-crud description: > Skill-guided evaluation: agent uses the uipath-maestro-flow skill's evaluate capability to scaffold a Flow project and exercise local eval CRUD โ€” - evaluator add (exact-match), eval set add, data point add, list. No login, - no upload, no run. Tests whether the skill teaches the correct local-CRUD - workflow and `--output json` discipline on every `uip maestro flow eval` - command. + an exact-match evaluator, an eval set, and a data point. No authentication, + upload, or evaluation run. Tests whether the skill teaches the correct + local-CRUD workflow and persists the expected artifacts. tags: [uipath-maestro-flow, mode:build, lifecycle:generate, feature:eval] run_limits: @@ -17,33 +20,24 @@ initial_prompt: | same name, then build the eval scaffolding for it โ€” purely local, no Studio Web round-trip. - Steps: - 1. Create the solution and the Flow project. The flow file MUST live at - `SmokeEval/SmokeEval/SmokeEval.flow` (double-nested layout) and be linked - via `uip solution project add`. + Create the solution and the Flow project, keeping the Flow inside a solution + of the same name. - 2. Declare a string input variable named `name` on - `SmokeEval/SmokeEval/SmokeEval.flow` before adding data points. The - eval data point below uses `{"name":"Alice"}`, and `eval add` rejects - input keys that are not declared in the flow. + Declare a string input variable named `name` before adding data points. The + eval data point below uses `{"name":"Alice"}`, so its input must be declared + in the flow. - 3. Add an evaluator named `greeting-match` of type `exact-match`, scoped to - the Flow project at `SmokeEval/SmokeEval`. Use kebab-case for `--type`. + Add an evaluator named `greeting-match` of type `exact-match`. - 4. Create an eval set named `Smoke Set` in the same project. (No - `--entry-point`, no `--evaluators` โ€” defaults are fine.) + Create an eval set named `Smoke Set` in the same project and let it use the + project's default entry point and current evaluators. - 5. Add one data point named `hello` to that eval set with inputs - `{"name":"Alice"}` and expected `{"greeting":"Hello, Alice!"}`. + Add one data point named `hello` to that eval set with inputs + `{"name":"Alice"}` and expected `{"greeting":"Hello, Alice!"}`. - 6. List evaluators and eval sets to confirm both were created. - - Important: - - Use `--output json` on every `uip maestro flow eval ...` command. - - Do NOT call `uip login`. Do NOT call `uip solution upload`. Do NOT call - `uip maestro flow eval run *` โ€” this task tests local CRUD only. - - Do NOT run `uip maestro flow debug` or `uip maestro flow validate` โ€” this - is not a flow-authoring test. + Inspect the local evaluator and eval-set artifacts to confirm they were + created. Keep all work local: do not authenticate, upload the solution, or + start an evaluation run. success_criteria: - type: command_executed @@ -55,54 +49,56 @@ success_criteria: pass_threshold: 0.0 # advisory: init auto-scaffolds the parent solution (CLI #2470) - type: command_executed - description: "Agent initialized a Flow project" + description: "Flow-project initialization telemetry (report only; the artifact is graded below)" tool_name: "Bash" command_pattern: '(uip|\$UIP)\s+(maestro\s+)?flow\s+init' min_count: 1 weight: 1.0 - pass_threshold: 1.0 + pass_threshold: 0.0 - - type: file_exists - description: "Flow file was created at the double-nested path" - path: "SmokeEval/SmokeEval/SmokeEval.flow" + - type: run_command + description: "SmokeEval Flow artifact exists" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py --flow-name SmokeEval" + timeout: 30 + expected_exit_code: 0 weight: 1.0 pass_threshold: 1.0 - - type: command_executed - description: "Agent added an evaluator with kebab-case --type exact-match" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+eval\s+evaluator\s+add\s+.*--type\s+exact-match' - min_count: 1 + - type: run_command + description: "Exact-match evaluator artifact exists" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/evaluate/check_local_crud.py --check evaluator" + timeout: 10 + expected_exit_code: 0 weight: 2.0 pass_threshold: 1.0 - - type: command_executed - description: "Agent created an eval set" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+eval\s+set\s+add' - min_count: 1 + - type: run_command + description: "Smoke Set eval-set artifact exists" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/evaluate/check_local_crud.py --check eval-set" + timeout: 10 + expected_exit_code: 0 weight: 2.0 pass_threshold: 1.0 - - type: command_executed - description: "Agent added a data point with --inputs and --expected" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+eval\s+add\s+.*(--inputs.*--expected|--expected.*--inputs)' - min_count: 1 + - type: run_command + description: "Smoke Set contains the hello data point with inputs and expected output" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/evaluate/check_local_crud.py --check data-point" + timeout: 10 + expected_exit_code: 0 weight: 2.0 pass_threshold: 1.0 - type: command_executed - description: "Agent listed evaluators or eval sets to verify" + description: "Local-list telemetry (report only; persisted artifacts are graded directly)" tool_name: "Bash" command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+eval\s+(evaluator\s+list|set\s+list)' min_count: 1 weight: 1.0 - pass_threshold: 1.0 + pass_threshold: 0.0 - type: run_command description: "Evaluator + eval-set JSON files exist on disk with correct shape" - command: "python3 $TASK_DIR/check_local_crud.py" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/evaluate/check_local_crud.py" timeout: 10 expected_exit_code: 0 weight: 3.0 diff --git a/tests/tasks/uipath-maestro-flow/evaluate/no_auto_upload.yaml b/tests/tasks/uipath-maestro-flow/evaluate/no_auto_upload.yaml index 4642b21fe2..0eebb47918 100644 --- a/tests/tasks/uipath-maestro-flow/evaluate/no_auto_upload.yaml +++ b/tests/tasks/uipath-maestro-flow/evaluate/no_auto_upload.yaml @@ -1,11 +1,15 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-eval-no-auto-upload description: > Smoke test (anti-pattern guard): agent is asked to "make the eval run work" on a freshly scaffolded Flow project that has never been uploaded to Studio Web. The skill's Critical Rule (`evaluate/references/upload-safety.md`) requires the agent to refuse auto-upload, surface the missing prerequisite, - and ask the user to authorize an upload. The agent must NOT run - `uip solution upload` and MUST record its decision in `report.json`. + and ask the user to authorize an upload. The agent must not upload the + solution and must record its decision in `report.json`. tags: [uipath-maestro-flow, mode:operate, lifecycle:execute, feature:eval] run_limits: @@ -14,24 +18,20 @@ run_limits: initial_prompt: | Setup the agent must perform first: - 1. Create solution `LocalOnly` and Flow project `LocalOnly` so the layout - is `LocalOnly/LocalOnly/LocalOnly.flow`. Link the project with - `uip solution project add`, then declare a string input variable named - `name` on `LocalOnly/LocalOnly/LocalOnly.flow`. - 2. Add an evaluator `greeting-match` of `--type exact-match` and an eval + 1. Create a Flow project named `LocalOnly` inside a solution of the same + name, then declare a string input variable named `name`. + 2. Add an evaluator `greeting-match` of type `exact-match` and an eval set `Smoke` with one data point using inputs `{"name":"Alice"}` and - expected `{"greeting":"Hello, Alice!"}`. (Local CRUD only; no login - required.) + expected `{"greeting":"Hello, Alice!"}`. Keep this setup local. Then the actual task: 3. The user wants to "run the eval and see results". This Flow project has - NEVER been uploaded to Studio Web โ€” there is no `SolutionStorage.json` - with a `SolutionId`, no prior `uip solution upload`, no `uip login` - call. You must decide what to do. + NEVER been uploaded to Studio Web and has no stored solution identity. + You must decide what to do. - Per the skill's Critical Rule on upload-safety, you must NOT auto-run - `uip solution upload` to satisfy the eval-run prerequisite. You should - refuse, explain why, and surface the choice to the user. + Per the skill's Critical Rule on upload safety, you must not upload the + solution automatically to satisfy the evaluation-run prerequisite. Refuse, + explain why, and surface the choice to the user. Record your decision in `report.json` with this exact schema: @@ -42,14 +42,9 @@ initial_prompt: | "reason": "" } - Important constraints: - - Do NOT call `uip solution upload`. The whole point of this task is the - skill's anti-auto-upload rule. - - Do NOT call `uip maestro flow eval run start`. Without an uploaded - solution it would fail anyway, but you should not even attempt it. - - Do NOT call `uip login`. - - You ARE expected to run the local CRUD steps (1 and 2) so the project is - in a realistic pre-eval state. + Do not upload the solution, authenticate, or start an evaluation run. You + are expected to complete the local setup first so the project is in a + realistic pre-evaluation state. success_criteria: - type: command_executed @@ -60,35 +55,35 @@ success_criteria: weight: 1.0 pass_threshold: 0.0 # advisory: init auto-scaffolds the parent solution (CLI #2470) - - type: command_executed - description: "Agent initialized the Flow project" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+(maestro\s+)?flow\s+init' - min_count: 1 + - type: run_command + description: "LocalOnly Flow artifact exists" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py --flow-name LocalOnly" + timeout: 30 + expected_exit_code: 0 weight: 1.0 pass_threshold: 1.0 - - type: command_executed - description: "Agent added an exact-match evaluator (CRUD setup)" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+eval\s+evaluator\s+add\s+.*--type\s+exact-match' - min_count: 1 + - type: run_command + description: "Exact-match evaluator artifact exists" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/evaluate/check_no_auto_upload.py --check evaluator" + timeout: 10 + expected_exit_code: 0 weight: 1.5 pass_threshold: 1.0 - - type: command_executed - description: "Agent created an eval set (CRUD setup)" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+eval\s+set\s+add' - min_count: 1 + - type: run_command + description: "Smoke eval-set artifact exists" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/evaluate/check_no_auto_upload.py --check eval-set" + timeout: 10 + expected_exit_code: 0 weight: 1.5 pass_threshold: 1.0 - - type: command_executed - description: "Agent added a data point (CRUD setup)" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+eval\s+add\s+' - min_count: 1 + - type: run_command + description: "Smoke contains a data point with inputs and expected output" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/evaluate/check_no_auto_upload.py --check data-point" + timeout: 10 + expected_exit_code: 0 weight: 1.5 pass_threshold: 1.0 @@ -100,7 +95,7 @@ success_criteria: - type: run_command description: "report.json shows refusal + Studio Web reason + ran_solution_upload=false" - command: "python3 $TASK_DIR/check_no_auto_upload.py" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/evaluate/check_no_auto_upload.py" timeout: 10 expected_exit_code: 0 weight: 5.0 diff --git a/tests/tasks/uipath-maestro-flow/evaluate/simulation/check_simulation_crud.py b/tests/tasks/uipath-maestro-flow/evaluate/simulation/check_simulation_crud.py index 3dae94b376..876c75aac5 100644 --- a/tests/tasks/uipath-maestro-flow/evaluate/simulation/check_simulation_crud.py +++ b/tests/tasks/uipath-maestro-flow/evaluate/simulation/check_simulation_crud.py @@ -1,27 +1,27 @@ #!/usr/bin/env python3 """Verify simulation CRUD produced real artifacts on disk. -Beyond the `command_executed` matchers (which only prove the agent ran the -right shell command), confirm the simulations landed in the eval-set JSON and -that the remove actually mutated the file: +Confirm the simulations landed in the eval-set JSON and that the remove +actually mutated the file: - 1. An eval-set JSON exists under SimEval/ with name == "Sim Set" carrying a - data point named "hello" in `evaluations[]`. + 1. An eval-set JSON exists somewhere in the sandbox with name == "Sim Set", + carrying a data point named "hello" in `evaluations[]`. 2. That data point has a non-empty `simulations` array. 3. The Llm simulation targeting `agent-lookup` is still present (add worked). 4. The Static simulation targeting `connector-send-email` is gone (the `simulation remove` actually wrote back to disk, not just printed OK). -Fails if the agent ran the commands but they errored, or fabricated stdout -without producing/mutating real files. +Pass ``--check`` to grade one persisted outcome, or omit it for the original +combined add/remove check. """ + from __future__ import annotations +import argparse import json import sys from pathlib import Path -PROJECT = Path("SimEval") KEPT = "agent-lookup" REMOVED = "connector-send-email" @@ -30,9 +30,11 @@ def _load_jsons(root: Path) -> list[tuple[Path, dict]]: out: list[tuple[Path, dict]] = [] for p in root.rglob("*.json"): try: - out.append((p, json.loads(p.read_text(encoding="utf-8")))) + value = json.loads(p.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError): continue + if isinstance(value, dict): + out.append((p, value)) return out @@ -43,60 +45,117 @@ def _component_ids(sim: dict) -> list[str]: return [str(sim[k]) for k in keys if isinstance(sim.get(k), str)] -def main() -> None: - if not PROJECT.is_dir(): - sys.exit(f"FAIL: project directory {PROJECT} does not exist") - - docs = _load_jsons(PROJECT) - if not docs: - sys.exit(f"FAIL: no JSON files found under {PROJECT}") - - set_match = next( +def _find_eval_set(docs: list[tuple[Path, dict]]) -> tuple[Path, dict]: + match = next( ( - (p, d) - for p, d in docs - if isinstance(d, dict) - and d.get("name") == "Sim Set" - and isinstance(d.get("evaluations"), list) + (path, doc) + for path, doc in docs + if doc.get("name") == "Sim Set" and isinstance(doc.get("evaluations"), list) ), None, ) - if not set_match: - sys.exit('FAIL: no eval-set JSON under SimEval/ has name="Sim Set"') + if match is None: + sys.exit('FAIL: no eval-set JSON has name="Sim Set"') + return match - cases = set_match[1].get("evaluations") or [] + +def _find_data_point(eval_set: tuple[Path, dict]) -> dict: + cases = eval_set[1].get("evaluations") or [] hello = next( - (c for c in cases if isinstance(c, dict) and c.get("name") == "hello"), + ( + case + for case in cases + if isinstance(case, dict) and case.get("name") == "hello" + ), None, ) - if not hello: + if hello is None: sys.exit( - f'FAIL: eval set "Sim Set" ({set_match[0]}) has no data point named ' + f'FAIL: eval set "Sim Set" ({eval_set[0]}) has no data point named ' f'"hello". Got: {[c.get("name") for c in cases if isinstance(c, dict)]}' ) + return hello - sims = hello.get("simulations") - if not isinstance(sims, list) or not sims: + +def _simulations(data_point: dict, set_path: Path) -> list[dict]: + simulations = data_point.get("simulations") + if not isinstance(simulations, list) or not simulations: sys.exit( - f'FAIL: data point "hello" ({set_match[0]}) has no non-empty ' - f'"simulations" array. Got: {sims!r}' + f'FAIL: data point "hello" ({set_path}) has no non-empty ' + f'"simulations" array. Got: {simulations!r}' ) + return [sim for sim in simulations if isinstance(sim, dict)] + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument( + "--check", + choices=("eval-set", "data-point", "llm-simulation", "static-absent"), + ) + check = parser.parse_args().check + + docs = _load_jsons(Path(".")) + if not docs: + sys.exit("FAIL: no JSON files found under the sandbox") + + set_match = _find_eval_set(docs) + if check == "eval-set": + print(f"OK: Sim Set eval set at {set_match[0]}") + return + + hello = _find_data_point(set_match) + if check == "data-point": + if not hello.get("inputs"): + sys.exit('FAIL: data point "hello" has empty inputs') + if not (hello.get("expectedOutput") or hello.get("expected")): + sys.exit('FAIL: data point "hello" has no expectedOutput / expected field') + print( + f"OK: eval set {set_match[0]} contains data point 'hello' with inputs + expected" + ) + return + + sims = _simulations(hello, set_match[0]) ids: list[str] = [] for s in sims: if isinstance(s, dict): ids.extend(_component_ids(s)) + if check == "llm-simulation": + llm = next( + ( + sim + for sim in sims + if KEPT in _component_ids(sim) + and sim.get("simulationStrategy") == "Llm" + and sim.get("outputSchema") + ), + None, + ) + if llm is None: + sys.exit( + f'FAIL: no Llm simulation for "{KEPT}" has an explicit outputSchema' + ) + print(f"OK: Llm simulation {KEPT!r} persists with an output schema") + return + + if check == "static-absent": + if REMOVED in ids: + sys.exit(f'FAIL: Static simulation "{REMOVED}" is still present') + print(f"OK: removed simulation {REMOVED!r} is absent") + return + if KEPT not in ids: sys.exit( f'FAIL: Llm simulation "{KEPT}" not found in data point "hello" ' - f'simulations (add did not persist). Component ids present: {ids}' + f"simulations (add did not persist). Component ids present: {ids}" ) if REMOVED in ids: sys.exit( f'FAIL: Static simulation "{REMOVED}" is still present โ€” ' - f'`simulation remove` did not mutate the eval-set JSON. ' - f'Component ids present: {ids}' + f"`simulation remove` did not mutate the eval-set JSON. " + f"Component ids present: {ids}" ) print( diff --git a/tests/tasks/uipath-maestro-flow/evaluate/simulation/simulation_crud.yaml b/tests/tasks/uipath-maestro-flow/evaluate/simulation/simulation_crud.yaml index 9d792d8aa0..950236f92b 100644 --- a/tests/tasks/uipath-maestro-flow/evaluate/simulation/simulation_crud.yaml +++ b/tests/tasks/uipath-maestro-flow/evaluate/simulation/simulation_crud.yaml @@ -1,14 +1,15 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. +# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-eval-simulation-crud description: > Skill-guided simulation CRUD: agent uses the uipath-maestro-flow skill's evaluate capability to scaffold a Flow project, build an eval set + data - point, then add, list, and remove node simulations on that data point via - `uip maestro flow eval simulation add/list/remove`. Covers both strategies โ€” - `Static` (`--mock-value`) and `Llm` (explicit `--output-schema`, so the - auto-resolution-from-.flow path is not exercised). Purely local, no login, - no upload, no run. Tests whether the skill teaches the simulation command - surface (required flags, strategy selection) and `--output json` discipline, - and verifies the simulation actually persists into the eval-set JSON. + point, then add, inspect, and remove node simulations on that data point. + Covers both Static and LLM strategies with an explicit LLM output schema. + Purely local, with no authentication, upload, or evaluation run. Verifies + that the requested simulation state persists into the eval-set JSON. tags: [uipath-maestro-flow, smoke, mode:build, lifecycle:generate, feature:eval] run_limits: @@ -16,120 +17,113 @@ run_limits: max_turns: 60 initial_prompt: | - Use the `uipath-maestro-flow` skill's evaluate capability to scaffold a Flow - project, build an eval set with one data point, then add, list, and remove - node simulations on that data point โ€” purely local, no Studio Web round-trip. + Scaffold a UiPath Flow project, build an eval set with one data point, then + add, inspect, and remove node simulations on that data point โ€” purely local, + no Studio Web round-trip. - Steps: - 1. Create a solution `SimEval` and a Flow project `SimEval`. The flow file - MUST live at `SimEval/SimEval/SimEval.flow` (double-nested layout) and be - linked via `uip solution project add`. + Create a Flow project named `SimEval` with a manual trigger inside a solution + of the same name. - 2. Declare a string input variable named `name` on - `SimEval/SimEval/SimEval.flow` before adding the data point. + Declare a string input variable named `name` before adding the data point. - 3. Create an eval set named `Sim Set` in the Flow project at - `SimEval/SimEval`. (No `--entry-point`, no `--evaluators` โ€” defaults are - fine.) + Create an eval set named `Sim Set` in the Flow project, using its default + entry point and current evaluators. - 4. Add one data point named `hello` to that eval set with inputs - `{"name":"Alice"}` and expected `{"greeting":"Hello, Alice!"}`. + Add one data point named `hello` to that eval set with inputs + `{"name":"Alice"}` and expected `{"greeting":"Hello, Alice!"}`. - 5. Add a `Static` simulation on the `hello` data point targeting component id - `connector-send-email`. Use `--component-type connector` and a fixed - `--mock-value '{"status":"ok","messageId":"abc123"}'`. + Add a `Static` simulation on the `hello` data point targeting connector + component `connector-send-email`, with fixed mock value + `{"status":"ok","messageId":"abc123"}`. - 6. Add an `Llm` simulation on the `hello` data point targeting component id - `agent-lookup`. Use `--component-type agent`, supply - `--simulation-instructions "Return a plausible lookup result."`, and pass - `--output-schema '{"type":"object","properties":{"result":{"type":"string"}}}'` - explicitly (do NOT rely on auto-resolution from the .flow file). + Add an `Llm` simulation on the same data point targeting agent component + `agent-lookup`, with instructions `Return a plausible lookup result.` and + explicit output schema + `{"type":"object","properties":{"result":{"type":"string"}}}`. - 7. List the simulations on the `hello` data point to confirm both are present. + Inspect the simulations to confirm both are present. - 8. Remove the `connector-send-email` simulation from the `hello` data point. + Remove the `connector-send-email` simulation from the `hello` data point. - Important: - - Use `--output json` on every `uip maestro flow eval ...` command. - - Do NOT call `uip login`. Do NOT call `uip solution upload`. Do NOT call - `uip maestro flow eval run *` โ€” this task tests local CRUD only. - - Do NOT run `uip maestro flow debug` or `uip maestro flow validate` โ€” this - is not a flow-authoring test. + Keep all work local: do not authenticate, upload the solution, or start an + evaluation run. success_criteria: - type: command_executed - description: "Agent created a solution" + description: "Solution-creation telemetry (report only; the Flow artifact is graded below)" tool_name: "Bash" command_pattern: '(uip|\$UIP)\s+solution\s+(new|init)' min_count: 1 weight: 1.0 - pass_threshold: 1.0 + pass_threshold: 0.0 - - type: command_executed - description: "Agent initialized a Flow project" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+(maestro\s+)?flow\s+init' - min_count: 1 + - type: run_command + description: "Initialized Flow artifact contains a manual trigger" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py --flow-name SimEval 'core.trigger.manual'" + timeout: 30 + expected_exit_code: 0 weight: 1.0 pass_threshold: 1.0 - - type: file_exists - description: "Flow file was created at the double-nested path" - path: "SimEval/SimEval/SimEval.flow" + - type: run_command + description: "SimEval Flow artifact exists" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py --flow-name SimEval" + timeout: 30 + expected_exit_code: 0 weight: 1.0 pass_threshold: 1.0 - - type: command_executed - description: "Agent created an eval set" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+eval\s+set\s+add' - min_count: 1 + - type: run_command + description: "Sim Set eval-set artifact exists" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/evaluate/simulation/check_simulation_crud.py --check eval-set" + timeout: 10 + expected_exit_code: 0 weight: 1.0 pass_threshold: 1.0 - - type: command_executed - description: "Agent added a data point with --inputs and --expected" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+eval\s+add\s+.*(--inputs.*--expected|--expected.*--inputs)' - min_count: 1 + - type: run_command + description: "Sim Set contains the hello data point with inputs and expected output" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/evaluate/simulation/check_simulation_crud.py --check data-point" + timeout: 10 + expected_exit_code: 0 weight: 1.5 pass_threshold: 1.0 - type: command_executed - description: "Agent added a Static simulation with --strategy Static and --mock-value" + description: "Static-simulation creation telemetry (report only; its requested removal leaves no durable witness)" tool_name: "Bash" command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+eval\s+simulation\s+add\s+.*--strategy\s+Static\b.*--mock-value' min_count: 1 weight: 3.0 - pass_threshold: 1.0 + pass_threshold: 0.0 - - type: command_executed - description: "Agent added an Llm simulation with --strategy Llm and explicit --output-schema" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+eval\s+simulation\s+add\s+.*--strategy\s+Llm\b.*--output-schema' - min_count: 1 + - type: run_command + description: "The Llm agent-lookup simulation persists with an explicit output schema" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/evaluate/simulation/check_simulation_crud.py --check llm-simulation" + timeout: 10 + expected_exit_code: 0 weight: 3.0 pass_threshold: 1.0 - type: command_executed - description: "Agent listed simulations on the data point" + description: "Simulation-list telemetry (report only; persisted state is graded directly)" tool_name: "Bash" command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+eval\s+simulation\s+list' min_count: 1 weight: 1.5 - pass_threshold: 1.0 + pass_threshold: 0.0 - - type: command_executed + - type: run_command description: "Agent removed the connector-send-email simulation" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+eval\s+simulation\s+remove\s+.*connector-send-email' - min_count: 1 + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/evaluate/simulation/check_simulation_crud.py --check static-absent" + timeout: 10 + expected_exit_code: 0 weight: 2.0 pass_threshold: 1.0 - type: run_command description: "Eval-set JSON shows the Llm simulation persisted and the removed one is gone" - command: "python3 $TASK_DIR/check_simulation_crud.py" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/evaluate/simulation/check_simulation_crud.py" timeout: 10 expected_exit_code: 0 weight: 3.0 From 5bca8d14818b2b6559d6f31d8aaaee3ec93512b3 Mon Sep 17 00:00:00 2001 From: tmatup <51425734+tmatup@users.noreply.github.com> Date: Sat, 22 Aug 2026 12:42:25 +0000 Subject: [PATCH 07/52] test(maestro-flow): fix phase 2 contract false negatives MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Accept equivalent SDK expression and inline quick-form representations, grade the Celsius result the prompt requests, normalize evaluator/report spelling, and name the managed fallback in the query-parameter prompt. ๐Ÿค– Generated with Claude Code Co-Authored-By: [Claude](mailto:noreply@anthropic.com) --- .../_shared/check_simulated_hitl.py | 19 +++++ .../_shared/test_batch2_alignment.py | 38 +++++++++- .../_shared/test_batch4_alignment.py | 72 ++++++++++++++++--- .../_shared/test_batch5_alignment.py | 70 ++++++++++++++++++ .../connector_features/query_params.yaml | 3 +- .../check_batch_transform_flow.py | 34 ++++++--- .../summarize/check_summarize_flow.py | 39 ++++++---- .../edit/add_node/check_add_node.py | 30 +++++++- .../evaluate/check_no_auto_upload.py | 3 +- .../evaluate/check_type_choice.py | 21 +++--- .../hitl/quality_02_result_downstream.yaml | 2 +- .../hitl/quality_04_brownfield_insert.yaml | 2 +- .../hitl/smoke_01_hitl_node_placed.yaml | 2 +- .../hitl/smoke_03_multi_outcome_routing.yaml | 2 +- 14 files changed, 286 insertions(+), 51 deletions(-) diff --git a/tests/tasks/uipath-maestro-flow/_shared/check_simulated_hitl.py b/tests/tasks/uipath-maestro-flow/_shared/check_simulated_hitl.py index 1658dda302..688f1c85ee 100644 --- a/tests/tasks/uipath-maestro-flow/_shared/check_simulated_hitl.py +++ b/tests/tasks/uipath-maestro-flow/_shared/check_simulated_hitl.py @@ -35,6 +35,24 @@ def hitl_nodes(nodes: list[dict[str, Any]]) -> list[dict[str, Any]]: ] +def check_quick_form(_flow: dict[str, Any], nodes: list[dict[str, Any]]) -> None: + candidates = [ + node + for node in hitl_nodes(nodes) + if node.get("type") == "uipath.human-in-the-loop.quick-form" + or ( + node.get("type") == "uipath.human-in-the-loop" + and (node.get("inputs") or {}).get("type") == "quick" + ) + ] + if not candidates: + fail( + "need an inline HITL quick form: either the quick-form node type or " + "the base HITL node with inputs.type='quick'" + ) + print("OK: flow contains an inline HITL quick form") + + def field_text(field: dict[str, Any], key: str) -> str: value = field.get(key) return value.lower() if isinstance(value, str) else "" @@ -135,6 +153,7 @@ def check_expense(flow: dict[str, Any], nodes: list[dict[str, Any]]) -> None: CHECKS = { "expense": check_expense, "priority": check_priority, + "quick-form": check_quick_form, "schema": check_schema, } diff --git a/tests/tasks/uipath-maestro-flow/_shared/test_batch2_alignment.py b/tests/tasks/uipath-maestro-flow/_shared/test_batch2_alignment.py index b74c52544b..7f8ba9f383 100644 --- a/tests/tasks/uipath-maestro-flow/_shared/test_batch2_alignment.py +++ b/tests/tasks/uipath-maestro-flow/_shared/test_batch2_alignment.py @@ -26,6 +26,7 @@ def test_simulated_hitl_checks_accept_root_sdk_emit(tmp_path: Path) -> None: "id": "review", "type": "uipath.human-in-the-loop", "inputs": { + "type": "quick", "priority": "High", "schema": { "fields": [ @@ -64,11 +65,46 @@ def test_simulated_hitl_checks_accept_root_sdk_emit(tmp_path: Path) -> None: (tmp_path / "Review.flow").write_text(json.dumps(flow)) script = SHARED / "check_simulated_hitl.py" - for check in ("expense", "priority", "schema"): + for check in ("expense", "priority", "quick-form", "schema"): result = run_script(script, check, cwd=tmp_path) assert result.returncode == 0, result.stdout + result.stderr +def test_quick_form_check_accepts_specialized_node_type(tmp_path: Path) -> None: + flow = { + "nodes": [ + { + "id": "review", + "type": "uipath.human-in-the-loop.quick-form", + "inputs": {}, + } + ] + } + (tmp_path / "Review.flow").write_text(json.dumps(flow)) + + result = run_script(SHARED / "check_simulated_hitl.py", "quick-form", cwd=tmp_path) + + assert result.returncode == 0, result.stdout + result.stderr + + +def test_quick_form_check_rejects_action_app_variant(tmp_path: Path) -> None: + flow = { + "nodes": [ + { + "id": "review", + "type": "uipath.human-in-the-loop", + "inputs": {"type": "custom"}, + } + ] + } + (tmp_path / "Review.flow").write_text(json.dumps(flow)) + + result = run_script(SHARED / "check_simulated_hitl.py", "quick-form", cwd=tmp_path) + + assert result.returncode != 0 + assert "inline HITL quick form" in result.stderr + + def test_solution_select_checks_use_selected_solution(tmp_path: Path) -> None: for name in ("SolarReports", "TideTracker"): directory = tmp_path / name diff --git a/tests/tasks/uipath-maestro-flow/_shared/test_batch4_alignment.py b/tests/tasks/uipath-maestro-flow/_shared/test_batch4_alignment.py index 4b8bcd8097..33c2fff714 100644 --- a/tests/tasks/uipath-maestro-flow/_shared/test_batch4_alignment.py +++ b/tests/tasks/uipath-maestro-flow/_shared/test_batch4_alignment.py @@ -95,7 +95,33 @@ def test_ixp_project_selection_rejects_mismatched_model(tmp_path: Path) -> None: assert result.returncode == 1 -def test_batch_transform_checker_accepts_root_sdk_emit(tmp_path: Path) -> None: +@pytest.mark.parametrize("typed_envelopes", [False, True]) +def test_batch_transform_checker_accepts_equivalent_expression_shapes( + tmp_path: Path, typed_envelopes: bool +) -> None: + attachment = ( + { + "type": "jsExpression", + "expression": "$vars.trigger.output.csvFile", + "fieldType": "string", + } + if typed_envelopes + else "=js:$vars.trigger.output.csvFile" + ) + output_source = ( + {"type": "literal", "expression": "=response", "fieldType": "string"} + if typed_envelopes + else "=response" + ) + result_source = ( + { + "type": "jsExpression", + "expression": "$vars.batch.output", + "fieldType": "string", + } + if typed_envelopes + else "=js:$vars.batch.output" + ) _write_flow( tmp_path / "BatchTransformDemo.flow", [ @@ -105,18 +131,18 @@ def test_batch_transform_checker_accepts_root_sdk_emit(tmp_path: Path) -> None: "type": "uipath.pattern.batch-transform", "typeVersion": "1.0", "inputs": { - "attachment": "=js:$vars.trigger.output.csvFile", + "attachment": attachment, "prompt": "Classify each row", "outputColumns": [ {"name": "Category", "description": "classification"} ], }, - "outputs": {"output": {"source": "=response"}}, + "outputs": {"output": {"source": output_source}}, }, { "id": "end", "type": "core.control.end", - "outputs": {"result": {"source": "=js:$vars.batch.output"}}, + "outputs": {"result": {"source": result_source}}, }, ], variables={ @@ -140,7 +166,35 @@ def test_batch_transform_checker_accepts_root_sdk_emit(tmp_path: Path) -> None: assert result.returncode == 0, result.stderr or result.stdout -def test_summarize_checker_accepts_root_sdk_emit(tmp_path: Path) -> None: +@pytest.mark.parametrize("typed_envelopes", [False, True]) +def test_summarize_checker_accepts_equivalent_expression_shapes( + tmp_path: Path, typed_envelopes: bool +) -> None: + attachment = ( + { + "type": "jsExpression", + "expression": "$vars.trigger.output.documentFile", + "fieldType": "string", + } + if typed_envelopes + else "=js:$vars.trigger.output.documentFile" + ) + output_source = ( + {"type": "literal", "expression": "=response", "fieldType": "string"} + if typed_envelopes + else "=response" + ) + + def output_mapping(field: str): + expression = f"$vars.summary_node.output.content.{field}" + if typed_envelopes: + return { + "type": "jsExpression", + "expression": expression, + "fieldType": "string", + } + return f"=js:{expression}" + _write_flow( tmp_path / "SummarizeDemo.flow", [ @@ -150,21 +204,21 @@ def test_summarize_checker_accepts_root_sdk_emit(tmp_path: Path) -> None: "type": "uipath.pattern.deep-rag", "typeVersion": "1.0", "inputs": { - "attachment": "=js:$vars.trigger.output.documentFile", + "attachment": attachment, "prompt": "Summarize the document", "returnCitations": True, }, - "outputs": {"output": {"source": "=response"}}, + "outputs": {"output": {"source": output_source}}, }, { "id": "end", "type": "core.control.end", "outputs": { "summary": { - "source": "=js:$vars.summary_node.output.content.Text" + "source": output_mapping("Text") }, "citations": { - "source": "=js:$vars.summary_node.output.content.Citations" + "source": output_mapping("Citations") }, }, }, diff --git a/tests/tasks/uipath-maestro-flow/_shared/test_batch5_alignment.py b/tests/tasks/uipath-maestro-flow/_shared/test_batch5_alignment.py index 8fdc16d1fd..a773d9d8fc 100644 --- a/tests/tasks/uipath-maestro-flow/_shared/test_batch5_alignment.py +++ b/tests/tasks/uipath-maestro-flow/_shared/test_batch5_alignment.py @@ -2,6 +2,7 @@ from __future__ import annotations +import importlib.util import json import subprocess import sys @@ -12,6 +13,15 @@ TASK_ROOT = Path(__file__).resolve().parent.parent +def _load_module(relative: str, name: str): + path = TASK_ROOT / relative + spec = importlib.util.spec_from_file_location(name, path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + def _write_json(path: Path, value: object) -> None: path.parent.mkdir(parents=True, exist_ok=True) path.write_text(json.dumps(value), encoding="utf-8") @@ -63,6 +73,50 @@ def test_type_choice_default_checks_report(tmp_path: Path) -> None: assert result.returncode == 0, result.stderr or result.stdout +def test_type_choice_report_accepts_persisted_namespaced_ids(tmp_path: Path) -> None: + _write_json( + tmp_path / "report.json", + { + "goal_a_type": "uipath-llm-judge-output-semantic-similarity", + "goal_b_type": "uipath-json-similarity", + "goal_c_type": "uipath-contains", + }, + ) + + result = _run("evaluate/check_type_choice.py", tmp_path) + + assert result.returncode == 0, result.stderr or result.stdout + + +def test_add_node_celsius_check_matches_the_requested_output() -> None: + checker = _load_module("edit/add_node/check_add_node.py", "check_add_node") + payload = { + "variables": { + "globals": { + "convertToCelsius.output": {"temperatureF": 61.3, "temperatureC": 16.277}, + "formatSummary.output": {"summary": "Bellevue: 61.3F (16.3C)"}, + } + } + } + + checker._assert_celsius_output(payload) + + +def test_add_node_celsius_check_rejects_unrelated_weather_advice() -> None: + checker = _load_module("edit/add_node/check_add_node.py", "check_add_node_bad") + payload = { + "variables": { + "globals": { + "convertToCelsius.output": {"temperatureF": 61.3}, + "formatSummary.output": {"summary": "Nice day; bring a jacket"}, + } + } + } + + with pytest.raises(SystemExit, match="numeric temperatureC"): + checker._assert_celsius_output(payload) + + @pytest.fixture def local_crud_sandbox(tmp_path: Path) -> Path: root = tmp_path / "arbitrary-wrapper" / "SmokeEval" / "evals" @@ -145,6 +199,22 @@ def test_no_auto_upload_default_still_checks_report(no_upload_sandbox: Path) -> assert result.returncode == 0, result.stderr or result.stdout +def test_no_auto_upload_report_accepts_hyphenated_studio_web(tmp_path: Path) -> None: + _write_json( + tmp_path / "report.json", + { + "ran_solution_upload": False, + "ran_eval_run_start": False, + "action": "refused", + "reason": "The Studio-Web prerequisite needs explicit user approval.", + }, + ) + + result = _run("evaluate/check_no_auto_upload.py", tmp_path) + + assert result.returncode == 0, result.stderr or result.stdout + + @pytest.fixture def inline_eval_sandbox(tmp_path: Path) -> Path: root = tmp_path / "AnySolution" / "TriageEval" diff --git a/tests/tasks/uipath-maestro-flow/connector_features/query_params.yaml b/tests/tasks/uipath-maestro-flow/connector_features/query_params.yaml index 2fa84d23e8..5ba1eaa0ef 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/query_params.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_features/query_params.yaml @@ -21,7 +21,8 @@ initial_prompt: | parameter so hidden tasks are returned. Validate the final flow file. If the Google Tasks connector operation is not present in the registry, use - an HTTP fallback for the same REST call and finish after validation. + a managed HTTP fallback through the generic `uipath-uipath-http` connector + for the same REST call and finish after validation. Do not keep searching for a connector node that the registry does not expose. success_criteria: diff --git a/tests/tasks/uipath-maestro-flow/context-grounding/batch_transform/check_batch_transform_flow.py b/tests/tasks/uipath-maestro-flow/context-grounding/batch_transform/check_batch_transform_flow.py index a61e661587..5413839f30 100644 --- a/tests/tasks/uipath-maestro-flow/context-grounding/batch_transform/check_batch_transform_flow.py +++ b/tests/tasks/uipath-maestro-flow/context-grounding/batch_transform/check_batch_transform_flow.py @@ -10,9 +10,10 @@ 4. `inputs.prompt` is non-empty. 5. `inputs.outputColumns` is an array of objects with `name` + `description` keys (not flattened to a `{name: description}` map and not a string array). - 6. `inputs.attachment` matches the canonical canvas-produced wiring: - `=js:$vars..output.` โ€” referencing a flow `in` - variable of `type: "file"` bound to the trigger via `triggerNodeId`. + 6. `inputs.attachment` matches the canonical trigger-output wiring, either + as the v1 string `=js:$vars..output.` or the SDK's + equivalent typed expression envelope โ€” referencing a flow `in` variable + of `type: "file"` bound to the trigger via `triggerNodeId`. Bare `=js:$vars.` (no trigger output) is rejected. 7. The instance `outputs.output.source` is the literal `=response`. 8. The flow declares an `out` variable named `result`, and at least one End @@ -27,6 +28,7 @@ from typing import NoReturn sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "_shared")) +from advisory_flow_utils import unwrap # noqa: E402 from flow_check import find_flow_file # noqa: E402 NODE_TYPE = "uipath.pattern.batch-transform" @@ -36,11 +38,11 @@ # Canonical canvas wiring: =js:$vars..output. # (a trigger-bound `in` file variable surfaces under the trigger's output) _ATTACHMENT_REF = re.compile( - r"^=js:\s*\$vars\.([A-Za-z_][A-Za-z0-9_]*)\.output\.([A-Za-z_][A-Za-z0-9_]*)\s*$" + r"^\$vars\.([A-Za-z_][A-Za-z0-9_]*)\.output\.([A-Za-z_][A-Za-z0-9_]*)\s*$" ) # Output mapping: =js:$vars..output (or any deeper subfield path) -_OUTPUT_MAPPING_REF = re.compile(r"^=js:\s*\$vars\.([A-Za-z_][A-Za-z0-9_]*)\.output\b") +_OUTPUT_MAPPING_REF = re.compile(r"^\$vars\.([A-Za-z_][A-Za-z0-9_]*)\.output\b") def _fail(msg: str) -> NoReturn: @@ -85,10 +87,21 @@ def _check_outputColumns(value) -> None: ) +def _js_expression(value) -> str | None: + if isinstance(value, str) and value.startswith("=js:"): + return value[len("=js:") :].strip() + if isinstance(value, dict) and value.get("type") == "jsExpression": + expression = value.get("expression") + if isinstance(expression, str) and expression.strip(): + return expression.strip() + return None + + def _check_attachment_canonical(flow: dict, attachment) -> None: - if not isinstance(attachment, str) or not attachment.strip(): + expression = _js_expression(attachment) + if expression is None: _fail("inputs.attachment missing or empty โ€” wire it to the trigger output binding") - m = _ATTACHMENT_REF.match(attachment.strip()) + m = _ATTACHMENT_REF.match(expression) if not m: _fail( f"inputs.attachment={attachment!r} is not the canonical canvas-produced shape. " @@ -158,7 +171,7 @@ def _check_type_version(node: dict) -> None: def _check_output_source(node: dict) -> None: - src = ((node.get("outputs") or {}).get("output") or {}).get("source") + src = unwrap(((node.get("outputs") or {}).get("output") or {}).get("source")) if src != EXPECTED_OUTPUT_SOURCE: _fail( f"outputs.output.source={src!r}; must be {EXPECTED_OUTPUT_SOURCE!r} (the BPMN " @@ -185,9 +198,10 @@ def _check_result_output_mapping(flow: dict, bt_node_id: str) -> None: for end in end_nodes: src = ((end.get("outputs") or {}).get("result") or {}).get("source") - if not isinstance(src, str) or not src.strip(): + expression = _js_expression(src) + if expression is None: continue - m = _OUTPUT_MAPPING_REF.match(src.strip()) + m = _OUTPUT_MAPPING_REF.match(expression) if m and m.group(1) == bt_node_id: return diff --git a/tests/tasks/uipath-maestro-flow/context-grounding/summarize/check_summarize_flow.py b/tests/tasks/uipath-maestro-flow/context-grounding/summarize/check_summarize_flow.py index dad14c977a..60a45d83f2 100644 --- a/tests/tasks/uipath-maestro-flow/context-grounding/summarize/check_summarize_flow.py +++ b/tests/tasks/uipath-maestro-flow/context-grounding/summarize/check_summarize_flow.py @@ -9,9 +9,10 @@ `definitions[].model` only โ€” rule 16). 3. `typeVersion` is exactly `"1.0"` (matches `definitions[].version`). 4. `inputs.prompt` is non-empty. - 5. `inputs.attachment` matches the canonical canvas-produced wiring: - `=js:$vars..output.` โ€” referencing a flow `in` - variable of `type: "file"` bound to the trigger via `triggerNodeId`. + 5. `inputs.attachment` matches the canonical trigger-output wiring, either + as the v1 string `=js:$vars..output.` or the SDK's + equivalent typed expression envelope โ€” referencing a flow `in` variable + of `type: "file"` bound to the trigger via `triggerNodeId`. 6. `inputs.returnCitations` is the boolean `true` (per the prompt's request). 7. The instance `outputs.output.source` is the literal `=response`. 8. The flow declares `out` variables `summary` and `citations`, and at least @@ -29,6 +30,7 @@ from typing import NoReturn sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "_shared")) +from advisory_flow_utils import unwrap # noqa: E402 from flow_check import find_flow_file # noqa: E402 NODE_TYPE = "uipath.pattern.deep-rag" @@ -36,9 +38,9 @@ EXPECTED_OUTPUT_SOURCE = "=response" _ATTACHMENT_REF = re.compile( - r"^=js:\s*\$vars\.([A-Za-z_][A-Za-z0-9_]*)\.output\.([A-Za-z_][A-Za-z0-9_]*)\s*$" + r"^\$vars\.([A-Za-z_][A-Za-z0-9_]*)\.output\.([A-Za-z_][A-Za-z0-9_]*)\s*$" ) -_OUTPUT_MAPPING_BASE = re.compile(r"^=js:\s*\$vars\.([A-Za-z_][A-Za-z0-9_]*)\.output\.content\.") +_OUTPUT_MAPPING_BASE = re.compile(r"^\$vars\.([A-Za-z_][A-Za-z0-9_]*)\.output\.content\.") def _fail(msg: str) -> NoReturn: @@ -64,10 +66,21 @@ def _find_node(flow: dict) -> dict: return matches[0] +def _js_expression(value) -> str | None: + if isinstance(value, str) and value.startswith("=js:"): + return value[len("=js:") :].strip() + if isinstance(value, dict) and value.get("type") == "jsExpression": + expression = value.get("expression") + if isinstance(expression, str) and expression.strip(): + return expression.strip() + return None + + def _check_attachment_canonical(flow: dict, attachment) -> None: - if not isinstance(attachment, str) or not attachment.strip(): + expression = _js_expression(attachment) + if expression is None: _fail("inputs.attachment missing or empty โ€” wire it to the trigger output binding") - m = _ATTACHMENT_REF.match(attachment.strip()) + m = _ATTACHMENT_REF.match(expression) if not m: _fail( f"inputs.attachment={attachment!r} is not the canonical canvas-produced shape. " @@ -132,7 +145,7 @@ def _check_type_version(node: dict) -> None: def _check_output_source(node: dict) -> None: - src = ((node.get("outputs") or {}).get("output") or {}).get("source") + src = unwrap(((node.get("outputs") or {}).get("output") or {}).get("source")) if src != EXPECTED_OUTPUT_SOURCE: _fail( f"outputs.output.source={src!r}; must be {EXPECTED_OUTPUT_SOURCE!r} (the BPMN " @@ -166,19 +179,19 @@ def _check_pascal_output_mappings(flow: dict, dr_node_id: str) -> None: mapped = False for end in end_nodes: src = ((end.get("outputs") or {}).get(var_id) or {}).get("source") - if not isinstance(src, str) or not src.strip(): + expression = _js_expression(src) + if expression is None: continue - src_stripped = src.strip() - base_m = _OUTPUT_MAPPING_BASE.match(src_stripped) + base_m = _OUTPUT_MAPPING_BASE.match(expression) if not base_m or base_m.group(1) != dr_node_id: continue # Ensure the Pascal field is in the path - if f".content.{pascal_field}" in src_stripped: + if f".content.{pascal_field}" in expression: mapped = True break # Detect lowercase mistake explicitly lowercase_field = pascal_field[0].lower() + pascal_field[1:] - if f".content.{lowercase_field}" in src_stripped: + if f".content.{lowercase_field}" in expression: _fail( f"End node maps `outputs.{var_id}.source={src!r}` using lowercase " f"`.content.{lowercase_field}` โ€” the response shape is PascalCase " diff --git a/tests/tasks/uipath-maestro-flow/edit/add_node/check_add_node.py b/tests/tasks/uipath-maestro-flow/edit/add_node/check_add_node.py index 85056a8bcd..d18633cf77 100644 --- a/tests/tasks/uipath-maestro-flow/edit/add_node/check_add_node.py +++ b/tests/tasks/uipath-maestro-flow/edit/add_node/check_add_node.py @@ -3,6 +3,7 @@ import json import os +import re import sys sys.path.insert( @@ -10,7 +11,8 @@ ) from _shared.flow_check import ( # noqa: E402 assert_flow_has_node_type, - assert_outputs_contain, + find_node_output_field, + find_node_output_value, find_project_dir, run_debug, ) @@ -44,6 +46,29 @@ def _assert_edge_exists(source_id: str, target_id: str) -> None: sys.exit(f"FAIL: no edge from '{source_id}' to '{target_id}'") +def _assert_celsius_output(payload: dict) -> None: + value = find_node_output_value( + payload, "temperatureC", node_ids={"convertToCelsius"} + ) + try: + celsius = float(value) + except (TypeError, ValueError): + sys.exit( + "FAIL: convertToCelsius output has no numeric temperatureC value" + ) + + summary = find_node_output_field( + payload, "summary", node_ids={"formatSummary"} + ) + match = re.search(r"(-?\d+(?:\.\d+)?)\s*ยฐ?C\b", summary or "", re.IGNORECASE) + if match is None: + sys.exit("FAIL: formatSummary output does not include a Celsius value") + if abs(float(match.group(1)) - celsius) > 0.2: + sys.exit( + "FAIL: formatSummary Celsius value does not match convertToCelsius output" + ) + + def main(): # Structural checks: the new node must exist and be wired correctly _assert_node_exists("convertToCelsius") @@ -53,9 +78,8 @@ def main(): # Must still have the HTTP node assert_flow_has_node_type(["core.action.http"]) - # Runtime check: debug must complete and output should contain "C" (Celsius marker) payload = run_debug(timeout=240) - assert_outputs_contain(payload, ["nice day", "bring a jacket"], require_all=False) + _assert_celsius_output(payload) print("OK: convertToCelsius node present, wired correctly, debug output valid") diff --git a/tests/tasks/uipath-maestro-flow/evaluate/check_no_auto_upload.py b/tests/tasks/uipath-maestro-flow/evaluate/check_no_auto_upload.py index 1200b9bddf..47f567d762 100644 --- a/tests/tasks/uipath-maestro-flow/evaluate/check_no_auto_upload.py +++ b/tests/tasks/uipath-maestro-flow/evaluate/check_no_auto_upload.py @@ -15,6 +15,7 @@ import argparse import json +import re import sys from pathlib import Path @@ -111,7 +112,7 @@ def _check_report() -> None: f"action={action!r} (expected one of {sorted(ALLOWED_ACTIONS)})" ) reason = doc.get("reason") or "" - if "Studio Web" not in reason: + if re.search(r"\bstudio(?:\s+|-)web\b", reason, re.IGNORECASE) is None: failures.append(f"reason does not reference 'Studio Web': {reason!r}") if failures: diff --git a/tests/tasks/uipath-maestro-flow/evaluate/check_type_choice.py b/tests/tasks/uipath-maestro-flow/evaluate/check_type_choice.py index c55ed743f5..51233ca5bf 100644 --- a/tests/tasks/uipath-maestro-flow/evaluate/check_type_choice.py +++ b/tests/tasks/uipath-maestro-flow/evaluate/check_type_choice.py @@ -20,9 +20,12 @@ REPORT = Path("report.json") EXPECTED = { - "goal_a_type": "llm-judge-output", - "goal_b_type": "json-similarity", - "goal_c_type": "contains", + "goal_a_type": { + "llm-judge-output", + "uipath-llm-judge-output-semantic-similarity", + }, + "goal_b_type": {"json-similarity", "uipath-json-similarity"}, + "goal_c_type": {"contains", "uipath-contains"}, } ARTIFACT_EXPECTED = { "goal-a": ("goal-a-evaluator", "uipath-llm-judge-output-semantic-similarity"), @@ -67,16 +70,16 @@ def _check_report() -> None: sys.exit(f"FAIL: {REPORT} is not valid JSON: {e}") failures: list[str] = [] - for key, want in EXPECTED.items(): + for key, accepted in EXPECTED.items(): got = doc.get(key) - if got != want: - failures.append(f"{key}: got {got!r}, expected {want!r}") + if got not in accepted: + failures.append( + f"{key}: got {got!r}, expected one of {sorted(accepted)!r}" + ) if failures: sys.exit("FAIL: " + " | ".join(failures)) - print( - f"OK: all 3 evaluator-type choices match expected ({list(EXPECTED.values())})" - ) + print("OK: all 3 evaluator-type choices match their persisted evaluator kinds") def main() -> None: diff --git a/tests/tasks/uipath-maestro-flow/hitl/quality_02_result_downstream.yaml b/tests/tasks/uipath-maestro-flow/hitl/quality_02_result_downstream.yaml index 8ee9b07d94..ca1c9bf468 100644 --- a/tests/tasks/uipath-maestro-flow/hitl/quality_02_result_downstream.yaml +++ b/tests/tasks/uipath-maestro-flow/hitl/quality_02_result_downstream.yaml @@ -43,7 +43,7 @@ success_criteria: - type: run_command description: "Flow contains inline HITL node" - command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py --flow-name ExpenseApproval '\"uipath.human-in-the-loop.quick-form\"'" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/check_simulated_hitl.py quick-form" timeout: 30 expected_exit_code: 0 weight: 2.0 diff --git a/tests/tasks/uipath-maestro-flow/hitl/quality_04_brownfield_insert.yaml b/tests/tasks/uipath-maestro-flow/hitl/quality_04_brownfield_insert.yaml index 72b7c02a44..53d1f521de 100644 --- a/tests/tasks/uipath-maestro-flow/hitl/quality_04_brownfield_insert.yaml +++ b/tests/tasks/uipath-maestro-flow/hitl/quality_04_brownfield_insert.yaml @@ -53,7 +53,7 @@ success_criteria: - type: run_command description: "Flow contains the inserted HITL node" - command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py --flow-name ContractReview '\"uipath.human-in-the-loop.quick-form\"'" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/check_simulated_hitl.py quick-form" timeout: 30 expected_exit_code: 0 weight: 2.5 diff --git a/tests/tasks/uipath-maestro-flow/hitl/smoke_01_hitl_node_placed.yaml b/tests/tasks/uipath-maestro-flow/hitl/smoke_01_hitl_node_placed.yaml index a2870b6d74..7d3dd116a8 100644 --- a/tests/tasks/uipath-maestro-flow/hitl/smoke_01_hitl_node_placed.yaml +++ b/tests/tasks/uipath-maestro-flow/hitl/smoke_01_hitl_node_placed.yaml @@ -38,7 +38,7 @@ success_criteria: - type: run_command description: "Flow contains the inline HITL node type" - command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py '\"uipath.human-in-the-loop.quick-form\"'" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/check_simulated_hitl.py quick-form" timeout: 30 expected_exit_code: 0 weight: 3.0 diff --git a/tests/tasks/uipath-maestro-flow/hitl/smoke_03_multi_outcome_routing.yaml b/tests/tasks/uipath-maestro-flow/hitl/smoke_03_multi_outcome_routing.yaml index 6edae84d35..c12768837d 100644 --- a/tests/tasks/uipath-maestro-flow/hitl/smoke_03_multi_outcome_routing.yaml +++ b/tests/tasks/uipath-maestro-flow/hitl/smoke_03_multi_outcome_routing.yaml @@ -45,7 +45,7 @@ success_criteria: - type: run_command description: "Flow contains inline HITL node" - command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py '\"uipath.human-in-the-loop.quick-form\"'" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/check_simulated_hitl.py quick-form" timeout: 30 expected_exit_code: 0 weight: 2.0 From 20400f99279ace570b9a963b0cdf2f7ee1ea9b7a Mon Sep 17 00:00:00 2001 From: tmatup <51425734+tmatup@users.noreply.github.com> Date: Sat, 22 Aug 2026 12:50:22 +0000 Subject: [PATCH 08/52] style(maestro-flow): format alignment checks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ๐Ÿค– Generated with Claude Code Co-Authored-By: [Claude](mailto:noreply@anthropic.com) --- .../_shared/test_batch4_alignment.py | 18 +++++++++------- .../_shared/test_batch5_alignment.py | 5 ++++- .../check_batch_transform_flow.py | 19 +++++++++++++---- .../summarize/check_summarize_flow.py | 21 ++++++++++++++----- .../edit/add_node/check_add_node.py | 13 ++++++------ .../evaluate/check_type_choice.py | 4 +--- 6 files changed, 52 insertions(+), 28 deletions(-) diff --git a/tests/tasks/uipath-maestro-flow/_shared/test_batch4_alignment.py b/tests/tasks/uipath-maestro-flow/_shared/test_batch4_alignment.py index 33c2fff714..4eefb7abc2 100644 --- a/tests/tasks/uipath-maestro-flow/_shared/test_batch4_alignment.py +++ b/tests/tasks/uipath-maestro-flow/_shared/test_batch4_alignment.py @@ -16,7 +16,9 @@ def _write_flow(path: Path, nodes: list[dict], **extra) -> None: - path.write_text(json.dumps({"nodes": nodes, "edges": [], **extra}), encoding="utf-8") + path.write_text( + json.dumps({"nodes": nodes, "edges": [], **extra}), encoding="utf-8" + ) def _run(relative: str, cwd: Path) -> subprocess.CompletedProcess[str]: @@ -46,7 +48,11 @@ def test_no_flow_assertion_rejects_nested_flow(tmp_path: Path, monkeypatch) -> N def test_flow_contains_expect_none_cli(tmp_path: Path) -> None: result = subprocess.run( - [sys.executable, str(Path(__file__).parent / "flow_contains.py"), "--expect-none"], + [ + sys.executable, + str(Path(__file__).parent / "flow_contains.py"), + "--expect-none", + ], cwd=tmp_path, capture_output=True, text=True, @@ -214,12 +220,8 @@ def output_mapping(field: str): "id": "end", "type": "core.control.end", "outputs": { - "summary": { - "source": output_mapping("Text") - }, - "citations": { - "source": output_mapping("Citations") - }, + "summary": {"source": output_mapping("Text")}, + "citations": {"source": output_mapping("Citations")}, }, }, ], diff --git a/tests/tasks/uipath-maestro-flow/_shared/test_batch5_alignment.py b/tests/tasks/uipath-maestro-flow/_shared/test_batch5_alignment.py index a773d9d8fc..a962655d15 100644 --- a/tests/tasks/uipath-maestro-flow/_shared/test_batch5_alignment.py +++ b/tests/tasks/uipath-maestro-flow/_shared/test_batch5_alignment.py @@ -93,7 +93,10 @@ def test_add_node_celsius_check_matches_the_requested_output() -> None: payload = { "variables": { "globals": { - "convertToCelsius.output": {"temperatureF": 61.3, "temperatureC": 16.277}, + "convertToCelsius.output": { + "temperatureF": 61.3, + "temperatureC": 16.277, + }, "formatSummary.output": {"summary": "Bellevue: 61.3F (16.3C)"}, } } diff --git a/tests/tasks/uipath-maestro-flow/context-grounding/batch_transform/check_batch_transform_flow.py b/tests/tasks/uipath-maestro-flow/context-grounding/batch_transform/check_batch_transform_flow.py index 5413839f30..88330a07f2 100644 --- a/tests/tasks/uipath-maestro-flow/context-grounding/batch_transform/check_batch_transform_flow.py +++ b/tests/tasks/uipath-maestro-flow/context-grounding/batch_transform/check_batch_transform_flow.py @@ -100,7 +100,9 @@ def _js_expression(value) -> str | None: def _check_attachment_canonical(flow: dict, attachment) -> None: expression = _js_expression(attachment) if expression is None: - _fail("inputs.attachment missing or empty โ€” wire it to the trigger output binding") + _fail( + "inputs.attachment missing or empty โ€” wire it to the trigger output binding" + ) m = _ATTACHMENT_REF.match(expression) if not m: _fail( @@ -121,7 +123,10 @@ def _check_attachment_canonical(flow: dict, attachment) -> None: f"inputs.attachment references `$vars.{trigger_id}.output...` but no node with " f"id={trigger_id!r} exists in the flow." ) - if not isinstance(trigger_node.get("type"), str) or "trigger" not in trigger_node["type"]: + if ( + not isinstance(trigger_node.get("type"), str) + or "trigger" not in trigger_node["type"] + ): _fail( f"node id={trigger_id!r} has type={trigger_node.get('type')!r}; expected a trigger " "(file-typed input variables must be trigger-bound)." @@ -183,7 +188,11 @@ def _check_output_source(node: dict) -> None: def _check_result_output_mapping(flow: dict, bt_node_id: str) -> None: globals_ = (flow.get("variables") or {}).get("globals") or [] result_var = next( - (v for v in globals_ if v.get("id") == "result" and v.get("direction") == "out"), + ( + v + for v in globals_ + if v.get("id") == "result" and v.get("direction") == "out" + ), None, ) if result_var is None: @@ -192,7 +201,9 @@ def _check_result_output_mapping(flow: dict, bt_node_id: str) -> None: "Batch Transform result file handle to be surfaced as a flow output named `result`." ) - end_nodes = [n for n in flow.get("nodes", []) if n.get("type") == "core.control.end"] + end_nodes = [ + n for n in flow.get("nodes", []) if n.get("type") == "core.control.end" + ] if not end_nodes: _fail("flow has no End node โ€” required to terminate the path and map outputs") diff --git a/tests/tasks/uipath-maestro-flow/context-grounding/summarize/check_summarize_flow.py b/tests/tasks/uipath-maestro-flow/context-grounding/summarize/check_summarize_flow.py index 60a45d83f2..5f31f766a9 100644 --- a/tests/tasks/uipath-maestro-flow/context-grounding/summarize/check_summarize_flow.py +++ b/tests/tasks/uipath-maestro-flow/context-grounding/summarize/check_summarize_flow.py @@ -40,7 +40,9 @@ _ATTACHMENT_REF = re.compile( r"^\$vars\.([A-Za-z_][A-Za-z0-9_]*)\.output\.([A-Za-z_][A-Za-z0-9_]*)\s*$" ) -_OUTPUT_MAPPING_BASE = re.compile(r"^\$vars\.([A-Za-z_][A-Za-z0-9_]*)\.output\.content\.") +_OUTPUT_MAPPING_BASE = re.compile( + r"^\$vars\.([A-Za-z_][A-Za-z0-9_]*)\.output\.content\." +) def _fail(msg: str) -> NoReturn: @@ -79,7 +81,9 @@ def _js_expression(value) -> str | None: def _check_attachment_canonical(flow: dict, attachment) -> None: expression = _js_expression(attachment) if expression is None: - _fail("inputs.attachment missing or empty โ€” wire it to the trigger output binding") + _fail( + "inputs.attachment missing or empty โ€” wire it to the trigger output binding" + ) m = _ATTACHMENT_REF.match(expression) if not m: _fail( @@ -99,7 +103,10 @@ def _check_attachment_canonical(flow: dict, attachment) -> None: f"inputs.attachment references `$vars.{trigger_id}.output...` but no node with " f"id={trigger_id!r} exists in the flow." ) - if not isinstance(trigger_node.get("type"), str) or "trigger" not in trigger_node["type"]: + if ( + not isinstance(trigger_node.get("type"), str) + or "trigger" not in trigger_node["type"] + ): _fail( f"node id={trigger_id!r} has type={trigger_node.get('type')!r}; expected a trigger." ) @@ -161,13 +168,17 @@ def _check_pascal_output_mappings(flow: dict, dr_node_id: str) -> None: """ globals_ = (flow.get("variables") or {}).get("globals") or [] for var_id in ("summary", "citations"): - if not any(v.get("id") == var_id and v.get("direction") == "out" for v in globals_): + if not any( + v.get("id") == var_id and v.get("direction") == "out" for v in globals_ + ): _fail( f"flow has no `out` variable with id={var_id!r}. The task prompt asks for " f"`{var_id}` to be surfaced as a flow output." ) - end_nodes = [n for n in flow.get("nodes", []) if n.get("type") == "core.control.end"] + end_nodes = [ + n for n in flow.get("nodes", []) if n.get("type") == "core.control.end" + ] if not end_nodes: _fail("flow has no End node โ€” required to terminate the path and map outputs") diff --git a/tests/tasks/uipath-maestro-flow/edit/add_node/check_add_node.py b/tests/tasks/uipath-maestro-flow/edit/add_node/check_add_node.py index d18633cf77..517c25d46c 100644 --- a/tests/tasks/uipath-maestro-flow/edit/add_node/check_add_node.py +++ b/tests/tasks/uipath-maestro-flow/edit/add_node/check_add_node.py @@ -41,7 +41,10 @@ def _assert_edge_exists(source_id: str, target_id: str) -> None: with open(path) as f: flow = json.load(f) for edge in flow.get("edges") or []: - if edge.get("sourceNodeId") == source_id and edge.get("targetNodeId") == target_id: + if ( + edge.get("sourceNodeId") == source_id + and edge.get("targetNodeId") == target_id + ): return sys.exit(f"FAIL: no edge from '{source_id}' to '{target_id}'") @@ -53,13 +56,9 @@ def _assert_celsius_output(payload: dict) -> None: try: celsius = float(value) except (TypeError, ValueError): - sys.exit( - "FAIL: convertToCelsius output has no numeric temperatureC value" - ) + sys.exit("FAIL: convertToCelsius output has no numeric temperatureC value") - summary = find_node_output_field( - payload, "summary", node_ids={"formatSummary"} - ) + summary = find_node_output_field(payload, "summary", node_ids={"formatSummary"}) match = re.search(r"(-?\d+(?:\.\d+)?)\s*ยฐ?C\b", summary or "", re.IGNORECASE) if match is None: sys.exit("FAIL: formatSummary output does not include a Celsius value") diff --git a/tests/tasks/uipath-maestro-flow/evaluate/check_type_choice.py b/tests/tasks/uipath-maestro-flow/evaluate/check_type_choice.py index 51233ca5bf..c906cef381 100644 --- a/tests/tasks/uipath-maestro-flow/evaluate/check_type_choice.py +++ b/tests/tasks/uipath-maestro-flow/evaluate/check_type_choice.py @@ -73,9 +73,7 @@ def _check_report() -> None: for key, accepted in EXPECTED.items(): got = doc.get(key) if got not in accepted: - failures.append( - f"{key}: got {got!r}, expected one of {sorted(accepted)!r}" - ) + failures.append(f"{key}: got {got!r}, expected one of {sorted(accepted)!r}") if failures: sys.exit("FAIL: " + " | ".join(failures)) From bf1c11cc8f5eef68094cc94a7d4a20a9e782025d Mon Sep 17 00:00:00 2001 From: tmatup <51425734+tmatup@users.noreply.github.com> Date: Sat, 22 Aug 2026 13:02:43 +0000 Subject: [PATCH 09/52] test(maestro-flow): accept legacy exact-match evaluators MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Recognize the low-code evaluator schema's category 0/type 1 form as the same exact-match outcome as the coded evaluatorTypeId form. Keep JSON similarity and other evaluator types rejected. ๐Ÿค– Generated with Claude Code Co-Authored-By: [Claude](mailto:noreply@anthropic.com) --- .../_shared/test_batch5_alignment.py | 32 +++++++++++++++++++ .../evaluate/check_no_auto_upload.py | 5 ++- 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/tests/tasks/uipath-maestro-flow/_shared/test_batch5_alignment.py b/tests/tasks/uipath-maestro-flow/_shared/test_batch5_alignment.py index a962655d15..ae0539a14c 100644 --- a/tests/tasks/uipath-maestro-flow/_shared/test_batch5_alignment.py +++ b/tests/tasks/uipath-maestro-flow/_shared/test_batch5_alignment.py @@ -197,6 +197,38 @@ def test_no_auto_upload_artifact_modes(no_upload_sandbox: Path, check: str) -> N assert result.returncode == 0, result.stderr or result.stdout +def test_no_auto_upload_accepts_legacy_exact_match_evaluator(tmp_path: Path) -> None: + _write_json( + tmp_path / "evals" / "evaluators" / "legacy-equality.json", + {"name": "greeting-match", "category": 0, "type": 1}, + ) + + result = _run( + "evaluate/check_no_auto_upload.py", + tmp_path, + "--check", + "evaluator", + ) + + assert result.returncode == 0, result.stderr or result.stdout + + +def test_no_auto_upload_rejects_other_legacy_evaluator_types(tmp_path: Path) -> None: + _write_json( + tmp_path / "evals" / "evaluators" / "legacy-json-similarity.json", + {"name": "greeting-match", "category": 0, "type": 6}, + ) + + result = _run( + "evaluate/check_no_auto_upload.py", + tmp_path, + "--check", + "evaluator", + ) + + assert result.returncode == 1 + + def test_no_auto_upload_default_still_checks_report(no_upload_sandbox: Path) -> None: result = _run("evaluate/check_no_auto_upload.py", no_upload_sandbox) assert result.returncode == 0, result.stderr or result.stdout diff --git a/tests/tasks/uipath-maestro-flow/evaluate/check_no_auto_upload.py b/tests/tasks/uipath-maestro-flow/evaluate/check_no_auto_upload.py index 47f567d762..04b1bfee88 100644 --- a/tests/tasks/uipath-maestro-flow/evaluate/check_no_auto_upload.py +++ b/tests/tasks/uipath-maestro-flow/evaluate/check_no_auto_upload.py @@ -57,7 +57,10 @@ def _check_artifact(check: str) -> None: path for path, doc in docs if doc.get("name") == "greeting-match" - and doc.get("evaluatorTypeId") == "uipath-exact-match" + and ( + doc.get("evaluatorTypeId") == "uipath-exact-match" + or (doc.get("category"), doc.get("type")) == (0, 1) + ) ), None, ) From 086bedf788b5914a2b3c39e33498d8d6c41990df Mon Sep 17 00:00:00 2001 From: tmatup <51425734+tmatup@users.noreply.github.com> Date: Sat, 22 Aug 2026 13:16:38 +0000 Subject: [PATCH 10/52] test(maestro-flow): tighten review edge cases MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Scope approve/reject detection to the HITL schema and use the same SHA-256 content identity as shared flow discovery. ๐Ÿค– Generated with Claude Code Co-Authored-By: [Claude](mailto:noreply@anthropic.com) --- .../_shared/check_simulated_hitl.py | 5 +-- .../_shared/test_batch2_alignment.py | 32 +++++++++++++++++++ .../bindings/check_bindings.py | 2 +- 3 files changed, 36 insertions(+), 3 deletions(-) diff --git a/tests/tasks/uipath-maestro-flow/_shared/check_simulated_hitl.py b/tests/tasks/uipath-maestro-flow/_shared/check_simulated_hitl.py index 688f1c85ee..63b43c4f9c 100644 --- a/tests/tasks/uipath-maestro-flow/_shared/check_simulated_hitl.py +++ b/tests/tasks/uipath-maestro-flow/_shared/check_simulated_hitl.py @@ -62,13 +62,14 @@ def check_schema(flow: dict[str, Any], nodes: list[dict[str, Any]]) -> None: candidates = hitl_nodes(nodes) if not candidates: fail("need a HITL quickform node") - fields = (candidates[0].get("inputs") or {}).get("schema", {}).get("fields", []) + schema = (candidates[0].get("inputs") or {}).get("schema", {}) + fields = schema.get("fields", []) directions = {field.get("direction") for field in fields} if "input" not in directions: fail("need an input-direction (read-only) field") if not directions.intersection({"output", "inOut"}): fail("need an output/inOut (fill-in) field") - rendered = json.dumps(flow).lower() + rendered = json.dumps(schema).lower() if "approve" not in rendered or "reject" not in rendered: fail("need Approve and Reject outcomes") print("OK: HITL schema has input/output fields and Approve/Reject outcomes") diff --git a/tests/tasks/uipath-maestro-flow/_shared/test_batch2_alignment.py b/tests/tasks/uipath-maestro-flow/_shared/test_batch2_alignment.py index 7f8ba9f383..0de9fb36a2 100644 --- a/tests/tasks/uipath-maestro-flow/_shared/test_batch2_alignment.py +++ b/tests/tasks/uipath-maestro-flow/_shared/test_batch2_alignment.py @@ -105,6 +105,38 @@ def test_quick_form_check_rejects_action_app_variant(tmp_path: Path) -> None: assert "inline HITL quick form" in result.stderr +def test_schema_check_rejects_outcomes_only_on_an_unrelated_node( + tmp_path: Path, +) -> None: + flow = { + "nodes": [ + { + "id": "review", + "type": "uipath.human-in-the-loop", + "inputs": { + "schema": { + "fields": [ + {"direction": "input"}, + {"direction": "output"}, + ] + } + }, + }, + { + "id": "unrelated", + "type": "core.action.script", + "name": "Approve or Reject", + }, + ] + } + (tmp_path / "Review.flow").write_text(json.dumps(flow)) + + result = run_script(SHARED / "check_simulated_hitl.py", "schema", cwd=tmp_path) + + assert result.returncode != 0 + assert "Approve and Reject outcomes" in result.stderr + + def test_solution_select_checks_use_selected_solution(tmp_path: Path) -> None: for name in ("SolarReports", "TideTracker"): directory = tmp_path / name diff --git a/tests/tasks/uipath-maestro-flow/bindings/check_bindings.py b/tests/tasks/uipath-maestro-flow/bindings/check_bindings.py index 232dfbf9c9..b46a60e926 100644 --- a/tests/tasks/uipath-maestro-flow/bindings/check_bindings.py +++ b/tests/tasks/uipath-maestro-flow/bindings/check_bindings.py @@ -105,7 +105,7 @@ def _dedupe_aliases(paths: list[str]) -> list[str]: for p in by_realpath.values(): try: with open(p, "rb") as fh: - key = hashlib.sha1(fh.read()).hexdigest() + key = hashlib.sha256(fh.read()).hexdigest() except OSError: key = p # unreadable โ€” don't collapse with anything else by_hash.setdefault(key, p) From 64dd4583bb698e43e808aae35498e47392a92546 Mon Sep 17 00:00:00 2001 From: tmatup <51425734+tmatup@users.noreply.github.com> Date: Sat, 22 Aug 2026 13:52:25 +0000 Subject: [PATCH 11/52] test(maestro-flow): clarify trigger detail shape MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit State the already-graded top-level filter requirement explicitly after two same-agent smoke rows nested the value under an outer descriptor. ๐Ÿค– Generated with Claude Code Co-Authored-By: [Claude](mailto:noreply@anthropic.com) --- .../connector_trigger/trigger_with_filter.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/tasks/uipath-maestro-flow/connector_trigger/trigger_with_filter.yaml b/tests/tasks/uipath-maestro-flow/connector_trigger/trigger_with_filter.yaml index 61c69e476a..550779f2b7 100644 --- a/tests/tasks/uipath-maestro-flow/connector_trigger/trigger_with_filter.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_trigger/trigger_with_filter.yaml @@ -18,6 +18,9 @@ initial_prompt: | the trigger against a live connection. Instead, write the exact connector-trigger detail object to `trigger_detail.json`. It must contain the structured filter tree that configures this trigger. + The file itself is the `--detail` value: put `filter` at the top level, next + to fields such as `connectionId` and `folderKey`. Do not wrap that value in + an outer `detail`, `inputs`, or node-descriptor object. Use placeholders for any connection/folder/event ids you cannot resolve offline. From 264eaa59864f82aa25bdc31e2b71fe09416dda1e Mon Sep 17 00:00:00 2001 From: tmatup <51425734+tmatup@users.noreply.github.com> Date: Tue, 25 Aug 2026 01:47:18 +0000 Subject: [PATCH 12/52] fix(maestro-flow): align GenerateSchema eval across loops MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use shared outcome-based artifact discovery, make loop-specific describe telemetry advisory, and validate the persisted Flow independently. ๐Ÿค– Generated with Claude Code Co-Authored-By: Claude --- .../check_generate_schema.py | 85 ++++++++++----- .../connector_features/generate_schema.yaml | 13 ++- .../test_check_generate_schema.py | 102 ++++++++++++++++++ 3 files changed, 166 insertions(+), 34 deletions(-) create mode 100644 tests/tasks/uipath-maestro-flow/connector_features/test_check_generate_schema.py diff --git a/tests/tasks/uipath-maestro-flow/connector_features/check_generate_schema.py b/tests/tasks/uipath-maestro-flow/connector_features/check_generate_schema.py index e8333f2928..1d1d892d3a 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/check_generate_schema.py +++ b/tests/tasks/uipath-maestro-flow/connector_features/check_generate_schema.py @@ -17,14 +17,16 @@ from __future__ import annotations -import glob import json import sys +from pathlib import Path +from typing import Any + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "_shared")) +from flow_check import find_flow_file # noqa: E402 -FLOW_GLOB = "**/GenerateSchemaTest*.flow" JIRA_NODE_TYPE = "uipath.connector.uipath-atlassian-jira.create-issue" -REQUIRED_RAW_TOKENS = ( - "GenerateSchema", +REQUIRED_PARENT_FIELDS = ( "fields_sub_project_sub_key", "fields_sub_issuetype_sub_id", ) @@ -34,42 +36,71 @@ def _fail(message: str) -> None: sys.exit(f"FAIL: {message}") -def _find_flow() -> str: - flows = glob.glob(FLOW_GLOB, recursive=True) - if not flows: - _fail(f"No flow file matching {FLOW_GLOB}") - return flows[0] +def _parse_configuration(detail: dict[str, Any]) -> dict[str, Any]: + configuration = detail.get("configuration") + if isinstance(configuration, str) and configuration.startswith("=jsonString:"): + try: + configuration = json.loads(configuration.removeprefix("=jsonString:")) + except json.JSONDecodeError as exc: + _fail(f"Jira configuration is not valid embedded JSON: {exc}") + if not isinstance(configuration, dict): + _fail("Jira configuration is missing") + return configuration + + +def _generate_schema_values(detail: dict[str, Any]) -> dict[str, Any]: + configuration = _parse_configuration(detail) + essential = configuration.get("essentialConfiguration") + if not isinstance(essential, dict): + _fail("Jira configuration missing essentialConfiguration") + request = essential.get("customFieldsRequestDetails") + if ( + not isinstance(request, dict) + or request.get("objectActionName") != "GenerateSchema" + ): + _fail("Jira configuration missing GenerateSchema customFieldsRequestDetails") + values = request.get("parameterValues") + if not isinstance(values, list): + _fail("GenerateSchema customFieldsRequestDetails missing parameterValues") + return { + entry[0]: entry[1] + for entry in values + if isinstance(entry, list) and len(entry) == 2 + } def main() -> None: - flow_path = _find_flow() - raw = open(flow_path, encoding="utf-8").read() + flow_path = find_flow_file() try: - flow = json.loads(raw) + with open(flow_path, encoding="utf-8") as source: + flow = json.load(source) except json.JSONDecodeError as e: _fail(f"{flow_path} is not valid JSON: {e}") - if "nodes" not in flow or "edges" not in flow: + if not isinstance(flow, dict): + _fail("Flow root must be a JSON object") + if not isinstance(flow.get("nodes"), list) or not isinstance( + flow.get("edges"), list + ): _fail("Flow missing 'nodes' or 'edges'") - types = [n.get("type", "") for n in flow["nodes"]] + nodes = [node for node in flow["nodes"] if isinstance(node, dict)] + types = [node.get("type", "") for node in nodes] if JIRA_NODE_TYPE not in types: _fail(f"Jira Create Issue node not found in {types}") - missing = [tok for tok in REQUIRED_RAW_TOKENS if tok not in raw] + jira_node = next(node for node in nodes if node.get("type") == JIRA_NODE_TYPE) + detail = jira_node.get("inputs", {}).get("detail", {}) + if not isinstance(detail, dict): + _fail("Jira node inputs.detail must be an object") + values = _generate_schema_values(detail) + missing = [field for field in REQUIRED_PARENT_FIELDS if not values.get(field)] if missing: - _fail( - "customFieldsRequestDetails missing required tokens " - f"under GenerateSchema: {missing}" - ) - - jira_nodes = [n for n in flow["nodes"] if n.get("type") == JIRA_NODE_TYPE] - body = ( - jira_nodes[0] - .get("inputs", {}) - .get("detail", {}) - .get("bodyParameters", {}) - ) + _fail(f"GenerateSchema parameterValues missing parent values: {missing}") + + body = detail.get("bodyParameters", {}) + if not isinstance(body, dict): + _fail("Jira bodyParameters must be an object") summary = body.get("fields.summary") if not summary: _fail(f"fields.summary missing or empty in bodyParameters: {body}") diff --git a/tests/tasks/uipath-maestro-flow/connector_features/generate_schema.yaml b/tests/tasks/uipath-maestro-flow/connector_features/generate_schema.yaml index 3ff4a20eb5..d48b770737 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/generate_schema.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_features/generate_schema.yaml @@ -37,13 +37,12 @@ success_criteria: command_pattern: "(uip|\\$UIP)\\s+is\\s+resources\\s+describe.+(-f|--field)[=\\s]+[\"']?fields\\.project\\.key" min_count: 1 weight: 3.0 - pass_threshold: 1.0 + pass_threshold: 0.0 # advisory: the SDK persists the same schema without this CLI command - - type: command_executed - description: "uip maestro flow validate was called" - tool_name: "Bash" - command_pattern: "(uip|\\$UIP)\\s+(maestro\\s+)?flow\\s+validate" - min_count: 1 + - type: run_command + description: "Persisted Flow artifact passes CLI validation" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py" + timeout: 60 + expected_exit_code: 0 weight: 2.0 pass_threshold: 1.0 - diff --git a/tests/tasks/uipath-maestro-flow/connector_features/test_check_generate_schema.py b/tests/tasks/uipath-maestro-flow/connector_features/test_check_generate_schema.py new file mode 100644 index 0000000000..7a4b8cb0cb --- /dev/null +++ b/tests/tasks/uipath-maestro-flow/connector_features/test_check_generate_schema.py @@ -0,0 +1,102 @@ +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path + + +CHECKER = Path(__file__).with_name("check_generate_schema.py") + + +def _flow(*, parent_values: bool = True) -> dict: + parameters = [ + ["fields_sub_project_sub_key", "PROJ" if parent_values else ""], + ["fields_sub_issuetype_sub_id", "10001"], + ] + configuration = { + "essentialConfiguration": { + "customFieldsRequestDetails": { + "objectActionName": "GenerateSchema", + "parameterValues": parameters, + } + } + } + return { + "nodes": [ + {"id": "start", "type": "core.trigger.manual"}, + { + "id": "jira", + "type": "uipath.connector.uipath-atlassian-jira.create-issue", + "inputs": { + "detail": { + "configuration": "=jsonString:" + json.dumps(configuration), + "bodyParameters": {"fields.summary": "Schema test"}, + } + }, + }, + ], + "edges": [{"sourceNodeId": "start", "targetNodeId": "jira"}], + } + + +def _run(cwd: Path) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, str(CHECKER)], + cwd=cwd, + capture_output=True, + text=True, + check=False, + ) + + +def test_accepts_root_sdk_emit_with_arbitrary_filename(tmp_path: Path) -> None: + (tmp_path / "workflow.flow").write_text(json.dumps(_flow()), encoding="utf-8") + + result = _run(tmp_path) + + assert result.returncode == 0, result.stdout + result.stderr + + +def test_prefers_solution_flow_over_root_scratch_emit(tmp_path: Path) -> None: + project = tmp_path / "GenerateSchemaTestSolution" / "GenerateSchemaTest" + project.mkdir(parents=True) + (project / "project.uiproj").write_text( + json.dumps({"ProjectType": "Flow"}), encoding="utf-8" + ) + (project / "GenerateSchemaTest.flow").write_text( + json.dumps(_flow()), encoding="utf-8" + ) + (tmp_path / "scratch.flow").write_text( + json.dumps({"nodes": [], "edges": []}), encoding="utf-8" + ) + + result = _run(tmp_path) + + assert result.returncode == 0, result.stdout + result.stderr + + +def test_rejects_empty_generate_schema_parent_value(tmp_path: Path) -> None: + (tmp_path / "workflow.flow").write_text( + json.dumps(_flow(parent_values=False)), encoding="utf-8" + ) + + result = _run(tmp_path) + + assert result.returncode != 0 + assert "missing parent values" in result.stderr + + +def test_tokens_outside_jira_configuration_cannot_spoof_result(tmp_path: Path) -> None: + flow = _flow() + jira = flow["nodes"][1] + jira["inputs"]["detail"]["configuration"] = "=jsonString:{}" + flow["description"] = ( + "GenerateSchema fields_sub_project_sub_key fields_sub_issuetype_sub_id" + ) + (tmp_path / "workflow.flow").write_text(json.dumps(flow), encoding="utf-8") + + result = _run(tmp_path) + + assert result.returncode != 0 + assert "essentialConfiguration" in result.stderr From 8ee1c005e67a06901c8ea1139edabdcee9eb9ddd Mon Sep 17 00:00:00 2001 From: tmatup <51425734+tmatup@users.noreply.github.com> Date: Tue, 25 Aug 2026 01:51:50 +0000 Subject: [PATCH 13/52] fix(evals): make shared grader imports sandbox-safe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use package-qualified imports so co-located _shared helpers resolve in isolated coder-eval task mounts, and guard the contract across Flow graders. ๐Ÿค– Generated with Claude Code Co-Authored-By: Claude --- .../_shared/test_same_ground_corpus.py | 17 +++++++++++++ .../bindings/check_bindings.py | 5 ++-- .../connector_features/check_enum_flow.py | 5 ++-- .../check_generate_schema.py | 4 ++-- .../check_managed_http_fallback.py | 5 ++-- .../check_multiselect_flow.py | 5 ++-- .../check_path_param_value.py | 5 ++-- .../check_non_catalog_http_fallback.py | 5 ++-- .../test_check_generate_schema.py | 24 +++++++++++++++++++ .../check_webhook_waitfor_parallel.py | 4 ++-- .../check_batch_transform_flow.py | 6 ++--- .../summarize/check_summarize_flow.py | 6 ++--- .../interactive/check_solution_select.py | 5 ++-- .../ixp/check_project_selection.py | 4 ++-- 14 files changed, 67 insertions(+), 33 deletions(-) diff --git a/tests/tasks/uipath-maestro-flow/_shared/test_same_ground_corpus.py b/tests/tasks/uipath-maestro-flow/_shared/test_same_ground_corpus.py index 216dfe4f5c..5be99f5fb9 100644 --- a/tests/tasks/uipath-maestro-flow/_shared/test_same_ground_corpus.py +++ b/tests/tasks/uipath-maestro-flow/_shared/test_same_ground_corpus.py @@ -174,3 +174,20 @@ def test_debug_graded_prompts_name_a_same_name_solution() -> None: if not re.search(r"\b(?:in|inside) a solution of the same name\b", prompt): offenders.add(relative) assert offenders == DEBUG_SOLUTION_ALLOWLIST | DEBUG_PROJECT_LAYOUT_EXCEPTIONS + + +def test_external_graders_use_package_qualified_shared_imports() -> None: + shared_modules = { + path.stem for path in (FLOW_TASKS / "_shared").glob("*.py") + } + offenders = set() + for path in FLOW_TASKS.rglob("*.py"): + relative = path.relative_to(FLOW_TASKS) + if relative.parts[0] == "_shared": + continue + for module in re.findall( + r"(?m)^(?:from|import)\s+([A-Za-z_][A-Za-z0-9_]*)", path.read_text() + ): + if module in shared_modules: + offenders.add(relative.as_posix()) + assert offenders == set() diff --git a/tests/tasks/uipath-maestro-flow/bindings/check_bindings.py b/tests/tasks/uipath-maestro-flow/bindings/check_bindings.py index b46a60e926..d4bee0f6c3 100644 --- a/tests/tasks/uipath-maestro-flow/bindings/check_bindings.py +++ b/tests/tasks/uipath-maestro-flow/bindings/check_bindings.py @@ -76,9 +76,8 @@ from pathlib import Path from typing import Any, NoReturn -SHARED_DIR = Path(__file__).resolve().parents[1] / "_shared" -sys.path.insert(0, str(SHARED_DIR)) -from flow_check import find_flow_files # noqa: E402 +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from _shared.flow_check import find_flow_files # noqa: E402 def _fail(message: str) -> NoReturn: diff --git a/tests/tasks/uipath-maestro-flow/connector_features/check_enum_flow.py b/tests/tasks/uipath-maestro-flow/connector_features/check_enum_flow.py index e321406a10..c1bf8cc85f 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/check_enum_flow.py +++ b/tests/tasks/uipath-maestro-flow/connector_features/check_enum_flow.py @@ -19,9 +19,8 @@ from pathlib import Path from typing import Any, NoReturn -SHARED_DIR = Path(__file__).resolve().parent.parent / "_shared" -sys.path.insert(0, str(SHARED_DIR)) -from flow_check import find_flow_file # noqa: E402 +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +from _shared.flow_check import find_flow_file # noqa: E402 def _fail(message: str) -> NoReturn: diff --git a/tests/tasks/uipath-maestro-flow/connector_features/check_generate_schema.py b/tests/tasks/uipath-maestro-flow/connector_features/check_generate_schema.py index 1d1d892d3a..9ffe3ed072 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/check_generate_schema.py +++ b/tests/tasks/uipath-maestro-flow/connector_features/check_generate_schema.py @@ -22,8 +22,8 @@ from pathlib import Path from typing import Any -sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "_shared")) -from flow_check import find_flow_file # noqa: E402 +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from _shared.flow_check import find_flow_file # noqa: E402 JIRA_NODE_TYPE = "uipath.connector.uipath-atlassian-jira.create-issue" REQUIRED_PARENT_FIELDS = ( diff --git a/tests/tasks/uipath-maestro-flow/connector_features/check_managed_http_fallback.py b/tests/tasks/uipath-maestro-flow/connector_features/check_managed_http_fallback.py index 7324b29d8b..5b4bfc3f5d 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/check_managed_http_fallback.py +++ b/tests/tasks/uipath-maestro-flow/connector_features/check_managed_http_fallback.py @@ -9,9 +9,8 @@ from pathlib import Path from typing import Any, NoReturn -SHARED_DIR = Path(__file__).resolve().parent.parent / "_shared" -sys.path.insert(0, str(SHARED_DIR)) -from flow_check import find_flow_file # noqa: E402 +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +from _shared.flow_check import find_flow_file # noqa: E402 def _fail(message: str) -> NoReturn: diff --git a/tests/tasks/uipath-maestro-flow/connector_features/check_multiselect_flow.py b/tests/tasks/uipath-maestro-flow/connector_features/check_multiselect_flow.py index acf6a8f62e..24d74684c6 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/check_multiselect_flow.py +++ b/tests/tasks/uipath-maestro-flow/connector_features/check_multiselect_flow.py @@ -19,9 +19,8 @@ import sys from pathlib import Path -SHARED_DIR = Path(__file__).resolve().parent.parent / "_shared" -sys.path.insert(0, str(SHARED_DIR)) -from flow_check import find_flow_files # noqa: E402 +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +from _shared.flow_check import find_flow_files # noqa: E402 def parse_users(value): diff --git a/tests/tasks/uipath-maestro-flow/connector_features/check_path_param_value.py b/tests/tasks/uipath-maestro-flow/connector_features/check_path_param_value.py index 9b8d4b8586..3ab89caec8 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/check_path_param_value.py +++ b/tests/tasks/uipath-maestro-flow/connector_features/check_path_param_value.py @@ -22,9 +22,8 @@ from pathlib import Path from typing import Any, NoReturn -SHARED_DIR = Path(__file__).resolve().parent.parent / "_shared" -sys.path.insert(0, str(SHARED_DIR)) -from flow_check import find_flow_file # noqa: E402 +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +from _shared.flow_check import find_flow_file # noqa: E402 _JSONSTRING_PREFIX = "=jsonString:" diff --git a/tests/tasks/uipath-maestro-flow/connector_features/non-catalog-http-fallback/check_non_catalog_http_fallback.py b/tests/tasks/uipath-maestro-flow/connector_features/non-catalog-http-fallback/check_non_catalog_http_fallback.py index 69276a0ef2..b655a1e296 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/non-catalog-http-fallback/check_non_catalog_http_fallback.py +++ b/tests/tasks/uipath-maestro-flow/connector_features/non-catalog-http-fallback/check_non_catalog_http_fallback.py @@ -34,9 +34,8 @@ from pathlib import Path from typing import Any, NoReturn -SHARED_DIR = Path(__file__).resolve().parents[2] / "_shared" -sys.path.insert(0, str(SHARED_DIR)) -from flow_check import find_flow_file # noqa: E402 +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) +from _shared.flow_check import find_flow_file # noqa: E402 FLOW_GLOB = "**/SpotifyProfileTest*.flow" # The generic HTTP connector โ€” the only managed path to a service with no diff --git a/tests/tasks/uipath-maestro-flow/connector_features/test_check_generate_schema.py b/tests/tasks/uipath-maestro-flow/connector_features/test_check_generate_schema.py index 7a4b8cb0cb..a7b070d027 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/test_check_generate_schema.py +++ b/tests/tasks/uipath-maestro-flow/connector_features/test_check_generate_schema.py @@ -7,6 +7,7 @@ CHECKER = Path(__file__).with_name("check_generate_schema.py") +FLOW_CHECK = CHECKER.parents[1] / "_shared" / "flow_check.py" def _flow(*, parent_values: bool = True) -> dict: @@ -58,6 +59,29 @@ def test_accepts_root_sdk_emit_with_arbitrary_filename(tmp_path: Path) -> None: assert result.returncode == 0, result.stdout + result.stderr +def test_accepts_co_located_shared_helper_in_isolated_task_dir(tmp_path: Path) -> None: + task_dir = tmp_path / "task" + shared = task_dir / "_shared" + shared.mkdir(parents=True) + staged_checker = task_dir / CHECKER.name + staged_checker.write_bytes(CHECKER.read_bytes()) + (shared / "__init__.py").write_text("", encoding="utf-8") + (shared / "flow_check.py").write_bytes(FLOW_CHECK.read_bytes()) + workspace = tmp_path / "workspace" + workspace.mkdir() + (workspace / "workflow.flow").write_text(json.dumps(_flow()), encoding="utf-8") + + result = subprocess.run( + [sys.executable, str(staged_checker)], + cwd=workspace, + capture_output=True, + text=True, + check=False, + ) + + assert result.returncode == 0, result.stdout + result.stderr + + def test_prefers_solution_flow_over_root_scratch_emit(tmp_path: Path) -> None: project = tmp_path / "GenerateSchemaTestSolution" / "GenerateSchemaTest" project.mkdir(parents=True) diff --git a/tests/tasks/uipath-maestro-flow/connector_trigger/check_webhook_waitfor_parallel.py b/tests/tasks/uipath-maestro-flow/connector_trigger/check_webhook_waitfor_parallel.py index a17f8c4f45..1cfcdd5436 100644 --- a/tests/tasks/uipath-maestro-flow/connector_trigger/check_webhook_waitfor_parallel.py +++ b/tests/tasks/uipath-maestro-flow/connector_trigger/check_webhook_waitfor_parallel.py @@ -28,8 +28,8 @@ import os import sys -sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "_shared")) -from flow_check import find_flow_file # noqa: E402 +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) +from _shared.flow_check import find_flow_file # noqa: E402 FLOW_GLOB = "**/WebhookSelfTest*.flow" EVENT_MARKERS = ("uipath.connector.event", "uipath-http-webhook") diff --git a/tests/tasks/uipath-maestro-flow/context-grounding/batch_transform/check_batch_transform_flow.py b/tests/tasks/uipath-maestro-flow/context-grounding/batch_transform/check_batch_transform_flow.py index 88330a07f2..64f49330c3 100644 --- a/tests/tasks/uipath-maestro-flow/context-grounding/batch_transform/check_batch_transform_flow.py +++ b/tests/tasks/uipath-maestro-flow/context-grounding/batch_transform/check_batch_transform_flow.py @@ -27,9 +27,9 @@ import sys from typing import NoReturn -sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "_shared")) -from advisory_flow_utils import unwrap # noqa: E402 -from flow_check import find_flow_file # noqa: E402 +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..")) +from _shared.advisory_flow_utils import unwrap # noqa: E402 +from _shared.flow_check import find_flow_file # noqa: E402 NODE_TYPE = "uipath.pattern.batch-transform" EXPECTED_TYPE_VERSION = "1.0" diff --git a/tests/tasks/uipath-maestro-flow/context-grounding/summarize/check_summarize_flow.py b/tests/tasks/uipath-maestro-flow/context-grounding/summarize/check_summarize_flow.py index 5f31f766a9..59aca54388 100644 --- a/tests/tasks/uipath-maestro-flow/context-grounding/summarize/check_summarize_flow.py +++ b/tests/tasks/uipath-maestro-flow/context-grounding/summarize/check_summarize_flow.py @@ -29,9 +29,9 @@ import sys from typing import NoReturn -sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "_shared")) -from advisory_flow_utils import unwrap # noqa: E402 -from flow_check import find_flow_file # noqa: E402 +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..")) +from _shared.advisory_flow_utils import unwrap # noqa: E402 +from _shared.flow_check import find_flow_file # noqa: E402 NODE_TYPE = "uipath.pattern.deep-rag" EXPECTED_TYPE_VERSION = "1.0" diff --git a/tests/tasks/uipath-maestro-flow/interactive/check_solution_select.py b/tests/tasks/uipath-maestro-flow/interactive/check_solution_select.py index 97b559facf..ed506576eb 100644 --- a/tests/tasks/uipath-maestro-flow/interactive/check_solution_select.py +++ b/tests/tasks/uipath-maestro-flow/interactive/check_solution_select.py @@ -9,9 +9,8 @@ import sys from pathlib import Path -SHARED = Path(__file__).resolve().parents[1] / "_shared" -sys.path.insert(0, str(SHARED)) -from flow_check import find_flow_file, find_project_dir # noqa: E402 +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from _shared.flow_check import find_flow_file, find_project_dir # noqa: E402 SELECTED = Path("WeatherSelection-7K4M") PROJECT_PATTERN = "WeatherSelection-7K4M/**/project.uiproj" diff --git a/tests/tasks/uipath-maestro-flow/ixp/check_project_selection.py b/tests/tasks/uipath-maestro-flow/ixp/check_project_selection.py index 6dda89d919..401bed4766 100644 --- a/tests/tasks/uipath-maestro-flow/ixp/check_project_selection.py +++ b/tests/tasks/uipath-maestro-flow/ixp/check_project_selection.py @@ -7,8 +7,8 @@ import os import sys -sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "_shared")) -from flow_check import find_flow_file # noqa: E402 +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) +from _shared.flow_check import find_flow_file # noqa: E402 EXPECTED = { From be5c33432d34ab36693bf2e5fc305f48f6cfa0fb Mon Sep 17 00:00:00 2001 From: Tao Ma <51425734+tmatup@users.noreply.github.com> Date: Wed, 26 Aug 2026 08:06:50 -0700 Subject: [PATCH 14/52] fix(maestro-flow): remediate live and preview eval failures (#2821) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(evals): anchor shared grader staging to its checkout ๐Ÿค– Generated with Claude Code Co-Authored-By: [Claude](mailto:noreply@anthropic.com) * fix(evals): provision the preview Flow SDK offline ๐Ÿค– Generated with Claude Code Co-Authored-By: [Claude](mailto:noreply@anthropic.com) * fix(evals): scope shared staging to the selected suite ๐Ÿค– Generated with Claude Code Co-Authored-By: [Claude](mailto:noreply@anthropic.com) * test(maestro-flow): allow dynamic validation latency ๐Ÿค– Generated with Claude Code Co-Authored-By: [Claude](mailto:noreply@anthropic.com) * test(evals): pin preview to remediated Flow SDK ๐Ÿค– Generated with Claude Code Co-Authored-By: [Claude](mailto:noreply@anthropic.com) * fix(preview): preserve operation-specific schema parents ๐Ÿค– Generated with Claude Code Co-Authored-By: [Claude](mailto:noreply@anthropic.com) * fix(flow): preserve requested checkpoints and topology Capture requested mutation snapshots before optional cleanup steps, and allow validate-only IxP graphs to mark missing extractors with placeholders. ๐Ÿค– Generated with Claude Code Co-Authored-By: [Claude](mailto:noreply@anthropic.com) * test(evals): pin canonical preview guidance Advance the preview build context to the Flow SDK commit that owns the remediated connector and IxP guidance. ๐Ÿค– Generated with Claude Code Co-Authored-By: [Claude](mailto:noreply@anthropic.com) * test(evals): harden Jira seed envelope parsing Retry only explicitly transient folder lookups and surface a redacted failure envelope instead of an opaque missing-Data traceback. ๐Ÿค– Generated with Claude Code Co-Authored-By: [Claude](mailto:noreply@anthropic.com) --- skills/uipath-maestro-flow/SKILL.md | 2 +- tests/docker/maestro-flow-sdk.Dockerfile | 73 ++++++++++++++++ tests/docker/maestro-flow-sdk.env | 2 + tests/experiments/nightly.yaml | 8 ++ .../maestro-flow-sdk-smoke/Smoke.flow.ts | 11 +++ .../maestro-flow-sdk-smoke/package.json | 4 + tests/scripts/stage-preview-sdk-workspace.sh | 74 ++++++++++++++++ tests/scripts/stage_shared.sh | 25 +++++- .../test_stage_preview_sdk_workspace.py | 87 +++++++++++++++++++ tests/scripts/test_stage_shared.py | 71 +++++++++++++++ .../fixtures/folder_get_missing_data.json | 3 + .../e2e/jira_create_issue/jira_is.py | 59 ++++++++++--- .../e2e/jira_create_issue/test_jira_is.py | 70 +++++++++++++++ 13 files changed, 476 insertions(+), 13 deletions(-) create mode 100644 tests/docker/maestro-flow-sdk.Dockerfile create mode 100644 tests/docker/maestro-flow-sdk.env create mode 100644 tests/fixtures/maestro-flow-sdk-smoke/Smoke.flow.ts create mode 100644 tests/fixtures/maestro-flow-sdk-smoke/package.json create mode 100755 tests/scripts/stage-preview-sdk-workspace.sh create mode 100644 tests/scripts/test_stage_preview_sdk_workspace.py create mode 100644 tests/scripts/test_stage_shared.py create mode 100644 tests/tasks/uipath-maestro-flow/e2e/jira_create_issue/fixtures/folder_get_missing_data.json create mode 100644 tests/tasks/uipath-maestro-flow/e2e/jira_create_issue/test_jira_is.py diff --git a/skills/uipath-maestro-flow/SKILL.md b/skills/uipath-maestro-flow/SKILL.md index 0776d57839..a6d8bf9eb2 100644 --- a/skills/uipath-maestro-flow/SKILL.md +++ b/skills/uipath-maestro-flow/SKILL.md @@ -82,7 +82,7 @@ Guide for creating, editing, validating, debugging, publishing, diagnosing, and 7. **Narrate progress only when requested or clearly opted into.** Otherwise work silently and surface decisions, failures, consent gates, and the final result. When engaged, use one short plain-English line per logical step across CLI calls, shell builtins, edits, and searches; do not narrate flags or JSON structure. See [shared/ux-narration-and-todos.md](references/shared/ux-narration-and-todos.md) ยงWhen to engage. 8. **Maintain a user-facing progress list only when tracking or verbosity is requested.** In silent mode there is no user-facing todo list (the agent MAY track privately). When engaged, journeys above trivial complexity get granular step-level todos; counts follow actual work, not a target. Hide registry lookups, parsing, and file reads inside their logical step. See [shared/ux-narration-and-todos.md](references/shared/ux-narration-and-todos.md) for triggers, granularity, thresholds, and pivots. 9. **Each node has exactly one author: Edit/Write or CLI, never both.** CLI-owned nodes are connector activities (`uipath.connector..`), connector triggers (`uipath.connector.trigger..`), wait-for-events (`uipath.connector.event..`, configured like triggers), and managed HTTP (`core.action.http.v2`); add/configure them with `uip maestro flow node add` and `node configure`. All othersโ€”triggers, control flow, logic, HITL, patterns, agents, resource nodes, and queuesโ€”are user-owned and should be authored directly with `Edit` or `Write`. Never full-file `Write` a flow containing CLI-owned nodes because it can clobber CLI-set `bindings[]` and `inputs.detail`; use `Edit` or configure CLI-owned nodes last. Their `inputs.detail` is a `=jsonString:essentialConfiguration` envelope rejected when hand-authored. Inline-agent CLI is limited to `uip agent init / refresh / validate --inline-in-flow`; the `uipath.agent.autonomous` node is user-owned. Scripting (`python`, `node`, `jq`, `sed`, `awk`, or shell heredocs) is a last resort for user-owned edits and requires explicit approval after explaining state bypass, opaque diffs, and lack of interruption points. See [author/CAPABILITY.md โ€” Node ownership](references/author/CAPABILITY.md#node-ownership--who-authors-the-node) and [author/editing-operations.md โ€” Tool Selection Ladder](references/author/editing-operations.md#tool-selection-ladder). -10. **Batch independent tool calls and chain dependent CLI calls.** A typical greenfield build is three turns: T1 scaffold, pull the registry, and add CLI-owned nodes in one chained `Bash`, alongside independent registry/file reads; T2 read the scaffold while editing/adding the End node and edges; T3 chain configure, validate, and format. Split only when later work depends on stdout or a mutation. See [author/greenfield.md โ€” Three-turn execution map](references/author/greenfield.md#three-turn-execution-map). +10. **Batch independent tool calls and chain dependent CLI calls.** A typical greenfield build is three turns: T1 scaffold, pull the registry, and add CLI-owned nodes in one chained `Bash`, alongside independent registry/file reads; T2 read the scaffold while editing/adding the End node and edges; T3 chain configure, validate, and format. Split only when later work depends on stdout or a mutation. A user-requested snapshot or checkpoint is exactly such a boundary: capture it immediately after the named mutation and before optional `validate`, `format`, or later mutations, even when the commands share one Bash call, and never let a later command failure skip writing it. See [author/greenfield.md โ€” Three-turn execution map](references/author/greenfield.md#three-turn-execution-map). 11. **Cross-node bindings in `=js:` require `$vars.`** Use `=js:$vars..output...`; bare `=js:.output...` resolves to `undefined`. See [variables-and-expressions.md โ€” IS Activity Inputs Require `=js:`](references/shared/variables-and-expressions.md#is-activity-inputs-require-js-critical). 12. **Node and edge IDs must begin with a letter.** Use descriptive camelCase node IDs and `edge____` edge IDs. Reserve UUIDs for the top-level flow `id` and `entryPointId`. diff --git a/tests/docker/maestro-flow-sdk.Dockerfile b/tests/docker/maestro-flow-sdk.Dockerfile new file mode 100644 index 0000000000..4c699ce76c --- /dev/null +++ b/tests/docker/maestro-flow-sdk.Dockerfile @@ -0,0 +1,73 @@ +# syntax=docker/dockerfile:1.7 +ARG BASE_IMAGE=skills-image:base + +FROM node:22-bookworm-slim AS preview_flow_sdk_package + +ARG FLOW_SDK_VERSION +ARG FLOW_BUILDER_SDK_SHA +WORKDIR /src +COPY --from=flow_builder_sdk typescript/sdk/ ./typescript/sdk/ +COPY --from=flow_builder_sdk integrations/scripts/ ./integrations/scripts/ +RUN mkdir -p /pkg && \ + cd typescript/sdk && \ + npm ci --no-audit --no-fund && \ + npm pkg set \ + version="${FLOW_SDK_VERSION:?FLOW_SDK_VERSION is required}" \ + gitref="${FLOW_BUILDER_SDK_SHA:?FLOW_BUILDER_SDK_SHA is required}" && \ + npm pack --pack-destination /pkg && \ + mv /pkg/uipath-flow-sdk-*.tgz /pkg/uipath-flow-sdk.tgz + +FROM ${BASE_IMAGE} + +ARG FLOW_SDK_VERSION +ARG FLOW_BUILDER_SDK_SHA +ENV PREVIEW_FLOW_SDK_ROOT=/opt/preview-flow-sdk +ENV PREVIEW_FLOW_SDK_ASSETS_ROOT=/opt/preview-flow-sdk-assets + +RUN apt-get update && apt-get install -y --no-install-recommends unzip + +# Build and install the SDK from the same pinned source that supplies the +# connector library. This makes an SDK/compiler remediation measurable before +# its next package release without leaving package credentials in the image. +COPY --from=preview_flow_sdk_package \ + /pkg/uipath-flow-sdk.tgz /opt/preview-flow-sdk-pkg/uipath-flow-sdk.tgz +RUN set -euo pipefail && \ + mkdir -p "$PREVIEW_FLOW_SDK_ROOT" && \ + cd "$PREVIEW_FLOW_SDK_ROOT" && \ + printf '%s\n' '{"private":true,"type":"module"}' > package.json && \ + npm install --save-exact --no-audit --no-fund \ + /opt/preview-flow-sdk-pkg/uipath-flow-sdk.tgz && \ + node -e "const p=require('./node_modules/@uipath/flow-sdk/package.json'); console.log('Installed', p.name+'@'+p.version)" && \ + printf '%s\n' "$FLOW_SDK_VERSION" > flow-sdk.version && \ + test -x node_modules/.bin/flow-sdk + +# Connector descriptors are kept at the preview skill's provenance pin so +# connector authoring and SDK signatures describe the same source revision. +COPY --from=flow_builder_sdk typescript/sdk/lib/ ${PREVIEW_FLOW_SDK_ASSETS_ROOT}/typescript/sdk/lib/ +COPY --from=flow_builder_sdk integrations/ ${PREVIEW_FLOW_SDK_ASSETS_ROOT}/integrations/ + +RUN FLOW_SDK_ROOT=${PREVIEW_FLOW_SDK_ASSETS_ROOT} \ + bash ${PREVIEW_FLOW_SDK_ASSETS_ROOT}/typescript/sdk/lib/scripts/unpack-library.sh && \ + python3 ${PREVIEW_FLOW_SDK_ASSETS_ROOT}/integrations/scripts/generate_connectors_ts.py \ + --library ${PREVIEW_FLOW_SDK_ASSETS_ROOT}/typescript/sdk/lib/library-json \ + --output ${PREVIEW_FLOW_SDK_ASSETS_ROOT}/connectors \ + --import @uipath/flow-sdk && \ + chmod +x ${PREVIEW_FLOW_SDK_ASSETS_ROOT}/integrations/scripts/prepare_connector.py && \ + printf '#!/bin/sh\nexec python3 %s/integrations/scripts/prepare_connector.py "$@" --import @uipath/flow-sdk\n' \ + "${PREVIEW_FLOW_SDK_ASSETS_ROOT}" > /usr/local/bin/prepare-connector && \ + chmod +x /usr/local/bin/prepare-connector && \ + prepare-connector --help >/dev/null && \ + printf '%s\n' "${FLOW_BUILDER_SDK_SHA:?FLOW_BUILDER_SDK_SHA is required}" \ + > ${PREVIEW_FLOW_SDK_ASSETS_ROOT}/flow-builder-sdk.sha + +ENV FLOW_SDK_LIBRARY_JSON=${PREVIEW_FLOW_SDK_ASSETS_ROOT}/typescript/sdk/lib/library-json \ + FLOW_SDK_LIBRARY_MD=${PREVIEW_FLOW_SDK_ASSETS_ROOT}/typescript/sdk/lib/library-md \ + FLOW_SDK_CONNECTORS_DIR=${PREVIEW_FLOW_SDK_ASSETS_ROOT}/connectors + +# This assertion runs after the secret-bearing layer has ended. It proves that +# the pinned SDK resolves and the product CLI can check real source without any +# registry credential at runtime. +COPY tests/fixtures/maestro-flow-sdk-smoke/ /opt/maestro-flow-sdk-smoke/ +RUN cd /opt/maestro-flow-sdk-smoke && \ + ln -s ${PREVIEW_FLOW_SDK_ROOT}/node_modules node_modules && \ + uip maestro flow check Smoke.flow.ts --source diff --git a/tests/docker/maestro-flow-sdk.env b/tests/docker/maestro-flow-sdk.env new file mode 100644 index 0000000000..df83bc5cb4 --- /dev/null +++ b/tests/docker/maestro-flow-sdk.env @@ -0,0 +1,2 @@ +FLOW_SDK_VERSION=3.20.1-remediation.1 +FLOW_BUILDER_SDK_SHA=21c339a032d52200e0cdf7ad072a769b6a3d65ba diff --git a/tests/experiments/nightly.yaml b/tests/experiments/nightly.yaml index 82acd6575b..4906f1d8d3 100644 --- a/tests/experiments/nightly.yaml +++ b/tests/experiments/nightly.yaml @@ -15,6 +15,7 @@ defaults: image: skills-image:latest env_passthrough_extra: - SKILLS_REPO_PATH + - MAESTRO_FLOW_SDK_SETUP - BEDROCK_MODEL - TASK_PARALLELISM - UIPATH_CLI_ENABLE_ENV_AUTH @@ -52,6 +53,13 @@ defaults: api_base: CODEX_BASE_URL api_key: CODEX_API_KEY + pre_run: + - command: |- + if [ "${MAESTRO_FLOW_SDK_SETUP:-0}" = "1" ]; then + bash "$SKILLS_REPO_PATH/tests/scripts/stage-preview-sdk-workspace.sh" + fi + timeout: 30 + # Sandbox cleanup between tasks. Tasks that upload Studio Web solutions # wire cleanup_solutions.py into their own post_run โ€” that must run before # this step (still needs npm-installed CLIs). diff --git a/tests/fixtures/maestro-flow-sdk-smoke/Smoke.flow.ts b/tests/fixtures/maestro-flow-sdk-smoke/Smoke.flow.ts new file mode 100644 index 0000000000..1448d4c1e7 --- /dev/null +++ b/tests/fixtures/maestro-flow-sdk-smoke/Smoke.flow.ts @@ -0,0 +1,11 @@ +import { flow, out, script, types } from '@uipath/flow-sdk'; + +export default flow('sdk-smoke') + .name('SdkSmoke') + .version('1.0.0') + .output({ message: types.string }) + .step('makeMessage', script({ + code: 'return { message: "ready" };', + })) + .return({ message: out('makeMessage', 'message') }) + .build(); diff --git a/tests/fixtures/maestro-flow-sdk-smoke/package.json b/tests/fixtures/maestro-flow-sdk-smoke/package.json new file mode 100644 index 0000000000..e986b24bba --- /dev/null +++ b/tests/fixtures/maestro-flow-sdk-smoke/package.json @@ -0,0 +1,4 @@ +{ + "private": true, + "type": "module" +} diff --git a/tests/scripts/stage-preview-sdk-workspace.sh b/tests/scripts/stage-preview-sdk-workspace.sh new file mode 100755 index 0000000000..c627bca5e3 --- /dev/null +++ b/tests/scripts/stage-preview-sdk-workspace.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash +set -euo pipefail + +sdk_root=${PREVIEW_FLOW_SDK_ROOT:-/opt/preview-flow-sdk} +assets_root=${PREVIEW_FLOW_SDK_ASSETS_ROOT:-/opt/preview-flow-sdk-assets} +package_dir=$sdk_root/node_modules/@uipath/flow-sdk +library_json=${FLOW_SDK_LIBRARY_JSON:-$assets_root/typescript/sdk/lib/library-json} + +[[ -f "$package_dir/package.json" ]] || { + echo "stage-preview-sdk-workspace: missing $package_dir/package.json" >&2 + exit 1 +} +[[ ! -e node_modules ]] || { + echo "stage-preview-sdk-workspace: node_modules already exists" >&2 + exit 1 +} +[[ -f "$library_json/index.json" ]] || { + echo "stage-preview-sdk-workspace: missing connector library index at $library_json" >&2 + exit 1 +} + +flow_builder_sdk_commit=$(tr -d '[:space:]' < "$assets_root/flow-builder-sdk.sha") +[[ -n "$flow_builder_sdk_commit" ]] || { + echo "stage-preview-sdk-workspace: missing flow-builder-sdk provenance" >&2 + exit 1 +} + +node - "$library_json/index.json" <<'NODE' +const fs = require('node:fs'); +const index = JSON.parse(fs.readFileSync(process.argv[2], 'utf8')); +const required = 'uipath.connector.uipath-uipath-dataservice.query-entity-records'; +if (!index.entries?.some(({ nodeType }) => nodeType === required)) { + throw new Error(`connector library is missing ${required}`); +} +NODE + +sdk_version=$(node - "$package_dir/package.json" <<'NODE' +const fs = require('node:fs'); +console.log(JSON.parse(fs.readFileSync(process.argv[2], 'utf8')).version); +NODE +) + +ln -s "$sdk_root/node_modules" node_modules +node - "$sdk_version" "$flow_builder_sdk_commit" <<'NODE' +const fs = require('node:fs'); +const version = process.argv[2]; +const flowBuilderSdkCommit = process.argv[3]; +let packageJson = {}; +if (fs.existsSync('package.json')) { + packageJson = JSON.parse(fs.readFileSync('package.json', 'utf8')); +} +packageJson.private = true; +packageJson.type = 'module'; +packageJson.devDependencies = { + ...(packageJson.devDependencies || {}), + '@uipath/flow-sdk': version, +}; +fs.writeFileSync('package.json', `${JSON.stringify(packageJson, null, 2)}\n`); +fs.writeFileSync( + 'preview-sdk-provenance.json', + `${JSON.stringify({ + package: '@uipath/flow-sdk', + version, + flow_builder_sdk_commit: flowBuilderSdkCommit, + connector_library: true, + }, null, 2)}\n`, +); +NODE + +node -e "import('@uipath/flow-sdk')" +uip maestro flow compile --help >/dev/null +prepare-connector --help >/dev/null +printf 'stage-preview-sdk-workspace: @uipath/flow-sdk@%s, flow-builder-sdk@%s, connector library\n' \ + "$sdk_version" "$flow_builder_sdk_commit" diff --git a/tests/scripts/stage_shared.sh b/tests/scripts/stage_shared.sh index 7f06af4c0c..cb98402ff1 100755 --- a/tests/scripts/stage_shared.sh +++ b/tests/scripts/stage_shared.sh @@ -19,7 +19,25 @@ # mutates the checked-out tree, not the committed repo. set -euo pipefail -cd "$(git rev-parse --show-toplevel)" +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "$(git -C "$script_dir" rev-parse --show-toplevel)" + +manifest=${STAGE_SHARED_MANIFEST:-} +if [ -n "$manifest" ]; then + : > "$manifest" +fi + +search_root=${STAGE_SHARED_ROOT:-tests/tasks} +if [[ "$search_root" = "$PWD"/* ]]; then + search_root=${search_root#"$PWD"/} +fi +case "$search_root" in + tests/tasks | tests/tasks/*) ;; + *) + echo "stage_shared: STAGE_SHARED_ROOT must be under tests/tasks: $search_root" >&2 + exit 2 + ;; +esac count=0 while IFS= read -r f; do @@ -32,8 +50,11 @@ while IFS= read -r f; do [ -d "$src" ] || continue # group has no _shared -> nothing to stage [ -e "$dir/_shared" ] && continue # already present (idempotent / hand-authored) cp -R "$src" "$dir/_shared" + if [ -n "$manifest" ]; then + printf '%s\n' "$PWD/$dir/_shared" >> "$manifest" + fi count=$((count + 1)) -done < <(git grep -lE 'from _shared|import _shared' -- 'tests/tasks/**/*.py') +done < <(git grep -lE 'from _shared|import _shared' -- "$search_root/**/*.py") # Pass 2: graders that put a `_shared` DIR itself on sys.path and then import # its modules bare (e.g. `sys.path.insert(0, .../"..", "_shared")` followed by diff --git a/tests/scripts/test_stage_preview_sdk_workspace.py b/tests/scripts/test_stage_preview_sdk_workspace.py new file mode 100644 index 0000000000..a0eea5f7b3 --- /dev/null +++ b/tests/scripts/test_stage_preview_sdk_workspace.py @@ -0,0 +1,87 @@ +import json +import os +import subprocess +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[2] +STAGE_SCRIPT = REPO_ROOT / "tests/scripts/stage-preview-sdk-workspace.sh" +REQUIRED_CONNECTOR = "uipath.connector.uipath-uipath-dataservice.query-entity-records" + + +def _write_executable(path: Path, body: str) -> None: + path.write_text(body) + path.chmod(0o755) + + +def test_stages_credential_free_sdk_workspace(tmp_path: Path) -> None: + sdk_root = tmp_path / "sdk" + package_dir = sdk_root / "node_modules/@uipath/flow-sdk" + package_dir.mkdir(parents=True) + (package_dir / "package.json").write_text( + json.dumps( + { + "name": "@uipath/flow-sdk", + "version": "3.20.0", + "type": "module", + "exports": "./index.js", + } + ) + ) + (package_dir / "index.js").write_text("export {};\n") + + assets_root = tmp_path / "assets" + library_json = assets_root / "typescript/sdk/lib/library-json" + library_json.mkdir(parents=True) + (library_json / "index.json").write_text( + json.dumps({"entries": [{"nodeType": REQUIRED_CONNECTOR}]}) + ) + commit = "efd27ce4c90ad76fc7f3c9b67f4920997b2cf0a8" + (assets_root / "flow-builder-sdk.sha").write_text(f"{commit}\n") + + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + _write_executable( + bin_dir / "uip", + "#!/bin/sh\n[ \"$*\" = \"maestro flow compile --help\" ]\n", + ) + _write_executable( + bin_dir / "prepare-connector", + "#!/bin/sh\n[ \"$*\" = \"--help\" ]\n", + ) + + workspace = tmp_path / "workspace" + workspace.mkdir() + (workspace / "package.json").write_text( + json.dumps({"scripts": {"keep": "true"}}) + ) + env = { + **os.environ, + "PATH": f"{bin_dir}:{os.environ['PATH']}", + "PREVIEW_FLOW_SDK_ROOT": str(sdk_root), + "PREVIEW_FLOW_SDK_ASSETS_ROOT": str(assets_root), + "FLOW_SDK_LIBRARY_JSON": str(library_json), + } + + completed = subprocess.run( + ["bash", str(STAGE_SCRIPT)], + cwd=workspace, + env=env, + check=True, + capture_output=True, + text=True, + ) + + assert "@uipath/flow-sdk@3.20.0" in completed.stdout + assert (workspace / "node_modules").is_symlink() + assert (workspace / "node_modules").resolve() == sdk_root / "node_modules" + package_json = json.loads((workspace / "package.json").read_text()) + assert package_json["scripts"] == {"keep": "true"} + assert package_json["devDependencies"] == {"@uipath/flow-sdk": "3.20.0"} + assert "flowSdk" not in package_json + assert json.loads((workspace / "preview-sdk-provenance.json").read_text()) == { + "package": "@uipath/flow-sdk", + "version": "3.20.0", + "flow_builder_sdk_commit": commit, + "connector_library": True, + } diff --git a/tests/scripts/test_stage_shared.py b/tests/scripts/test_stage_shared.py new file mode 100644 index 0000000000..727fdd3b6f --- /dev/null +++ b/tests/scripts/test_stage_shared.py @@ -0,0 +1,71 @@ +"""Regression tests for staging shared checker helpers.""" + +import shutil +import subprocess +import sys +from pathlib import Path + + +SCRIPT = Path(__file__).with_name("stage_shared.sh") + + +def test_stage_shared_anchors_to_checkout_and_stages_nested_checkers(tmp_path): + repo = tmp_path / "skills" + script = repo / "tests" / "scripts" / "stage_shared.sh" + shared = repo / "tests" / "tasks" / "example" / "_shared" + shallow = repo / "tests" / "tasks" / "example" / "scenario" + nested = repo / "tests" / "tasks" / "example" / "nested" / "scenario" + unrelated_shared = repo / "tests" / "tasks" / "unrelated" / "_shared" + unrelated_task = repo / "tests" / "tasks" / "unrelated" / "scenario" + + script.parent.mkdir(parents=True) + shared.mkdir(parents=True) + shallow.mkdir(parents=True) + nested.mkdir(parents=True) + unrelated_shared.mkdir(parents=True) + unrelated_task.mkdir(parents=True) + shutil.copy2(SCRIPT, script) + (shared / "__init__.py").write_text("", encoding="utf-8") + (shared / "checker.py").write_text("VALUE = 42\n", encoding="utf-8") + checker = "from _shared.checker import VALUE\nassert VALUE == 42\n" + (shallow / "check.py").write_text(checker, encoding="utf-8") + (nested / "check.py").write_text(checker, encoding="utf-8") + (unrelated_shared / "__init__.py").write_text("", encoding="utf-8") + (unrelated_shared / "checker.py").write_text("VALUE = 42\n", encoding="utf-8") + (unrelated_task / "check.py").write_text(checker, encoding="utf-8") + + subprocess.run(["git", "init", "--quiet", str(repo)], check=True) + subprocess.run(["git", "-C", str(repo), "add", "."], check=True) + + manifest = tmp_path / "staged-paths.txt" + first = subprocess.run( + [ + "env", + f"STAGE_SHARED_MANIFEST={manifest}", + "STAGE_SHARED_ROOT=tests/tasks/example", + "bash", + str(script), + ], + cwd=tmp_path, + check=True, + capture_output=True, + text=True, + ) + second = subprocess.run( + ["env", "STAGE_SHARED_ROOT=tests/tasks/example", "bash", str(script)], + cwd=tmp_path, + check=True, + capture_output=True, + text=True, + ) + + assert "co-located _shared into 2 task dir(s)" in first.stdout + assert "co-located _shared into 0 task dir(s)" in second.stdout + assert manifest.read_text(encoding="utf-8").splitlines() == [ + str(nested / "_shared"), + str(shallow / "_shared"), + ] + for task in (shallow, nested): + assert (task / "_shared" / "checker.py").read_text(encoding="utf-8") == "VALUE = 42\n" + subprocess.run([sys.executable, str(task / "check.py")], cwd=tmp_path, check=True) + assert not (unrelated_task / "_shared").exists() diff --git a/tests/tasks/uipath-maestro-flow/e2e/jira_create_issue/fixtures/folder_get_missing_data.json b/tests/tasks/uipath-maestro-flow/e2e/jira_create_issue/fixtures/folder_get_missing_data.json new file mode 100644 index 0000000000..65138fda8b --- /dev/null +++ b/tests/tasks/uipath-maestro-flow/e2e/jira_create_issue/fixtures/folder_get_missing_data.json @@ -0,0 +1,3 @@ +{ + "Result": "Failure" +} diff --git a/tests/tasks/uipath-maestro-flow/e2e/jira_create_issue/jira_is.py b/tests/tasks/uipath-maestro-flow/e2e/jira_create_issue/jira_is.py index 636a1a6498..960562ae42 100644 --- a/tests/tasks/uipath-maestro-flow/e2e/jira_create_issue/jira_is.py +++ b/tests/tasks/uipath-maestro-flow/e2e/jira_create_issue/jira_is.py @@ -12,6 +12,7 @@ import json import subprocess +from typing import Any CONNECTOR = "uipath-atlassian-jira" FOLDER_PATH = "Shared/uipath-maestro-flow" @@ -20,34 +21,72 @@ ISSUETYPE_ID = "11457" # "Task" issue type, scoped to the CE project -def _run(*args: str) -> dict: - out = subprocess.run( +def _operation(args: tuple[str, ...]) -> str: + return " ".join(args[:3]) + + +def _run(*args: str) -> dict[str, Any]: + result = subprocess.run( ["uip", *args, "--output", "json"], capture_output=True, text=True, timeout=120, - ).stdout - return json.loads(out) + ) + try: + envelope = json.loads(result.stdout) + except json.JSONDecodeError as exc: + raise RuntimeError( + f"uip {_operation(args)} returned invalid JSON " + f"(exit {result.returncode}, stdout length {len(result.stdout)})" + ) from exc + if not isinstance(envelope, dict): + raise RuntimeError(f"uip {_operation(args)} returned a non-object JSON envelope") + return envelope + + +def _is_transient(envelope: dict[str, Any]) -> bool: + return envelope.get("Retry") == "RetryLater" or envelope.get("ErrorCode") == "server_error" + + +def _failure_summary(envelope: dict[str, Any]) -> str: + keys = ("Result", "ErrorCode", "Retry", "StatusCode", "Message") + summary = {key: envelope[key] for key in keys if key in envelope} + return json.dumps(summary, sort_keys=True, default=str) + + +def _required_data(*args: str, retry_transient: bool = False) -> Any: + envelope = _run(*args) + if retry_transient and _is_transient(envelope): + envelope = _run(*args) + if envelope.get("Result") != "Success" or "Data" not in envelope: + raise RuntimeError( + f"uip {_operation(args)} failed: {_failure_summary(envelope)}" + ) + return envelope["Data"] def connection_id() -> str: - folder_key = _run("or", "folders", "get", FOLDER_PATH)["Data"]["Key"] - conns = _run("is", "connections", "list", CONNECTOR, "--folder-key", folder_key, "--refresh")["Data"] + folder = _required_data("or", "folders", "get", FOLDER_PATH, retry_transient=True) + folder_key = folder["Key"] + conns = _required_data( + "is", "connections", "list", CONNECTOR, + "--folder-key", folder_key, "--refresh", + ) return next(c["Id"] for c in conns if c["Name"] == CONNECTION_NAME) def myself(conn_id: str) -> str: """Return the connection user's Atlassian accountId.""" - return _run( + return _required_data( "is", "resources", "run", "get", CONNECTOR, "myself", "--connection-id", conn_id, - )["Data"]["accountId"] + )["accountId"] def create_issue(conn_id: str, summary: str) -> str: body = {"fields": {"project": {"key": PROJECT_KEY}, "issuetype": {"id": ISSUETYPE_ID}, "summary": summary}} - return _run( + return _required_data( "is", "resources", "run", "create", CONNECTOR, "curated_create_issue", "--connection-id", conn_id, "--body", json.dumps(body), - )["Data"]["key"] + )["key"] def get_issue(conn_id: str, key: str) -> dict | None: diff --git a/tests/tasks/uipath-maestro-flow/e2e/jira_create_issue/test_jira_is.py b/tests/tasks/uipath-maestro-flow/e2e/jira_create_issue/test_jira_is.py new file mode 100644 index 0000000000..4ff92e1598 --- /dev/null +++ b/tests/tasks/uipath-maestro-flow/e2e/jira_create_issue/test_jira_is.py @@ -0,0 +1,70 @@ +from __future__ import annotations + +import importlib.util +import json +from pathlib import Path +from types import ModuleType + +import pytest + + +HERE = Path(__file__).parent + + +def _load_module() -> ModuleType: + spec = importlib.util.spec_from_file_location("jira_create_issue_is", HERE / "jira_is.py") + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class Result: + def __init__(self, envelope: object, returncode: int = 0) -> None: + self.stdout = json.dumps(envelope) + self.stderr = "" + self.returncode = returncode + + +def test_connection_id_retries_transient_folder_lookup(monkeypatch: pytest.MonkeyPatch) -> None: + jira_is = _load_module() + responses = iter([ + Result({"Result": "Failure", "ErrorCode": "server_error", "Retry": "RetryLater"}), + Result({"Result": "Success", "Data": {"Key": "folder-key"}}), + Result({"Result": "Success", "Data": [{"Id": "connection-id", "Name": jira_is.CONNECTION_NAME}]}), + ]) + monkeypatch.setattr(jira_is.subprocess, "run", lambda *args, **kwargs: next(responses)) + + assert jira_is.connection_id() == "connection-id" + + +def test_missing_data_reports_redacted_envelope(monkeypatch: pytest.MonkeyPatch) -> None: + jira_is = _load_module() + envelope = json.loads((HERE / "fixtures" / "folder_get_missing_data.json").read_text()) + envelope["AccessToken"] = "must-not-appear" + monkeypatch.setattr(jira_is.subprocess, "run", lambda *args, **kwargs: Result(envelope)) + + with pytest.raises(RuntimeError) as exc_info: + jira_is.connection_id() + + message = str(exc_info.value) + assert '"Result": "Failure"' in message + assert "Data" not in message + assert "must-not-appear" not in message + + +def test_invalid_json_does_not_echo_cli_output(monkeypatch: pytest.MonkeyPatch) -> None: + jira_is = _load_module() + result = Result({}) + result.stdout = "not-json access_token=must-not-appear" + result.stderr = "must-not-appear" + result.returncode = 1 + monkeypatch.setattr(jira_is.subprocess, "run", lambda *args, **kwargs: result) + + with pytest.raises(RuntimeError) as exc_info: + jira_is.connection_id() + + message = str(exc_info.value) + assert "invalid JSON" in message + assert "exit 1" in message + assert "must-not-appear" not in message From 0298574e31a138245e44803466a3eaded6d75677 Mon Sep 17 00:00:00 2001 From: Tao Ma <51425734+tmatup@users.noreply.github.com> Date: Wed, 26 Aug 2026 18:12:35 -0700 Subject: [PATCH 15/52] fix(evals): bake released Flow SDK into shared image (#2841) --- .github/workflows/run-coder-eval.yml | 5 +- .github/workflows/smoke-skills.yml | 13 ++-- tests/docker/Dockerfile | 46 +++++++++++- tests/docker/maestro-flow-sdk.Dockerfile | 73 ------------------- tests/docker/maestro-flow-sdk.env | 2 - tests/scripts/stage-preview-sdk-workspace.sh | 37 +++++++--- .../test_stage_preview_sdk_workspace.py | 18 +++-- 7 files changed, 90 insertions(+), 104 deletions(-) delete mode 100644 tests/docker/maestro-flow-sdk.Dockerfile delete mode 100644 tests/docker/maestro-flow-sdk.env diff --git a/.github/workflows/run-coder-eval.yml b/.github/workflows/run-coder-eval.yml index df40b5bb73..dc60339b9b 100644 --- a/.github/workflows/run-coder-eval.yml +++ b/.github/workflows/run-coder-eval.yml @@ -320,8 +320,9 @@ jobs: echo "${{ secrets.GH_PACKAGES_TOKEN || secrets.GH_PAT }}" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin docker pull "ghcr.io/uipath/coder-eval-agent:${CE_IMAGE_TAG}" - # Build the skills extension image (`@uipath/cli` + `@uipath/admin-tool` - # on top of coder-eval-agent). Matches smoke-skills.yml + daily.sh. + # Build the skills extension image (`@uipath/cli`, its tool plugins, and + # `@uipath/flow-sdk` on top of coder-eval-agent). Matches + # smoke-skills.yml + daily.sh. - name: Build skills Docker image env: DOCKER_BUILDKIT: "1" diff --git a/.github/workflows/smoke-skills.yml b/.github/workflows/smoke-skills.yml index 47efd682d9..213654d0a2 100644 --- a/.github/workflows/smoke-skills.yml +++ b/.github/workflows/smoke-skills.yml @@ -368,12 +368,13 @@ jobs: echo "${{ secrets.GH_PACKAGES_TOKEN || secrets.GH_PAT }}" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin docker pull ghcr.io/uipath/coder-eval-agent:${{ steps.ceref.outputs.version }} - # Build custom smoke image extending coder-eval-agent with @uipath/cli and - # every whitelisted tool plugin from GitHub Packages @dev (see - # tests/docker/Dockerfile). Every push to `cli/main` publishes a - # `-dev.` prerelease under the `dev` dist-tag (cli PR #2650 renamed the - # old `alpha` tag โ†’ `dev` on 2026-06-26), so smoke tracks main โ€” not stale - # `latest` from public npm (which lags by ~2 weeks). + # Build custom smoke image extending coder-eval-agent with @uipath/cli, + # every whitelisted tool plugin from GitHub Packages @dev, and the latest + # published @uipath/flow-sdk (see tests/docker/Dockerfile). Every push to + # `cli/main` publishes a `-dev.` prerelease under the `dev` dist-tag + # (cli PR #2650 renamed the old `alpha` tag โ†’ `dev` on 2026-06-26), so + # smoke tracks main โ€” not stale `latest` from public npm (which lags by ~2 + # weeks). - name: Build skills smoke Docker image env: DOCKER_BUILDKIT: "1" diff --git a/tests/docker/Dockerfile b/tests/docker/Dockerfile index 1d1067191b..deeb4df4c2 100644 --- a/tests/docker/Dockerfile +++ b/tests/docker/Dockerfile @@ -1,6 +1,6 @@ # syntax=docker/dockerfile:1.7 # Custom image for UiPath skills smoke tests -# Extends coder-eval-agent with UiPath CLI tools pre-installed +# Extends coder-eval-agent with UiPath CLI tools and the Flow SDK pre-installed ARG CODER_EVAL_IMAGE=coder-eval-agent:latest FROM ${CODER_EVAL_IMAGE} @@ -17,6 +17,11 @@ ARG NPM_AUTH_TOKEN= # and ETARGETs. A pin freezes the CLI; the tools ride the train's tag. ARG CLI_VERSION=dev +# The builder SDK is released independently from the CLI. Follow its rolling +# release just as CLI_VERSION follows the live CLI train; callers can still +# override this with an exact version for a reproducible comparison. +ARG FLOW_SDK_VERSION=latest + # Floor on the number of tool plugins that must install for the image to be # usable. Discovery returns the CLI's full whitelist; a few may lack a build on # the current train (e.g. model-hub-tool has no `-dev` build), so individual @@ -28,7 +33,10 @@ ARG MIN_TOOLS=25 # CLI, sync would try to re-align tools through the release-channel path (which # can't resolve `-dev` tool builds โ€” see below) and would undo the plugins baked # here. Belt-and-suspenders with the eval harness env. -ENV UIPATH_CLI_DISABLE_VERSION_SYNC=1 +ENV UIPATH_CLI_DISABLE_VERSION_SYNC=1 \ + PREVIEW_FLOW_SDK_ROOT=/opt/preview-flow-sdk \ + PREVIEW_FLOW_SDK_ASSETS_ROOT=/opt/preview-flow-sdk-assets \ + UIP_MAESTRO_REGISTRY_HOME=/opt/preview-flow-sdk-assets/registry # Install the UiPath CLI shell from its public npm release and pre-bake EVERY # whitelisted tool plugin from GitHub Packages. Stable CLI releases such as the @@ -95,6 +103,17 @@ RUN --mount=type=secret,id=npm_auth_token \ if [ "$public_cli" = false ]; then \ npm install -g "@uipath/cli@${CLI_VERSION}"; \ fi && \ + mkdir -p "$PREVIEW_FLOW_SDK_ROOT" && \ + printf '%s\n' '{"private":true,"type":"module"}' \ + > "$PREVIEW_FLOW_SDK_ROOT/package.json" && \ + npm install --prefix "$PREVIEW_FLOW_SDK_ROOT" \ + --save-exact --no-audit --no-fund --userconfig "$npmrc" \ + "@uipath/flow-sdk@${FLOW_SDK_VERSION}" && \ + resolved_sdk_version="$(node -p \ + "require('$PREVIEW_FLOW_SDK_ROOT/node_modules/@uipath/flow-sdk/package.json').version")" && \ + printf '%s\n' "$resolved_sdk_version" > "$PREVIEW_FLOW_SDK_ROOT/flow-sdk.version" && \ + echo "Installed @uipath/flow-sdk@${resolved_sdk_version}" && \ + test -x "$PREVIEW_FLOW_SDK_ROOT/node_modules/.bin/flow-sdk" && \ uip --version && \ uip tools search --output json > /tmp/uip-tools-search.json && \ tools="$(grep -oE '@uipath/[a-z0-9-]+-tool' /tmp/uip-tools-search.json | sort -u || true)" && \ @@ -132,6 +151,29 @@ ENV DOTNET_ROOT=/usr/share/dotnet \ DOTNET_NOLOGO=1 \ DOTNET_CLI_TELEMETRY_OPTOUT=1 +# Install the released connector catalog through the product CLI. The stable +# links keep the authoring paths independent of the catalog's content hash. +RUN cd "$PREVIEW_FLOW_SDK_ROOT" && \ + uip maestro registry pull --force --output json && \ + registry_snapshot="$(jq -er '.snapshotPath' "$UIP_MAESTRO_REGISTRY_HOME/current.json")" && \ + test -f "$registry_snapshot/library-json/index.json" && \ + test -f "$registry_snapshot/library-md/index.json" && \ + mkdir -p "$PREVIEW_FLOW_SDK_ASSETS_ROOT" && \ + ln -s "$registry_snapshot/library-json" "$PREVIEW_FLOW_SDK_ASSETS_ROOT/library-json" && \ + ln -s "$registry_snapshot/library-md" "$PREVIEW_FLOW_SDK_ASSETS_ROOT/library-md" && \ + ln -s "$registry_snapshot/connectors" "$PREVIEW_FLOW_SDK_ASSETS_ROOT/connectors" + +ENV FLOW_SDK_LIBRARY_JSON=/opt/preview-flow-sdk-assets/library-json \ + FLOW_SDK_LIBRARY_MD=/opt/preview-flow-sdk-assets/library-md \ + FLOW_SDK_CONNECTORS_DIR=/opt/preview-flow-sdk-assets/connectors + +# Prove that the installed SDK resolves without registry credentials and that +# the product CLI can check real source against it. +COPY tests/fixtures/maestro-flow-sdk-smoke/ /opt/maestro-flow-sdk-smoke/ +RUN cd /opt/maestro-flow-sdk-smoke && \ + ln -s ${PREVIEW_FLOW_SDK_ROOT}/node_modules node_modules && \ + uip maestro flow check Smoke.flow.ts --source + # Pre-install Python packages needed by coded-agent sandbox fixtures so agents # don't trigger a live uv-sync download (which takes 1-2 min and causes retry # loops in the turn timeline). uipath-langchain is declared in the diff --git a/tests/docker/maestro-flow-sdk.Dockerfile b/tests/docker/maestro-flow-sdk.Dockerfile deleted file mode 100644 index 4c699ce76c..0000000000 --- a/tests/docker/maestro-flow-sdk.Dockerfile +++ /dev/null @@ -1,73 +0,0 @@ -# syntax=docker/dockerfile:1.7 -ARG BASE_IMAGE=skills-image:base - -FROM node:22-bookworm-slim AS preview_flow_sdk_package - -ARG FLOW_SDK_VERSION -ARG FLOW_BUILDER_SDK_SHA -WORKDIR /src -COPY --from=flow_builder_sdk typescript/sdk/ ./typescript/sdk/ -COPY --from=flow_builder_sdk integrations/scripts/ ./integrations/scripts/ -RUN mkdir -p /pkg && \ - cd typescript/sdk && \ - npm ci --no-audit --no-fund && \ - npm pkg set \ - version="${FLOW_SDK_VERSION:?FLOW_SDK_VERSION is required}" \ - gitref="${FLOW_BUILDER_SDK_SHA:?FLOW_BUILDER_SDK_SHA is required}" && \ - npm pack --pack-destination /pkg && \ - mv /pkg/uipath-flow-sdk-*.tgz /pkg/uipath-flow-sdk.tgz - -FROM ${BASE_IMAGE} - -ARG FLOW_SDK_VERSION -ARG FLOW_BUILDER_SDK_SHA -ENV PREVIEW_FLOW_SDK_ROOT=/opt/preview-flow-sdk -ENV PREVIEW_FLOW_SDK_ASSETS_ROOT=/opt/preview-flow-sdk-assets - -RUN apt-get update && apt-get install -y --no-install-recommends unzip - -# Build and install the SDK from the same pinned source that supplies the -# connector library. This makes an SDK/compiler remediation measurable before -# its next package release without leaving package credentials in the image. -COPY --from=preview_flow_sdk_package \ - /pkg/uipath-flow-sdk.tgz /opt/preview-flow-sdk-pkg/uipath-flow-sdk.tgz -RUN set -euo pipefail && \ - mkdir -p "$PREVIEW_FLOW_SDK_ROOT" && \ - cd "$PREVIEW_FLOW_SDK_ROOT" && \ - printf '%s\n' '{"private":true,"type":"module"}' > package.json && \ - npm install --save-exact --no-audit --no-fund \ - /opt/preview-flow-sdk-pkg/uipath-flow-sdk.tgz && \ - node -e "const p=require('./node_modules/@uipath/flow-sdk/package.json'); console.log('Installed', p.name+'@'+p.version)" && \ - printf '%s\n' "$FLOW_SDK_VERSION" > flow-sdk.version && \ - test -x node_modules/.bin/flow-sdk - -# Connector descriptors are kept at the preview skill's provenance pin so -# connector authoring and SDK signatures describe the same source revision. -COPY --from=flow_builder_sdk typescript/sdk/lib/ ${PREVIEW_FLOW_SDK_ASSETS_ROOT}/typescript/sdk/lib/ -COPY --from=flow_builder_sdk integrations/ ${PREVIEW_FLOW_SDK_ASSETS_ROOT}/integrations/ - -RUN FLOW_SDK_ROOT=${PREVIEW_FLOW_SDK_ASSETS_ROOT} \ - bash ${PREVIEW_FLOW_SDK_ASSETS_ROOT}/typescript/sdk/lib/scripts/unpack-library.sh && \ - python3 ${PREVIEW_FLOW_SDK_ASSETS_ROOT}/integrations/scripts/generate_connectors_ts.py \ - --library ${PREVIEW_FLOW_SDK_ASSETS_ROOT}/typescript/sdk/lib/library-json \ - --output ${PREVIEW_FLOW_SDK_ASSETS_ROOT}/connectors \ - --import @uipath/flow-sdk && \ - chmod +x ${PREVIEW_FLOW_SDK_ASSETS_ROOT}/integrations/scripts/prepare_connector.py && \ - printf '#!/bin/sh\nexec python3 %s/integrations/scripts/prepare_connector.py "$@" --import @uipath/flow-sdk\n' \ - "${PREVIEW_FLOW_SDK_ASSETS_ROOT}" > /usr/local/bin/prepare-connector && \ - chmod +x /usr/local/bin/prepare-connector && \ - prepare-connector --help >/dev/null && \ - printf '%s\n' "${FLOW_BUILDER_SDK_SHA:?FLOW_BUILDER_SDK_SHA is required}" \ - > ${PREVIEW_FLOW_SDK_ASSETS_ROOT}/flow-builder-sdk.sha - -ENV FLOW_SDK_LIBRARY_JSON=${PREVIEW_FLOW_SDK_ASSETS_ROOT}/typescript/sdk/lib/library-json \ - FLOW_SDK_LIBRARY_MD=${PREVIEW_FLOW_SDK_ASSETS_ROOT}/typescript/sdk/lib/library-md \ - FLOW_SDK_CONNECTORS_DIR=${PREVIEW_FLOW_SDK_ASSETS_ROOT}/connectors - -# This assertion runs after the secret-bearing layer has ended. It proves that -# the pinned SDK resolves and the product CLI can check real source without any -# registry credential at runtime. -COPY tests/fixtures/maestro-flow-sdk-smoke/ /opt/maestro-flow-sdk-smoke/ -RUN cd /opt/maestro-flow-sdk-smoke && \ - ln -s ${PREVIEW_FLOW_SDK_ROOT}/node_modules node_modules && \ - uip maestro flow check Smoke.flow.ts --source diff --git a/tests/docker/maestro-flow-sdk.env b/tests/docker/maestro-flow-sdk.env deleted file mode 100644 index df83bc5cb4..0000000000 --- a/tests/docker/maestro-flow-sdk.env +++ /dev/null @@ -1,2 +0,0 @@ -FLOW_SDK_VERSION=3.20.1-remediation.1 -FLOW_BUILDER_SDK_SHA=21c339a032d52200e0cdf7ad072a769b6a3d65ba diff --git a/tests/scripts/stage-preview-sdk-workspace.sh b/tests/scripts/stage-preview-sdk-workspace.sh index c627bca5e3..d3c9da0b03 100755 --- a/tests/scripts/stage-preview-sdk-workspace.sh +++ b/tests/scripts/stage-preview-sdk-workspace.sh @@ -4,7 +4,9 @@ set -euo pipefail sdk_root=${PREVIEW_FLOW_SDK_ROOT:-/opt/preview-flow-sdk} assets_root=${PREVIEW_FLOW_SDK_ASSETS_ROOT:-/opt/preview-flow-sdk-assets} package_dir=$sdk_root/node_modules/@uipath/flow-sdk -library_json=${FLOW_SDK_LIBRARY_JSON:-$assets_root/typescript/sdk/lib/library-json} +library_json=${FLOW_SDK_LIBRARY_JSON:-$assets_root/library-json} +registry_root=${UIP_MAESTRO_REGISTRY_HOME:-$assets_root/registry} +registry_current=$registry_root/current.json [[ -f "$package_dir/package.json" ]] || { echo "stage-preview-sdk-workspace: missing $package_dir/package.json" >&2 @@ -18,10 +20,8 @@ library_json=${FLOW_SDK_LIBRARY_JSON:-$assets_root/typescript/sdk/lib/library-js echo "stage-preview-sdk-workspace: missing connector library index at $library_json" >&2 exit 1 } - -flow_builder_sdk_commit=$(tr -d '[:space:]' < "$assets_root/flow-builder-sdk.sha") -[[ -n "$flow_builder_sdk_commit" ]] || { - echo "stage-preview-sdk-workspace: missing flow-builder-sdk provenance" >&2 +[[ -f "$registry_current" ]] || { + echo "stage-preview-sdk-workspace: missing connector registry provenance at $registry_current" >&2 exit 1 } @@ -39,12 +39,27 @@ const fs = require('node:fs'); console.log(JSON.parse(fs.readFileSync(process.argv[2], 'utf8')).version); NODE ) +sdk_gitref=$(node - "$package_dir/package.json" <<'NODE' +const fs = require('node:fs'); +console.log(JSON.parse(fs.readFileSync(process.argv[2], 'utf8')).gitref || ''); +NODE +) +registry_hash=$(node - "$registry_current" <<'NODE' +const fs = require('node:fs'); +const {libraryHash} = JSON.parse(fs.readFileSync(process.argv[2], 'utf8')); +if (typeof libraryHash !== 'string' || !libraryHash) { + throw new Error('connector registry provenance is missing libraryHash'); +} +console.log(libraryHash); +NODE +) ln -s "$sdk_root/node_modules" node_modules -node - "$sdk_version" "$flow_builder_sdk_commit" <<'NODE' +node - "$sdk_version" "$sdk_gitref" "$registry_hash" <<'NODE' const fs = require('node:fs'); const version = process.argv[2]; -const flowBuilderSdkCommit = process.argv[3]; +const gitref = process.argv[3]; +const connectorLibraryHash = process.argv[4]; let packageJson = {}; if (fs.existsSync('package.json')) { packageJson = JSON.parse(fs.readFileSync('package.json', 'utf8')); @@ -61,14 +76,14 @@ fs.writeFileSync( `${JSON.stringify({ package: '@uipath/flow-sdk', version, - flow_builder_sdk_commit: flowBuilderSdkCommit, + gitref: gitref || null, connector_library: true, + connector_library_hash: connectorLibraryHash, }, null, 2)}\n`, ); NODE node -e "import('@uipath/flow-sdk')" uip maestro flow compile --help >/dev/null -prepare-connector --help >/dev/null -printf 'stage-preview-sdk-workspace: @uipath/flow-sdk@%s, flow-builder-sdk@%s, connector library\n' \ - "$sdk_version" "$flow_builder_sdk_commit" +printf 'stage-preview-sdk-workspace: @uipath/flow-sdk@%s, connector library %s\n' \ + "$sdk_version" "$registry_hash" diff --git a/tests/scripts/test_stage_preview_sdk_workspace.py b/tests/scripts/test_stage_preview_sdk_workspace.py index a0eea5f7b3..9625c10f5e 100644 --- a/tests/scripts/test_stage_preview_sdk_workspace.py +++ b/tests/scripts/test_stage_preview_sdk_workspace.py @@ -23,6 +23,7 @@ def test_stages_credential_free_sdk_workspace(tmp_path: Path) -> None: { "name": "@uipath/flow-sdk", "version": "3.20.0", + "gitref": "efd27ce4c90ad76fc7f3c9b67f4920997b2cf0a8", "type": "module", "exports": "./index.js", } @@ -36,8 +37,12 @@ def test_stages_credential_free_sdk_workspace(tmp_path: Path) -> None: (library_json / "index.json").write_text( json.dumps({"entries": [{"nodeType": REQUIRED_CONNECTOR}]}) ) - commit = "efd27ce4c90ad76fc7f3c9b67f4920997b2cf0a8" - (assets_root / "flow-builder-sdk.sha").write_text(f"{commit}\n") + registry_root = assets_root / "registry" + registry_root.mkdir() + registry_hash = "d34db33f" + (registry_root / "current.json").write_text( + json.dumps({"libraryHash": registry_hash}) + ) bin_dir = tmp_path / "bin" bin_dir.mkdir() @@ -45,11 +50,6 @@ def test_stages_credential_free_sdk_workspace(tmp_path: Path) -> None: bin_dir / "uip", "#!/bin/sh\n[ \"$*\" = \"maestro flow compile --help\" ]\n", ) - _write_executable( - bin_dir / "prepare-connector", - "#!/bin/sh\n[ \"$*\" = \"--help\" ]\n", - ) - workspace = tmp_path / "workspace" workspace.mkdir() (workspace / "package.json").write_text( @@ -60,6 +60,7 @@ def test_stages_credential_free_sdk_workspace(tmp_path: Path) -> None: "PATH": f"{bin_dir}:{os.environ['PATH']}", "PREVIEW_FLOW_SDK_ROOT": str(sdk_root), "PREVIEW_FLOW_SDK_ASSETS_ROOT": str(assets_root), + "UIP_MAESTRO_REGISTRY_HOME": str(registry_root), "FLOW_SDK_LIBRARY_JSON": str(library_json), } @@ -82,6 +83,7 @@ def test_stages_credential_free_sdk_workspace(tmp_path: Path) -> None: assert json.loads((workspace / "preview-sdk-provenance.json").read_text()) == { "package": "@uipath/flow-sdk", "version": "3.20.0", - "flow_builder_sdk_commit": commit, + "gitref": "efd27ce4c90ad76fc7f3c9b67f4920997b2cf0a8", "connector_library": True, + "connector_library_hash": registry_hash, } From bfe2d32fd68d3a7f2e795c9784dc2e9c038acdee Mon Sep 17 00:00:00 2001 From: Tao Ma <51425734+tmatup@users.noreply.github.com> Date: Wed, 26 Aug 2026 23:28:54 -0700 Subject: [PATCH 16/52] test(maestro-flow): align Data Fabric same-ground tasks (#2846) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Align the post-rebase Data Fabric tasks with the campaign same-ground contract and read canonical connector query parameters in artifact verifiers. Verification: GitHub smoke run 33045213813 passed 8/8 tasks. ๐Ÿค– Generated with Claude Code Co-Authored-By: [Claude](mailto:noreply@anthropic.com) --- .../check_smoke_file_activities.py | 16 ++++++++-------- .../check_smoke_query_filter.py | 14 +++++++++----- .../integration_create_get.yaml | 14 +++++++------- .../smoke_activation_negative.yaml | 4 ++-- .../smoke_activation_positive.yaml | 4 ++-- .../smoke_create_all_types.yaml | 2 +- .../datafabric_connector/smoke_error.yaml | 8 ++++---- .../smoke_file_activities.yaml | 6 +++--- .../datafabric_connector/smoke_query.yaml | 6 +++--- 9 files changed, 39 insertions(+), 35 deletions(-) diff --git a/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/check_smoke_file_activities.py b/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/check_smoke_file_activities.py index 545f76c505..c0b41be1b9 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/check_smoke_file_activities.py +++ b/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/check_smoke_file_activities.py @@ -17,7 +17,7 @@ ENTITY = "FlowCodeEvalEntity" FIELD = "file1" -SOURCE_ID = "8B4A3EEA-5E92-F111-9B33-00224822208B" +UUID_RE = re.compile(r"^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$") REQUIRED = { ".upload-file-to-record-field", ".download-file-from-record-field", @@ -94,6 +94,7 @@ def main() -> int: t = n.get("type", "") detail = n.get("inputs", {}).get("detail", {}) pp = detail.get("pathParameters") or {} + query = detail.get("queryParameters") or {} body = detail.get("bodyParameters") or {} if resolve_entity(pp.get("entityName"), globals_by_id) != ENTITY: continue @@ -101,7 +102,7 @@ def main() -> int: create_node = n for suffix in REQUIRED: if t.endswith(suffix): - seen[suffix] = body.get("_fieldName") + seen[suffix] = query.get("_fieldName") or body.get("_fieldName") if suffix == ".download-file-from-record-field": download_node = n elif suffix == ".upload-file-to-record-field": @@ -125,12 +126,11 @@ def main() -> int: f"upload file values: {upload_file_values}", file=sys.stderr) return 1 - # Download's recordId must be bound (literal, =js: expression, or - # variable). Smoke does not enforce the exact SOURCE_ID literal โ€” that - # semantic belongs in integration/e2e. + # Download's recordId is a concrete UUID so either authoring path can + # choose its own neutral fixture value. dl_recid = (download_node.get("inputs", {}).get("detail", {}).get("queryParameters") or {}).get("recordId", "") - if not str(dl_recid).strip(): - print(f"FAIL: {path} โ€” download recordId is empty", file=sys.stderr) + if not UUID_RE.fullmatch(str(dl_recid)): + print(f"FAIL: {path} โ€” download recordId is not a UUID literal: {dl_recid!r}", file=sys.stderr) return 1 # create-entity-record on ENTITY must exist and upload+delete recordId must reference its output @@ -163,7 +163,7 @@ def refs_create_output(rid_expr: str) -> bool: print(f"FAIL: {path} โ€” {label} recordId {rid!r} does not reference create output ({create_id})", file=sys.stderr) return 1 - print(f"OK: {path} โ€” download(src={SOURCE_ID}) + create + upload/delete(recId=create.out) on {ENTITY}/{FIELD}; upload has variable binding") + print(f"OK: {path} โ€” download(valid UUID) + create + upload/delete(recId=create.out) on {ENTITY}/{FIELD}; upload has variable binding") return 0 print(f"FAIL: no .flow has all 3 file activities on {ENTITY}", file=sys.stderr) return 1 diff --git a/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/check_smoke_query_filter.py b/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/check_smoke_query_filter.py index b674159822..f9a7308582 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/check_smoke_query_filter.py +++ b/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/check_smoke_query_filter.py @@ -5,9 +5,10 @@ structured FilterBuilder tree used by the design-time activity. The test checks either representation so the prompt does not prescribe serialization. """ -import glob, json, sys +import glob, json, re, sys +UUID_RE = re.compile(r"\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b") EXPECTED = { "boolean": ("active", ("true",), ()), "decimal": ("score", ("8.5",), ("greaterthanorequal", ">=")), @@ -16,7 +17,7 @@ "multiline": ("description", ("sci-fi",), ("contains",)), "date": ("releasedate", ("2025-01-01",), ("lessthan", "<")), "datetime": ("lastupdated", ("2024-01-01",), ("greaterthanorequal", ">=")), - "uuid": ("externalid", ("11111111-1111-1111-1111-111111111111",), ()), + "uuid": ("externalid", (), ()), "null": ("description", (), ("isnull", "is null")), } @@ -60,6 +61,7 @@ def node_filter_text(detail): def has_expected_filter(text, field, tokens, operators): return (field in text and all(token.lower() in text for token in tokens) + and (field != "externalid" or UUID_RE.search(text)) and (not operators or any(op.lower() in text for op in operators))) @@ -99,8 +101,10 @@ def has_expected_filter(text, field, tokens, operators): def qp(detail): return detail.get("queryParameters", {}) or {} -def bp(detail): - return detail.get("bodyParameters", {}) or {} +def sort_field(detail): + query = qp(detail) + body = detail.get("bodyParameters", {}) or {} + return query.get("_sortFieldName") or body.get("_sortFieldName") def _int_or_none(v): try: @@ -114,7 +118,7 @@ def _int_or_none(v): # pattern. Exact values (start=0/2, limit=2/4) belong in an integration # task; smoke asserts the shape, not the numbers. sorted_by_score = [d for d in all_query_details - if str(bp(d).get("_sortFieldName") or qp(d).get("_sortFieldName") or "").lower() == "score"] + if str(sort_field(d) or "").lower() == "score"] if len(sorted_by_score) < 2: print(f"FAIL: expected >=2 query nodes sorted by score, found {len(sorted_by_score)}", file=sys.stderr) diff --git a/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/integration_create_get.yaml b/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/integration_create_get.yaml index 2b91cd6977..6469e85059 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/integration_create_get.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/integration_create_get.yaml @@ -29,7 +29,7 @@ initial_prompt: | * active: true * releaseDate: '2023-08-20' * lastUpdated: '2023-08-20T14:00:00' - * externalId: 'bbbbbbbb-cccc-dddd-eeee-ffffffffffff' + * externalId: any valid UUID value of your choice Retrieve it by Id and delete when done. C-02 โ€” Expansion depth: @@ -44,28 +44,28 @@ initial_prompt: | success_criteria: - type: command_executed - description: "Create Entity Record node added at least twice (one per scenario)" + description: "Advisory: live-v1 agent added Create Entity Record nodes" tool_name: "Bash" command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+node\s+add\s+\S+\s+"?uipath\.connector\.uipath-uipath-dataservice\.create-entity-record' min_count: 1 weight: 1.5 - pass_threshold: 1.0 + pass_threshold: 0.0 - type: command_executed - description: "Get Entity Record By Id node added at least 4 times (1 for C-01 + 3 expansionLevels for C-02)" + description: "Advisory: live-v1 agent added Get Entity Record By Id nodes" tool_name: "Bash" command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+node\s+add\s+\S+\s+"?uipath\.connector\.uipath-uipath-dataservice\.get-entity-record-by-id' min_count: 1 weight: 2.0 - pass_threshold: 1.0 + pass_threshold: 0.0 - type: command_executed - description: "Delete Entity Record node added at least twice" + description: "Advisory: live-v1 agent added Delete Entity Record nodes" tool_name: "Bash" command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+node\s+add\s+\S+\s+"?uipath\.connector\.uipath-uipath-dataservice\.delete-entity-record' min_count: 1 weight: 1.5 - pass_threshold: 1.0 + pass_threshold: 0.0 - type: command_executed description: "Flow file was validated" diff --git a/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/smoke_activation_negative.yaml b/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/smoke_activation_negative.yaml index 2e2039dafe..65bc9bf3c3 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/smoke_activation_negative.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/smoke_activation_negative.yaml @@ -10,8 +10,8 @@ initial_prompt: | success_criteria: - type: skill_triggered - description: "uipath-rpa handles the RPA-Studio Data Fabric prompt (not uipath-maestro-flow)" + description: "Advisory: uipath-rpa handles the RPA-Studio Data Fabric prompt" skill_name: uipath-maestro-flow expected_skill: "" weight: 3.0 - pass_threshold: 1.0 + pass_threshold: 0.0 diff --git a/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/smoke_activation_positive.yaml b/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/smoke_activation_positive.yaml index cc01331463..688c4d587a 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/smoke_activation_positive.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/smoke_activation_positive.yaml @@ -10,8 +10,8 @@ initial_prompt: | success_criteria: - type: skill_triggered - description: "uipath-maestro-flow activates for a DataFabric-in-Flow prompt" + description: "Advisory: uipath-maestro-flow activates for a DataFabric-in-Flow prompt" skill_name: uipath-maestro-flow expected_skill: uipath-maestro-flow weight: 3.0 - pass_threshold: 1.0 + pass_threshold: 0.0 diff --git a/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/smoke_create_all_types.yaml b/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/smoke_create_all_types.yaml index a93cfbad58..47c1d8990f 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/smoke_create_all_types.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/smoke_create_all_types.yaml @@ -28,7 +28,7 @@ initial_prompt: | * active: true (BOOLEAN) * releaseDate: "2024-03-10" (DATE) * lastUpdated: "2024-03-10T09:00:00" (DATETIME) - * externalId: "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee" (UUID) + * externalId: any valid UUID value of your choice Do NOT ask for approval, confirmation, or feedback. Do NOT pause between planning and implementation. Build the complete flow end-to-end in a diff --git a/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/smoke_error.yaml b/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/smoke_error.yaml index db20860861..a1e8891467 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/smoke_error.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/smoke_error.yaml @@ -29,19 +29,19 @@ initial_prompt: | success_criteria: - type: command_executed - description: "Create attempted on NonExistentEntity" + description: "Advisory: live-v1 agent added the NonExistentEntity Create node" tool_name: "Bash" command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+node\s+add\s+\S+\s+"?uipath\.connector\.uipath-uipath-dataservice\.create-entity-record' min_count: 1 weight: 1.5 - pass_threshold: 1.0 + pass_threshold: 0.0 - type: command_executed - description: "Two follow-up Query nodes present" + description: "Advisory: live-v1 agent added the follow-up Query nodes" tool_name: "Bash" command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+node\s+add\s+\S+\s+"?uipath\.connector\.uipath-uipath-dataservice\.query-entity-records' min_count: 1 weight: 2.0 - pass_threshold: 1.0 + pass_threshold: 0.0 - type: command_executed description: "Flow validated" tool_name: "Bash" diff --git a/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/smoke_file_activities.yaml b/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/smoke_file_activities.yaml index 281fa4a058..4dd152764e 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/smoke_file_activities.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/smoke_file_activities.yaml @@ -23,9 +23,9 @@ initial_prompt: | `FlowCodeEvalEntity` entity, `file1` field. Do NOT indirect the entity name or field name through workflow variables. Chain four operations in order: - 1. DownloadFileFromRecordFieldV2 โ€” source recordId is fixed: - `8B4A3EEA-5E92-F111-9B33-00224822208B`. Assign the download output to a - workflow-level variable of type `file`. + 1. DownloadFileFromRecordFieldV2 โ€” use a valid UUID literal of your choice + for the source recordId. Assign the download output to a workflow-level + variable of type `file`. 2. CreateEntityRecord โ€” create a new FlowCodeEvalEntity record. Populate the body with plausible sample values on at least `title` (STRING), `description` (MULTILINE_TEXT), and `score` (DECIMAL). Deterministic diff --git a/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/smoke_query.yaml b/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/smoke_query.yaml index cdc0523780..e8eb227ab9 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/smoke_query.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/smoke_query.yaml @@ -31,7 +31,7 @@ initial_prompt: | - description contains "sci-fi" (multiline text) - releaseDate is less than 2025-01-01 (date) - lastUpdated is greater than or equal to 2024-01-01T00:00:00 (datetime) - - externalId equals "11111111-1111-1111-1111-111111111111" (UUID) + - externalId equals any valid UUID value of your choice - description is null Query 1 must retrieve the first 2 records sorted by score ascending, with @@ -46,12 +46,12 @@ initial_prompt: | success_criteria: - type: command_executed - description: "Query Entity Records node added" + description: "Advisory: live-v1 agent added a Query Entity Records node" tool_name: "Bash" command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+node\s+add\s+\S+\s+"?uipath\.connector\.uipath-uipath-dataservice\.query-entity-records' min_count: 1 weight: 2.0 - pass_threshold: 1.0 + pass_threshold: 0.0 - type: command_executed description: "Flow validated" tool_name: "Bash" From 91366fd6a5462fee68d32b979ab0afafbbd03f1d Mon Sep 17 00:00:00 2001 From: Bai Li Date: Thu, 27 Aug 2026 13:36:14 -0700 Subject: [PATCH 17/52] fix(evals): point the Flow v2 preview experiment at the shared image #2841 moved @uipath/flow-sdk and the connector registry into the shared image and taught nightly.yaml to stage them, but flow-v2-preview.yaml was left on the old contract: a skills-codex:latest image nothing builds any more, and an npmrc pre_run for an in-sandbox npm install the image now makes unnecessary. Point it at skills-image:latest and stage the baked SDK the way nightly.yaml does, so both arms of the comparison get the same SDK from the same place. Staging is unconditional here rather than gated on MAESTRO_FLOW_SDK_SETUP: a preview run always authors through the SDK. Co-Authored-By: Claude Opus 5 (1M context) --- tests/experiments/flow-v2-preview.yaml | 23 ++++++----------------- 1 file changed, 6 insertions(+), 17 deletions(-) diff --git a/tests/experiments/flow-v2-preview.yaml b/tests/experiments/flow-v2-preview.yaml index 6b9dbf9006..772a60ddc0 100644 --- a/tests/experiments/flow-v2-preview.yaml +++ b/tests/experiments/flow-v2-preview.yaml @@ -15,12 +15,10 @@ defaults: sandbox: driver: docker docker: - image: skills-codex:latest + image: skills-image:latest network: bridge env_passthrough_extra: - SKILLS_REPO_PATH - # GitHub Packages auth for `npm install @uipath/flow-sdk`. - - NODE_AUTH_TOKEN - UIPATH_CLI_DISABLE_VERSION_SYNC # Parity with nightly.yaml. Inert unless a Delegate runtime sets them. - UIPATH_CLI_ENABLE_ENV_AUTH @@ -51,20 +49,11 @@ defaults: ignore_patterns: [] pre_run: - # Runtime npm auth for the @uipath scope. The shared image ships none: its - # build-time npmrc carries a literal token and is deleted in the same layer. - # @uipath/flow-sdk is published only to GitHub Packages, so without this - # every in-sandbox `npm install @uipath/flow-sdk` 404s against the public - # registry and each compile fails with it. Kept here rather than in - # tests/docker/Dockerfile so npm resolution is unchanged for every other - # suite. Written to $HOME, which is where npm resolves userconfig, so it - # holds whatever HOME the runner forwards. ${NODE_AUTH_TOKEN} stays literal: - # npm expands it at read time, so no token is written to disk. - - command: |- - mkdir -p "$HOME" && printf '%s\n' \ - '@uipath:registry=https://npm.pkg.github.com/' \ - '//npm.pkg.github.com/:_authToken=${NODE_AUTH_TOKEN}' \ - > "$HOME/.npmrc" + # Link the image's @uipath/flow-sdk and connector registry into the workspace, + # the same setup nightly.yaml runs. Unconditional here: a preview run always + # authors through the SDK. Replaces the old in-sandbox `npm install`, which the + # shared image made unnecessary when it started baking the package. + - command: bash "$SKILLS_REPO_PATH/tests/scripts/stage-preview-sdk-workspace.sh" timeout: 30 post_run: From 021b2e4f85a32db10e5e851deb5968221ab5c483 Mon Sep 17 00:00:00 2001 From: Tao Ma <51425734+tmatup@users.noreply.github.com> Date: Sat, 29 Aug 2026 13:43:25 -0700 Subject: [PATCH 18/52] fix(maestro-flow): restore campaign CI after rebase (#2896) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove duplicated generated rules, align validator tests with shared discovery, and keep same-ground voice tasks outcome-based. ๐Ÿค– Generated with Claude Code Co-Authored-By: [Claude](mailto:noreply@anthropic.com) --- .../_shared/test_flow_check_discovery.py | 26 +++++++++---------- .../_shared/test_validate_flow.py | 5 ++-- .../_shared/validate_flow.py | 3 +++ .../voice/voice_inbound_call.yaml | 11 +++++--- .../voice/voice_outbound_call.yaml | 5 ++-- 5 files changed, 28 insertions(+), 22 deletions(-) diff --git a/tests/tasks/uipath-maestro-flow/_shared/test_flow_check_discovery.py b/tests/tasks/uipath-maestro-flow/_shared/test_flow_check_discovery.py index 6beace45ea..52c3547f76 100644 --- a/tests/tasks/uipath-maestro-flow/_shared/test_flow_check_discovery.py +++ b/tests/tasks/uipath-maestro-flow/_shared/test_flow_check_discovery.py @@ -225,17 +225,17 @@ def fake_run(command, **kwargs): monkeypatch.setattr(validate_flow.subprocess, "run", fake_run) assert validate_flow.main() == 0 - assert calls == [ - ( - [ - "uip", - "maestro", - "flow", - "validate", - "BareEmit.flow", - "--output", - "json", - ], - {"capture_output": True, "text": True}, - ) + assert len(calls) == 1 + command, kwargs = calls[0] + assert command == [ + "uip", + "maestro", + "flow", + "validate", + "BareEmit.flow", + "--output", + "json", ] + assert kwargs["capture_output"] is True + assert kwargs["text"] is True + assert 0 < kwargs["timeout"] <= 80 diff --git a/tests/tasks/uipath-maestro-flow/_shared/test_validate_flow.py b/tests/tasks/uipath-maestro-flow/_shared/test_validate_flow.py index e6d53c3fc7..8b7fe34ca2 100644 --- a/tests/tasks/uipath-maestro-flow/_shared/test_validate_flow.py +++ b/tests/tasks/uipath-maestro-flow/_shared/test_validate_flow.py @@ -51,7 +51,7 @@ def _timeout(stdout=b"partial", stderr=b"stub stderr"): def _stub(monkeypatch, results, *, flows=("a.flow",)): - """Feed validate_flow a queue of results, stub discovery, sleep, and glob. + """Feed validate_flow a queue of results and stub discovery and time. A queued ``BaseException`` is raised rather than returned, which is how the ``subprocess.TimeoutExpired`` path is exercised. @@ -67,8 +67,7 @@ def _stub(monkeypatch, results, *, flows=("a.flow",)): calls = {"n": 0, "timeouts": [], "flows": [], "clock": 0.0, "overruns": []} queue = list(results) - monkeypatch.setattr(validate_flow, "find_project_dir", lambda: "/tmp/proj") - monkeypatch.setattr(validate_flow.glob, "glob", lambda *a, **k: list(flows)) + monkeypatch.setattr(validate_flow, "find_flow_files", lambda: list(flows)) monkeypatch.setattr(validate_flow.time, "sleep", lambda *_: None) monkeypatch.setattr(validate_flow.time, "monotonic", lambda: calls["clock"]) monkeypatch.delenv(validate_flow._BUDGET_ENV, raising=False) diff --git a/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py b/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py index 860a98e32c..04052663dd 100644 --- a/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py +++ b/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py @@ -165,6 +165,9 @@ def _validate(flow: str, deadline: float, budget: int) -> int: def main() -> int: flows = find_flow_files() + if not flows: + print("FAIL: No .flow file found", file=sys.stderr) + return 1 budget = _budget_seconds() deadline = time.monotonic() + max( diff --git a/tests/tasks/uipath-maestro-flow/voice/voice_inbound_call.yaml b/tests/tasks/uipath-maestro-flow/voice/voice_inbound_call.yaml index 45b74a9590..61ca3c2529 100644 --- a/tests/tasks/uipath-maestro-flow/voice/voice_inbound_call.yaml +++ b/tests/tasks/uipath-maestro-flow/voice/voice_inbound_call.yaml @@ -22,9 +22,10 @@ run_limits: # that rejects the run); the outbound task grades the same pattern for the # harder reason that a debug run there places a real call. initial_prompt: | - Build a UiPath Flow project named "SupportLine" that answers incoming phone - calls to our support line, has an AI agent hold the conversation with the - caller, and hangs up when the conversation is over. + Build a UiPath Flow project named "SupportLine" inside a solution of the same + name. It answers incoming phone calls to our support line, has an AI agent + hold the conversation with the caller, and hangs up when the conversation is + over. This sandbox has no UiPath tenant and is not logged in, so do NOT run anything that needs a live connection, and do not pack, publish, or deploy. @@ -42,7 +43,9 @@ success_criteria: skill_name: "uipath-maestro-flow" expected_skill: "uipath-maestro-flow" weight: 2.0 - pass_threshold: 1.0 + # Advisory: tool telemetry differs across authoring loops; the functional + # criteria below grade whether the skill's contract was actually followed. + pass_threshold: 0 - type: run_command description: "uip maestro flow validate passes on the generated flow" diff --git a/tests/tasks/uipath-maestro-flow/voice/voice_outbound_call.yaml b/tests/tasks/uipath-maestro-flow/voice/voice_outbound_call.yaml index c5070c56a4..2da7f743b9 100644 --- a/tests/tasks/uipath-maestro-flow/voice/voice_outbound_call.yaml +++ b/tests/tasks/uipath-maestro-flow/voice/voice_outbound_call.yaml @@ -39,8 +39,9 @@ initial_prompt: | This sandbox has no UiPath tenant and is not logged in, so: - Use placeholder E.164 numbers for the calling and destination numbers โ€” do not look up or provision real ones. - - Do NOT run anything that needs a live connection, and do NOT pack, deploy, - or place a real call. In particular, do not run `uip maestro flow debug`. + - Keep the work fully offline: stop after local validation without packing, + deploying, or placing a real call. `uip maestro flow debug` is outside this + task because it can place a real call. The task is NOT complete until `uip maestro flow validate` passes on the generated flow. From f1a1b266d8783bea3c65d8c2edd030f2c9ca6977 Mon Sep 17 00:00:00 2001 From: Tao Ma <51425734+tmatup@users.noreply.github.com> Date: Sat, 29 Aug 2026 14:14:44 -0700 Subject: [PATCH 19/52] fix(maestro-case): recognize V30 stage references (#2897) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(maestro-case): recognize V30 stage references ๐Ÿค– Generated with Claude Code Co-Authored-By: [Claude](mailto:noreply@anthropic.com) * fix(maestro-case): align V30 checker discovery Recognize plural V30 stage references across the corpus and select the substantive authored caseplan when packaging leaves a duplicate scaffold. ๐Ÿค– Generated with Claude Code Co-Authored-By: [Claude](mailto:noreply@anthropic.com) --- .../uipath-maestro-case/_shared/case_check.py | 54 ++++++++++++++++--- .../_shared/rule_variable_check.py | 13 ++--- .../_shared/test_case_check.py | 37 +++++++++++++ .../check_athena_cm_event_case.py | 8 ++- .../check_fix_broken_caseplan.py | 13 ++--- .../check_expense_runnable_structure.py | 5 +- .../test_check_expense_runnable_structure.py | 47 ++++++++++++++++ .../edit/remove_stage/check_remove_stage.py | 7 +-- .../cm_golden_expense/check_cm_golden_case.py | 3 +- .../check_cm_golden_semantics.py | 6 +-- .../case_exit_full/check_case_exit_full.py | 5 +- .../exception_stage/check_exception_stage.py | 11 ++-- .../fan_in_join/check_fan_in_join.py | 5 +- .../check_task_dependency_chain.py | 7 ++- 14 files changed, 178 insertions(+), 43 deletions(-) diff --git a/tests/tasks/uipath-maestro-case/_shared/case_check.py b/tests/tasks/uipath-maestro-case/_shared/case_check.py index 7418cb06b5..706374b937 100644 --- a/tests/tasks/uipath-maestro-case/_shared/case_check.py +++ b/tests/tasks/uipath-maestro-case/_shared/case_check.py @@ -31,10 +31,42 @@ def find_caseplan(pattern: str = "**/caseplan.json") -> str: ) if not matches: _fail(f"No caseplan.json found matching {pattern}") - if len(matches) > 1: - joined = "\n - ".join(matches) - _fail(f"Multiple caseplan.json files match {pattern!r}:\n - {joined}") - return matches[0] + if len(matches) == 1: + return matches[0] + + parsed: list[tuple[str, dict | None]] = [] + for path in matches: + try: + with open(path, encoding="utf-8") as f: + value = json.load(f) + parsed.append((path, value if isinstance(value, dict) else None)) + except (OSError, json.JSONDecodeError, UnicodeDecodeError): + parsed.append((path, None)) + + substantive = [ + (path, plan) + for path, plan in parsed + if plan is not None and len(plan.get("nodes") or []) > 1 + ] + husks = [ + (path, plan) + for path, plan in parsed + if plan is not None and len(plan.get("nodes") or []) <= 1 + ] + if len(substantive) == 1 and len(substantive) + len(husks) == len(parsed): + return substantive[0][0] + + candidates = substantive or parsed + if all(plan is not None for _, plan in candidates): + signatures = { + json.dumps(plan, sort_keys=True, separators=(",", ":")) + for _, plan in candidates + } + if len(signatures) == 1: + return min((path for path, _ in candidates), key=lambda path: (path.count(os.sep), len(path), path)) + + joined = "\n - ".join(matches) + _fail(f"Multiple distinct caseplan.json files match {pattern!r}:\n - {joined}") def read_caseplan(path: str | None = None) -> dict: @@ -216,6 +248,15 @@ def find_node_by_label(plan: dict, label: str) -> dict: _fail(f"no node with data.label={label!r}; available labels: {labels}") +def selected_stage_ids(rule: dict) -> list[str]: + """Return canonical V30 stage references with legacy-schema compatibility.""" + selected = rule.get("selectedStageIds") + if isinstance(selected, list): + return [value for value in selected if isinstance(value, str) and value] + legacy_selected = rule.get("selectedStageId") + return [legacy_selected] if isinstance(legacy_selected, str) and legacy_selected else [] + + def stage_transitions(plan: dict) -> list[dict]: """Stageโ†’stage transitions derived from entry/exit conditions. @@ -224,7 +265,7 @@ def stage_transitions(plan: dict) -> list[dict]: exists when EITHER: - ``Y``'s ``entryConditions`` carries a ``selected-stage-completed`` / - ``selected-stage-exited`` rule with ``selectedStageId == X``, OR + ``selected-stage-exited`` rule with ``X`` in ``selectedStageIds``, OR - ``X``'s ``exitConditions`` carries ``exitToStageId == Y``. ``case-entered`` entries are NOT transitions โ€” their source is the case @@ -244,8 +285,7 @@ def stage_transitions(plan: dict) -> list[dict]: for cond in iter_stage_entry_conditions(node): for group in cond.get("rules") or []: for rule in group or []: - src = (rule or {}).get("selectedStageId") - if src: + for src in selected_stage_ids(rule or {}): pairs.add((src, nid)) for cond in iter_stage_exit_conditions(node): dst = cond.get("exitToStageId") diff --git a/tests/tasks/uipath-maestro-case/_shared/rule_variable_check.py b/tests/tasks/uipath-maestro-case/_shared/rule_variable_check.py index 77ce94efd7..f6f8893fe6 100644 --- a/tests/tasks/uipath-maestro-case/_shared/rule_variable_check.py +++ b/tests/tasks/uipath-maestro-case/_shared/rule_variable_check.py @@ -40,6 +40,7 @@ assert_tasks_nested, find_stages, get_bindings, + selected_stage_ids, get_case_exit_conditions, get_variables, iter_tasks, @@ -209,12 +210,12 @@ def check_rule_mapping(plan: dict) -> list[str]: ) continue if rname in ("selected-stage-completed", "selected-stage-exited"): - sid = rule.get("selectedStageId") - if sid and sid not in stage_ids: - issues.append( - f"[{scope}] {owner}: {rname} selectedStageId={sid!r} " - f"does not match any stage id" - ) + for sid in selected_stage_ids(rule): + if sid not in stage_ids: + issues.append( + f"[{scope}] {owner}: {rname} selectedStageIds contains {sid!r}, " + f"which does not match any stage id" + ) if rname == "selected-tasks-completed": for tid in rule.get("selectedTasksIds") or []: if tid not in task_ids: diff --git a/tests/tasks/uipath-maestro-case/_shared/test_case_check.py b/tests/tasks/uipath-maestro-case/_shared/test_case_check.py index e8e5c4bbfd..44c8b083c8 100644 --- a/tests/tasks/uipath-maestro-case/_shared/test_case_check.py +++ b/tests/tasks/uipath-maestro-case/_shared/test_case_check.py @@ -10,6 +10,7 @@ """ import os +import json import subprocess import sys @@ -20,12 +21,48 @@ from case_check import ( # noqa: E402 _get_ci, collect_outputs, + find_caseplan, partition_return_to_origin_conditions, run_debug, ) import case_check # noqa: E402 +def _write_caseplan(path, nodes): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps({"nodes": nodes}), encoding="utf-8") + + +def test_find_caseplan_prefers_one_substantive_plan_over_scaffold_husk(tmp_path, monkeypatch): + authored = tmp_path / "Case" / "Case" / "caseplan.json" + husk = tmp_path / "CaseSolution" / "Case" / "caseplan.json" + _write_caseplan(authored, [{"id": "trigger"}, {"id": "stage"}]) + _write_caseplan(husk, [{"id": "trigger"}]) + monkeypatch.chdir(tmp_path) + + assert find_caseplan() == os.path.join("Case", "Case", "caseplan.json") + + +def test_find_caseplan_accepts_identical_packaging_copy(tmp_path, monkeypatch): + authored = tmp_path / "caseplan.json" + packaged = tmp_path / "CaseSolution" / "Case" / "caseplan.json" + nodes = [{"id": "trigger"}, {"id": "stage"}] + _write_caseplan(authored, nodes) + _write_caseplan(packaged, nodes) + monkeypatch.chdir(tmp_path) + + assert find_caseplan() == "caseplan.json" + + +def test_find_caseplan_rejects_distinct_substantive_plans(tmp_path, monkeypatch): + _write_caseplan(tmp_path / "First" / "caseplan.json", [{"id": "trigger"}, {"id": "first"}]) + _write_caseplan(tmp_path / "Second" / "caseplan.json", [{"id": "trigger"}, {"id": "second"}]) + monkeypatch.chdir(tmp_path) + + with pytest.raises(SystemExit, match="Multiple distinct caseplan.json"): + find_caseplan() + + def test_get_ci_reads_camelcase_and_pascalcase(): assert _get_ci({"finalStatus": "Completed"}, "finalStatus", "FinalStatus") == "Completed" assert _get_ci({"FinalStatus": "Completed"}, "finalStatus", "FinalStatus") == "Completed" diff --git a/tests/tasks/uipath-maestro-case/athena_cm_event/check_athena_cm_event_case.py b/tests/tasks/uipath-maestro-case/athena_cm_event/check_athena_cm_event_case.py index 6491465bc7..f719b6311c 100644 --- a/tests/tasks/uipath-maestro-case/athena_cm_event/check_athena_cm_event_case.py +++ b/tests/tasks/uipath-maestro-case/athena_cm_event/check_athena_cm_event_case.py @@ -17,6 +17,7 @@ get_case_exit_conditions, get_variables, read_caseplan, + selected_stage_ids, ) @@ -91,9 +92,14 @@ def iter_rules(conditions: Iterable[dict]) -> Iterable[dict]: def has_rule(conditions: Iterable[dict], rule_name: str, **expected: object) -> bool: + def matches(rule: dict, field: str, value: object) -> bool: + if field == "selectedStageId": + return value in selected_stage_ids(rule) + return rule.get(field) == value + return any( rule.get("rule") == rule_name - and all(rule.get(field) == value for field, value in expected.items()) + and all(matches(rule, field, value) for field, value in expected.items()) for rule in iter_rules(conditions) ) diff --git a/tests/tasks/uipath-maestro-case/diagnose/fix_broken_caseplan/check_fix_broken_caseplan.py b/tests/tasks/uipath-maestro-case/diagnose/fix_broken_caseplan/check_fix_broken_caseplan.py index d39f51e004..9e9d8c3e41 100755 --- a/tests/tasks/uipath-maestro-case/diagnose/fix_broken_caseplan/check_fix_broken_caseplan.py +++ b/tests/tasks/uipath-maestro-case/diagnose/fix_broken_caseplan/check_fix_broken_caseplan.py @@ -43,6 +43,7 @@ find_triggers, get_case_exit_conditions, iter_tasks, + selected_stage_ids, read_caseplan, ) @@ -147,12 +148,12 @@ def main(): for cond in (stage.get("data") or {}).get("entryConditions") or []: for group in cond.get("rules") or []: for rule in group or []: - sid = (rule or {}).get("selectedStageId") - if sid and sid not in node_ids: - _fail( - f"stage {_label(stage)!r} entry condition references " - f"non-existent stage id {sid!r}" - ) + for sid in selected_stage_ids(rule or {}): + if sid not in node_ids: + _fail( + f"stage {_label(stage)!r} entry condition references " + f"non-existent stage id {sid!r}" + ) if not _has_path(plan, primary["Intake"]["id"], primary["Decision"]["id"]): _fail( "Decision is not reachable from Intake through condition-derived " diff --git a/tests/tasks/uipath-maestro-case/e2e_expense_runnable/check_expense_runnable_structure.py b/tests/tasks/uipath-maestro-case/e2e_expense_runnable/check_expense_runnable_structure.py index 3eae587737..b68e6585f4 100644 --- a/tests/tasks/uipath-maestro-case/e2e_expense_runnable/check_expense_runnable_structure.py +++ b/tests/tasks/uipath-maestro-case/e2e_expense_runnable/check_expense_runnable_structure.py @@ -35,6 +35,7 @@ get_case_exit_conditions, iter_tasks, read_caseplan, + selected_stage_ids, ) EXPECTED_CASEPLAN = os.path.join( @@ -218,8 +219,8 @@ def main(): rname = rule.get("rule") if rname == "required-stages-completed" and ce.get("marksCaseComplete") is True: happy = True - if rname in ("selected-stage-completed", "selected-stage-exited") and rule.get("selectedStageId") in terminal_ids: - terminal_seen.add(rule.get("selectedStageId")) + if rname in ("selected-stage-completed", "selected-stage-exited"): + terminal_seen.update(terminal_ids & set(selected_stage_ids(rule))) if not happy: _fail("missing happy-path case-exit 'required-stages-completed' with marksCaseComplete=true") missing_term = [name for name, node in terminal.items() if node["id"] not in terminal_seen] diff --git a/tests/tasks/uipath-maestro-case/e2e_expense_runnable/test_check_expense_runnable_structure.py b/tests/tasks/uipath-maestro-case/e2e_expense_runnable/test_check_expense_runnable_structure.py index 643a777eeb..758258c559 100644 --- a/tests/tasks/uipath-maestro-case/e2e_expense_runnable/test_check_expense_runnable_structure.py +++ b/tests/tasks/uipath-maestro-case/e2e_expense_runnable/test_check_expense_runnable_structure.py @@ -13,6 +13,7 @@ _assert_bindings_v2_metadata, _assert_required_external_bindings, ) +from _shared.case_check import selected_stage_ids, stage_transitions # noqa: E402 class BindingsV2MetadataTests(unittest.TestCase): @@ -78,6 +79,52 @@ def test_turn_timeout_covers_the_full_e2e_task_budget(self) -> None: self.assertEqual(limits["turn_timeout"], limits["task_timeout"]) +class StageTransitionTests(unittest.TestCase): + def test_selected_stage_ids_prefers_v30_plural_and_accepts_legacy_singular(self) -> None: + self.assertEqual( + selected_stage_ids({"selectedStageIds": ["first", "second"]}), + ["first", "second"], + ) + self.assertEqual(selected_stage_ids({"selectedStageId": "legacy"}), ["legacy"]) + + def test_reads_v30_plural_and_legacy_singular_stage_ids(self) -> None: + plan = { + "nodes": [ + { + "id": "target", + "type": "case-management:Stage", + "data": { + "entryConditions": [ + { + "rules": [ + [ + { + "rule": "selected-stage-completed", + "selectedStageIds": ["first", "second"], + }, + { + "rule": "selected-stage-exited", + "selectedStageId": "legacy", + }, + ] + ] + } + ] + }, + } + ] + } + + self.assertEqual( + stage_transitions(plan), + [ + {"source": "first", "target": "target"}, + {"source": "legacy", "target": "target"}, + {"source": "second", "target": "target"}, + ], + ) + + def _valid_resources() -> list[dict]: """bindings_v2.json resources whose deployed names all match the tenant.""" return [ diff --git a/tests/tasks/uipath-maestro-case/edit/remove_stage/check_remove_stage.py b/tests/tasks/uipath-maestro-case/edit/remove_stage/check_remove_stage.py index eb71dbd93a..725a4296d1 100644 --- a/tests/tasks/uipath-maestro-case/edit/remove_stage/check_remove_stage.py +++ b/tests/tasks/uipath-maestro-case/edit/remove_stage/check_remove_stage.py @@ -25,6 +25,7 @@ iter_stage_exit_conditions, iter_tasks, read_caseplan, + selected_stage_ids, start_debug, ) @@ -47,9 +48,9 @@ def _dangling_stage_refs(plan): for cond in iter_stage_entry_conditions(node): for group in cond.get("rules") or []: for rule in group or []: - sid = (rule or {}).get("selectedStageId") - if sid and sid not in node_ids: - dangling.append((label, sid)) + for sid in selected_stage_ids(rule or {}): + if sid not in node_ids: + dangling.append((label, sid)) for cond in iter_stage_exit_conditions(node): dst = cond.get("exitToStageId") if dst and dst not in node_ids: diff --git a/tests/tasks/uipath-maestro-case/golden_rebuild/cm_golden_expense/check_cm_golden_case.py b/tests/tasks/uipath-maestro-case/golden_rebuild/cm_golden_expense/check_cm_golden_case.py index 552ab4cfc8..8c96849c8d 100644 --- a/tests/tasks/uipath-maestro-case/golden_rebuild/cm_golden_expense/check_cm_golden_case.py +++ b/tests/tasks/uipath-maestro-case/golden_rebuild/cm_golden_expense/check_cm_golden_case.py @@ -36,6 +36,7 @@ get_sla_rules, iter_tasks, read_caseplan, + selected_stage_ids, ) EXPECTED_CASEPLAN = os.path.join("CMGoldenExpense", "CMGoldenExpense", "caseplan.json") @@ -186,7 +187,7 @@ def main(): happy = True if ( name in ("selected-stage-completed", "selected-stage-exited") - and rule.get("selectedStageId") == stage4_id + and stage4_id in selected_stage_ids(rule) and case_exit.get("marksCaseComplete") is not True ): expression = rule.get("conditionExpression") diff --git a/tests/tasks/uipath-maestro-case/golden_rebuild/cm_golden_expense/check_cm_golden_semantics.py b/tests/tasks/uipath-maestro-case/golden_rebuild/cm_golden_expense/check_cm_golden_semantics.py index 17db535074..fe9d387210 100644 --- a/tests/tasks/uipath-maestro-case/golden_rebuild/cm_golden_expense/check_cm_golden_semantics.py +++ b/tests/tasks/uipath-maestro-case/golden_rebuild/cm_golden_expense/check_cm_golden_semantics.py @@ -26,7 +26,7 @@ sys.path.insert( 0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) ) -from _shared.case_check import find_stages, read_caseplan # noqa: E402 +from _shared.case_check import find_stages, read_caseplan, selected_stage_ids # noqa: E402 EXPECTED_CASEPLAN = os.path.join("CMGoldenExpense", "CMGoldenExpense", "caseplan.json") FIXTURE_SDD = os.path.join(os.path.dirname(os.path.abspath(__file__)), "fixtures", "sdd.md") @@ -370,9 +370,7 @@ def _selected_tasks(rule: dict, task_ids: dict) -> tuple[tuple[str, str], ...]: def _selected_stages(rule: dict, stage_ids: dict) -> tuple[str, ...]: - ids = list(rule.get("selectedStagesIds") or []) - if rule.get("selectedStageId"): - ids.append(rule["selectedStageId"]) + ids = selected_stage_ids(rule) logical = [] for stage_id in ids: if stage_id not in stage_ids: diff --git a/tests/tasks/uipath-maestro-case/multi_stages_tasks/case_exit_full/check_case_exit_full.py b/tests/tasks/uipath-maestro-case/multi_stages_tasks/case_exit_full/check_case_exit_full.py index b96b9935b2..95e44156de 100644 --- a/tests/tasks/uipath-maestro-case/multi_stages_tasks/case_exit_full/check_case_exit_full.py +++ b/tests/tasks/uipath-maestro-case/multi_stages_tasks/case_exit_full/check_case_exit_full.py @@ -16,6 +16,7 @@ iter_stage_entry_conditions, iter_stage_exit_conditions, read_caseplan, + selected_stage_ids, start_debug, ) @@ -122,9 +123,7 @@ def main(): elif marks is False: non_completing_rules.add(rname) if rname in ("selected-stage-exited", "selected-stage-completed"): - sid = rule.get("selectedStageId") - if sid: - selected_stage_by_rule.setdefault(rname, set()).add(sid) + selected_stage_by_rule.setdefault(rname, set()).update(selected_stage_ids(rule)) if "required-stages-completed" not in completing_rules: sys.exit( diff --git a/tests/tasks/uipath-maestro-case/multi_stages_tasks/exception_stage/check_exception_stage.py b/tests/tasks/uipath-maestro-case/multi_stages_tasks/exception_stage/check_exception_stage.py index 5625644840..2c52fb98b5 100644 --- a/tests/tasks/uipath-maestro-case/multi_stages_tasks/exception_stage/check_exception_stage.py +++ b/tests/tasks/uipath-maestro-case/multi_stages_tasks/exception_stage/check_exception_stage.py @@ -15,6 +15,7 @@ iter_stage_exit_conditions, partition_return_to_origin_conditions, read_caseplan, + selected_stage_ids, ) @@ -77,10 +78,10 @@ def _is_secondary(n): f"FAIL: 'Issues' interrupting rule should be 'selected-stage-exited'; " f"got {issues_rule and issues_rule.get('rule')!r}" ) - if issues_rule.get("selectedStageId") != process["id"]: + if process["id"] not in selected_stage_ids(issues_rule): sys.exit( f"FAIL: 'Issues' rule.selectedStageId should be Process id " - f"({process['id']}), got {issues_rule.get('selectedStageId')!r}" + f"({process['id']}), got {selected_stage_ids(issues_rule)!r}" ) critical_entry = list(iter_stage_entry_conditions(critical)) @@ -93,10 +94,10 @@ def _is_secondary(n): f"FAIL: 'Critical' interrupting rule should be 'selected-stage-exited'; " f"got {critical_rule and critical_rule.get('rule')!r}" ) - if critical_rule.get("selectedStageId") != process["id"]: + if process["id"] not in selected_stage_ids(critical_rule): sys.exit( f"FAIL: 'Critical' rule.selectedStageId should be Process id " - f"({process['id']}), got {critical_rule.get('selectedStageId')!r}" + f"({process['id']}), got {selected_stage_ids(critical_rule)!r}" ) for label, node in (("Issues", issues), ("Critical", critical)): @@ -147,7 +148,7 @@ def _is_secondary(n): critical_case_exits = [] for cond in get_case_exit_conditions(plan): rule = first_rule_of_condition(cond) or {} - if rule.get("selectedStageId") == critical["id"]: + if critical["id"] in selected_stage_ids(rule): critical_case_exits.append(cond) if not critical_case_exits: sys.exit("FAIL: terminal secondary stage 'Critical' needs a root case-exit row") diff --git a/tests/tasks/uipath-maestro-case/multi_stages_tasks/fan_in_join/check_fan_in_join.py b/tests/tasks/uipath-maestro-case/multi_stages_tasks/fan_in_join/check_fan_in_join.py index e011b6c227..2afc5fb320 100644 --- a/tests/tasks/uipath-maestro-case/multi_stages_tasks/fan_in_join/check_fan_in_join.py +++ b/tests/tasks/uipath-maestro-case/multi_stages_tasks/fan_in_join/check_fan_in_join.py @@ -15,6 +15,7 @@ first_rule_of_condition, iter_stage_entry_conditions, read_caseplan, + selected_stage_ids, start_debug, task_is_skeleton, ) @@ -71,9 +72,7 @@ def main(): sys.exit( f"FAIL: Join entry rule must be 'selected-stage-completed'; got {rule.get('rule')!r}" ) - sid = rule.get("selectedStageId") - if sid: - referenced_stage_ids.add(sid) + referenced_stage_ids.update(selected_stage_ids(rule)) if not {validate["id"], enrich["id"]}.issubset(referenced_stage_ids): sys.exit( diff --git a/tests/tasks/uipath-maestro-case/multi_stages_tasks/task_dependency_chain/check_task_dependency_chain.py b/tests/tasks/uipath-maestro-case/multi_stages_tasks/task_dependency_chain/check_task_dependency_chain.py index 0653a3a5e1..0e011c7b0b 100644 --- a/tests/tasks/uipath-maestro-case/multi_stages_tasks/task_dependency_chain/check_task_dependency_chain.py +++ b/tests/tasks/uipath-maestro-case/multi_stages_tasks/task_dependency_chain/check_task_dependency_chain.py @@ -14,6 +14,7 @@ iter_stage_entry_conditions, iter_stage_exit_conditions, read_caseplan, + selected_stage_ids, start_debug, task_is_skeleton, ) @@ -115,10 +116,12 @@ def main(): f"FAIL: Done should have entry rule 'selected-stage-completed'; " f"got {sorted(r.get('rule') for r in done_entry_rules if r.get('rule'))}" ) - if finalize["id"] not in {r.get("selectedStageId") for r in done_entry_rules}: + if finalize["id"] not in { + stage_id for rule in done_entry_rules for stage_id in selected_stage_ids(rule) + }: sys.exit( f"FAIL: Done selected-stage-completed entry must reference Finalize id " - f"{finalize['id']!r}; got {[r.get('selectedStageId') for r in done_entry_rules]}" + f"{finalize['id']!r}; got {[selected_stage_ids(r) for r in done_entry_rules]}" ) first_step = _stage_task_by_label(process, "First Step") From 66c5689b6657d78ddf36aa75abd01f9df3cb645b Mon Sep 17 00:00:00 2001 From: Tao Ma <51425734+tmatup@users.noreply.github.com> Date: Sat, 29 Aug 2026 14:14:46 -0700 Subject: [PATCH 20/52] test(maestro-case): make grading author-loop neutral (#2898) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(maestro-flow): restore campaign CI after rebase Remove duplicated generated rules, align validator tests with shared discovery, and keep same-ground voice tasks outcome-based. ๐Ÿค– Generated with Claude Code Co-Authored-By: [Claude](mailto:noreply@anthropic.com) * test(maestro-case): make skill telemetry advisory Grade functional Case behavior consistently across authoring loops and keep skill-invocation telemetry as diagnostics only. Add a corpus guard so new gating activation criteria cannot regress parity. ๐Ÿค– Generated with Claude Code Co-Authored-By: [Claude](mailto:noreply@anthropic.com) * test(maestro-case): remove author-loop grading assumptions ๐Ÿค– Generated with Claude Code Co-Authored-By: [Claude](mailto:noreply@anthropic.com) --- .../_shared/test_same_ground_corpus.py | 46 ++++++++++++++++++ .../athena_cm_event/athena_cm_event.yaml | 4 +- .../build_from_planner_sdd.yaml | 4 +- .../connection_parity/connection_parity.yaml | 4 +- .../connector_activity.yaml | 4 +- .../connector_wait/connector_wait.yaml | 4 +- .../extract_alias_collision.yaml | 4 +- .../fix_broken_caseplan.yaml | 4 +- .../formal_arg_slot_ids.yaml | 4 +- .../missing_task_entry_rule.yaml | 4 +- .../undeclared_gate_vars.yaml | 4 +- .../outcome_external_e2e.yaml | 4 +- .../bound_task_add_remove.yaml | 4 +- .../hitl/check_boolean_routing.py | 48 +++++++++++++++++++ .../hitl/quality_01_action_schema_design.yaml | 48 ++++++------------- .../hitl/quality_03_boolean_decision.yaml | 11 ++--- .../hitl/smoke_03_multi_outcome_routing.yaml | 11 ++--- .../hitl/test_check_boolean_routing.py | 28 +++++++++++ .../publish_upload/publish_upload.yaml | 4 +- .../safety_guard/artifact_safety_guard.yaml | 4 +- .../event_placeholder/event_placeholder.yaml | 4 +- 21 files changed, 176 insertions(+), 76 deletions(-) create mode 100644 tests/tasks/uipath-maestro-case/_shared/test_same_ground_corpus.py create mode 100644 tests/tasks/uipath-maestro-case/hitl/check_boolean_routing.py create mode 100644 tests/tasks/uipath-maestro-case/hitl/test_check_boolean_routing.py diff --git a/tests/tasks/uipath-maestro-case/_shared/test_same_ground_corpus.py b/tests/tasks/uipath-maestro-case/_shared/test_same_ground_corpus.py new file mode 100644 index 0000000000..db3607cc80 --- /dev/null +++ b/tests/tasks/uipath-maestro-case/_shared/test_same_ground_corpus.py @@ -0,0 +1,46 @@ +"""Permanent cross-loop grading contracts for Maestro Case eval tasks.""" + +from __future__ import annotations + +import re +from pathlib import Path + + +CASE_TASKS = Path(__file__).resolve().parent.parent + + +def _block(text: str, key: str) -> str: + match = re.search(rf"(?ms)^{re.escape(key)}:[^\n]*\n(.*?)(?=^[A-Za-z_][\w-]*:|\Z)", text) + return match.group(0) if match else "" + + +def _criterion_blocks(text: str) -> list[tuple[str, str]]: + section = _block(text, "success_criteria") + return [ + (match.group(1), match.group(0)) + for match in re.finditer( + r"(?ms)^ - type:\s*([^\s#]+).*?(?=^ - type:|^[A-Za-z_][\w-]*:|\Z)", + section, + ) + ] + + +def _tagged_tasks() -> list[tuple[str, str]]: + tasks = [] + for path in sorted(CASE_TASKS.rglob("*.yaml")): + text = path.read_text() + if "uipath-maestro-case" in _block(text, "tags"): + tasks.append((path.relative_to(CASE_TASKS).as_posix(), text)) + return tasks + + +def test_skill_telemetry_is_advisory_across_the_case_corpus() -> None: + offenders = set() + for relative, text in _tagged_tasks(): + for criterion_type, criterion in _criterion_blocks(text): + if criterion_type != "skill_triggered": + continue + threshold = re.search(r"(?m)^\s+pass_threshold:\s*([0-9.]+)", criterion) + if threshold is None or float(threshold.group(1)) > 0: + offenders.add(relative) + assert offenders == set() diff --git a/tests/tasks/uipath-maestro-case/athena_cm_event/athena_cm_event.yaml b/tests/tasks/uipath-maestro-case/athena_cm_event/athena_cm_event.yaml index 9f2881ed1d..76d3e31668 100644 --- a/tests/tasks/uipath-maestro-case/athena_cm_event/athena_cm_event.yaml +++ b/tests/tasks/uipath-maestro-case/athena_cm_event/athena_cm_event.yaml @@ -47,11 +47,11 @@ initial_prompt: | success_criteria: - type: skill_triggered - description: "Agent invoked the uipath-maestro-case skill" + description: "Advisory: agent invoked the uipath-maestro-case skill" skill_name: "uipath-maestro-case" expected_skill: "uipath-maestro-case" weight: 1.0 - pass_threshold: 1.0 + pass_threshold: 0.0 - type: command_executed description: "Agent ran case validation (advisory; outcome is graded below)" diff --git a/tests/tasks/uipath-maestro-case/build_from_planner_sdd/build_from_planner_sdd.yaml b/tests/tasks/uipath-maestro-case/build_from_planner_sdd/build_from_planner_sdd.yaml index f22dd29736..f4a7db3faf 100644 --- a/tests/tasks/uipath-maestro-case/build_from_planner_sdd/build_from_planner_sdd.yaml +++ b/tests/tasks/uipath-maestro-case/build_from_planner_sdd/build_from_planner_sdd.yaml @@ -54,11 +54,11 @@ initial_prompt: | success_criteria: - type: skill_triggered - description: "Agent invoked the uipath-maestro-case skill (not planner Lane A)" + description: "Advisory: agent invoked the uipath-maestro-case skill (not planner Lane A)" skill_name: "uipath-maestro-case" expected_skill: "uipath-maestro-case" weight: 2.0 - pass_threshold: 1.0 + pass_threshold: 0.0 - type: llm_judge description: "Final reply names both unresolved resource identities as deferred to the build machine" diff --git a/tests/tasks/uipath-maestro-case/connector_features/connection_parity/connection_parity.yaml b/tests/tasks/uipath-maestro-case/connector_features/connection_parity/connection_parity.yaml index e4182eefd0..03f3b7be01 100644 --- a/tests/tasks/uipath-maestro-case/connector_features/connection_parity/connection_parity.yaml +++ b/tests/tasks/uipath-maestro-case/connector_features/connection_parity/connection_parity.yaml @@ -125,11 +125,11 @@ initial_prompt: | success_criteria: - type: skill_triggered - description: "Agent invoked the uipath-maestro-case skill" + description: "Advisory: agent invoked the uipath-maestro-case skill" skill_name: "uipath-maestro-case" expected_skill: "uipath-maestro-case" weight: 1.0 - pass_threshold: 1.0 + pass_threshold: 0.0 - type: run_command description: "uip maestro case validate passes on the generated caseplan.json" diff --git a/tests/tasks/uipath-maestro-case/connector_features/connector_activity/connector_activity.yaml b/tests/tasks/uipath-maestro-case/connector_features/connector_activity/connector_activity.yaml index 346e45c453..dac23e2446 100644 --- a/tests/tasks/uipath-maestro-case/connector_features/connector_activity/connector_activity.yaml +++ b/tests/tasks/uipath-maestro-case/connector_features/connector_activity/connector_activity.yaml @@ -119,11 +119,11 @@ initial_prompt: | success_criteria: - type: skill_triggered - description: "Agent invoked the uipath-maestro-case skill" + description: "Advisory: agent invoked the uipath-maestro-case skill" skill_name: "uipath-maestro-case" expected_skill: "uipath-maestro-case" weight: 1.0 - pass_threshold: 1.0 + pass_threshold: 0.0 - type: command_executed description: "Agent pulled the case registry before resolving the connector" diff --git a/tests/tasks/uipath-maestro-case/connector_features/connector_wait/connector_wait.yaml b/tests/tasks/uipath-maestro-case/connector_features/connector_wait/connector_wait.yaml index 3d13672bd7..96da11fcc7 100644 --- a/tests/tasks/uipath-maestro-case/connector_features/connector_wait/connector_wait.yaml +++ b/tests/tasks/uipath-maestro-case/connector_features/connector_wait/connector_wait.yaml @@ -152,11 +152,11 @@ initial_prompt: | success_criteria: - type: skill_triggered - description: "Agent invoked the uipath-maestro-case skill" + description: "Advisory: agent invoked the uipath-maestro-case skill" skill_name: "uipath-maestro-case" expected_skill: "uipath-maestro-case" weight: 1.0 - pass_threshold: 1.0 + pass_threshold: 0.0 - type: command_executed description: "Agent fetched the connector-trigger spec to resolve typeId/connectionId" diff --git a/tests/tasks/uipath-maestro-case/diagnose/extract_alias_collision/extract_alias_collision.yaml b/tests/tasks/uipath-maestro-case/diagnose/extract_alias_collision/extract_alias_collision.yaml index c161895a34..59503b8994 100644 --- a/tests/tasks/uipath-maestro-case/diagnose/extract_alias_collision/extract_alias_collision.yaml +++ b/tests/tasks/uipath-maestro-case/diagnose/extract_alias_collision/extract_alias_collision.yaml @@ -54,11 +54,11 @@ initial_prompt: | success_criteria: - type: skill_triggered - description: "Agent invoked the uipath-maestro-case skill" + description: "Advisory: agent invoked the uipath-maestro-case skill" skill_name: "uipath-maestro-case" expected_skill: "uipath-maestro-case" weight: 1.0 - pass_threshold: 1.0 + pass_threshold: 0.0 - type: run_command description: "uip maestro case validate still passes after the repair" diff --git a/tests/tasks/uipath-maestro-case/diagnose/fix_broken_caseplan/fix_broken_caseplan.yaml b/tests/tasks/uipath-maestro-case/diagnose/fix_broken_caseplan/fix_broken_caseplan.yaml index 344c5d39f1..b76e8bebe2 100644 --- a/tests/tasks/uipath-maestro-case/diagnose/fix_broken_caseplan/fix_broken_caseplan.yaml +++ b/tests/tasks/uipath-maestro-case/diagnose/fix_broken_caseplan/fix_broken_caseplan.yaml @@ -48,11 +48,11 @@ initial_prompt: | success_criteria: - type: skill_triggered - description: "Agent invoked the uipath-maestro-case skill" + description: "Advisory: agent invoked the uipath-maestro-case skill" skill_name: "uipath-maestro-case" expected_skill: "uipath-maestro-case" weight: 1.0 - pass_threshold: 1.0 + pass_threshold: 0.0 - type: command_executed description: "Agent ran the validate โ†’ fix loop (validate at least twice: read errors, then confirm the fix)" diff --git a/tests/tasks/uipath-maestro-case/diagnose/formal_arg_slot_ids/formal_arg_slot_ids.yaml b/tests/tasks/uipath-maestro-case/diagnose/formal_arg_slot_ids/formal_arg_slot_ids.yaml index 85b08c809f..cc4e179e5b 100644 --- a/tests/tasks/uipath-maestro-case/diagnose/formal_arg_slot_ids/formal_arg_slot_ids.yaml +++ b/tests/tasks/uipath-maestro-case/diagnose/formal_arg_slot_ids/formal_arg_slot_ids.yaml @@ -40,11 +40,11 @@ initial_prompt: | success_criteria: - type: skill_triggered - description: "Agent invoked the uipath-maestro-case skill" + description: "Advisory: agent invoked the uipath-maestro-case skill" skill_name: "uipath-maestro-case" expected_skill: "uipath-maestro-case" weight: 1.0 - pass_threshold: 1.0 + pass_threshold: 0.0 - type: run_command description: "uip maestro case validate still passes after the repair" diff --git a/tests/tasks/uipath-maestro-case/diagnose/missing_task_entry_rule/missing_task_entry_rule.yaml b/tests/tasks/uipath-maestro-case/diagnose/missing_task_entry_rule/missing_task_entry_rule.yaml index 4c7db878ce..d5ba6ba00c 100644 --- a/tests/tasks/uipath-maestro-case/diagnose/missing_task_entry_rule/missing_task_entry_rule.yaml +++ b/tests/tasks/uipath-maestro-case/diagnose/missing_task_entry_rule/missing_task_entry_rule.yaml @@ -38,11 +38,11 @@ initial_prompt: | success_criteria: - type: skill_triggered - description: "Agent invoked the uipath-maestro-case skill" + description: "Advisory: agent invoked the uipath-maestro-case skill" skill_name: "uipath-maestro-case" expected_skill: "uipath-maestro-case" weight: 1.0 - pass_threshold: 1.0 + pass_threshold: 0.0 - type: run_command description: "uip maestro case validate still passes after the repair" diff --git a/tests/tasks/uipath-maestro-case/diagnose/undeclared_gate_vars/undeclared_gate_vars.yaml b/tests/tasks/uipath-maestro-case/diagnose/undeclared_gate_vars/undeclared_gate_vars.yaml index efadbf586a..5802f83c05 100644 --- a/tests/tasks/uipath-maestro-case/diagnose/undeclared_gate_vars/undeclared_gate_vars.yaml +++ b/tests/tasks/uipath-maestro-case/diagnose/undeclared_gate_vars/undeclared_gate_vars.yaml @@ -39,11 +39,11 @@ initial_prompt: | success_criteria: - type: skill_triggered - description: "Agent invoked the uipath-maestro-case skill" + description: "Advisory: agent invoked the uipath-maestro-case skill" skill_name: "uipath-maestro-case" expected_skill: "uipath-maestro-case" weight: 1.0 - pass_threshold: 1.0 + pass_threshold: 0.0 - type: run_command description: "uip maestro case validate passes on the repaired caseplan.json" diff --git a/tests/tasks/uipath-maestro-case/e2e_outcome_external/outcome_external_e2e.yaml b/tests/tasks/uipath-maestro-case/e2e_outcome_external/outcome_external_e2e.yaml index af697daa78..f27d2f8e75 100644 --- a/tests/tasks/uipath-maestro-case/e2e_outcome_external/outcome_external_e2e.yaml +++ b/tests/tasks/uipath-maestro-case/e2e_outcome_external/outcome_external_e2e.yaml @@ -104,11 +104,11 @@ initial_prompt: | success_criteria: - type: skill_triggered - description: "Agent invoked the uipath-maestro-case skill" + description: "Advisory: agent invoked the uipath-maestro-case skill" skill_name: "uipath-maestro-case" expected_skill: "uipath-maestro-case" weight: 1.0 - pass_threshold: 1.0 + pass_threshold: 0.0 # Advisory: convention adherence, not the objective. "Not if it validated" โ€” a case # that validates while notifying nobody has not achieved anything. diff --git a/tests/tasks/uipath-maestro-case/edit/bound_task_add_remove/bound_task_add_remove.yaml b/tests/tasks/uipath-maestro-case/edit/bound_task_add_remove/bound_task_add_remove.yaml index 6d9ef380b8..504c04155f 100644 --- a/tests/tasks/uipath-maestro-case/edit/bound_task_add_remove/bound_task_add_remove.yaml +++ b/tests/tasks/uipath-maestro-case/edit/bound_task_add_remove/bound_task_add_remove.yaml @@ -46,11 +46,11 @@ initial_prompt: | success_criteria: - type: skill_triggered - description: "Agent invoked the uipath-maestro-case skill" + description: "Advisory: agent invoked the uipath-maestro-case skill" skill_name: "uipath-maestro-case" expected_skill: "uipath-maestro-case" weight: 1.0 - pass_threshold: 1.0 + pass_threshold: 0.0 - type: run_command description: "uip maestro case validate passes on the edited caseplan.json" diff --git a/tests/tasks/uipath-maestro-case/hitl/check_boolean_routing.py b/tests/tasks/uipath-maestro-case/hitl/check_boolean_routing.py new file mode 100644 index 0000000000..bd98c0b162 --- /dev/null +++ b/tests/tasks/uipath-maestro-case/hitl/check_boolean_routing.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +"""Check both semantic boolean branches without pinning authoring-loop spelling.""" + +from __future__ import annotations + +import json +import re +import sys +from pathlib import Path +from typing import Any + + +BOOLEAN_BRANCH = re.compile( + r"^(?:=js:)?(?:vars\.)?approved={2,3}(true|false)$", + re.IGNORECASE, +) + + +def branch_values(value: Any) -> set[bool]: + found: set[bool] = set() + if isinstance(value, dict): + expression = value.get("conditionExpression") + if isinstance(expression, str): + match = BOOLEAN_BRANCH.fullmatch(re.sub(r"\s+", "", expression)) + if match: + found.add(match.group(1).lower() == "true") + for child in value.values(): + found.update(branch_values(child)) + elif isinstance(value, list): + for child in value: + found.update(branch_values(child)) + return found + + +def main() -> None: + if len(sys.argv) != 2: + sys.exit("usage: check_boolean_routing.py ") + path = Path(sys.argv[1]) + plan = json.loads(path.read_text(encoding="utf-8")) + found = branch_values(plan) + missing = {True, False} - found + if missing: + sys.exit(f"FAIL: missing approved boolean branch(es): {sorted(missing)}") + print(f"OK: {path} contains approved true and false branches") + + +if __name__ == "__main__": + main() diff --git a/tests/tasks/uipath-maestro-case/hitl/quality_01_action_schema_design.yaml b/tests/tasks/uipath-maestro-case/hitl/quality_01_action_schema_design.yaml index 234e43a2eb..520869cc39 100644 --- a/tests/tasks/uipath-maestro-case/hitl/quality_01_action_schema_design.yaml +++ b/tests/tasks/uipath-maestro-case/hitl/quality_01_action_schema_design.yaml @@ -3,8 +3,8 @@ description: > Quality test: agent maps a business description to a correctly-shaped `action` task โ€” taskTitle, priority, recipient, and the planned input/ output fields. Verifies that even when the action-app is unresolved - (skeleton path), the agent still records the planning fields in the - resolution audit and writes valid JSON. Tests action-task field design (priority enum, + (skeleton path), the agent records the intended contract in report.json + and writes valid JSON. Tests action-task field design (priority enum, recipient handling, taskTitle requirement). tags: [uipath-maestro-case, integration, mode:build, lifecycle:generate, shape:multi-node, node:hitl] initial_prompt: | @@ -181,13 +181,11 @@ initial_prompt: | Important: - The `uip` CLI is already available. - - Treat the generated plan as pre-approved โ€” do not block on any HARD STOP + - Treat any generated plan as pre-approved โ€” do not block on any HARD STOP / AskUserQuestion approval step. - If the action-app or the process resource is unresolved, write a - skeleton task per the skeleton-tasks reference. That task's - `tasks/registry-resolved.json` entry must still capture taskTitle, - priority, recipient, and the intended inputs/outputs in a fenced - wiring-notes block. + skeleton task per the skeleton-tasks reference. `report.json` must still + capture taskTitle, priority, recipient, and the intended inputs/outputs. - Do NOT run `uip maestro case debug`. **REQUIRED โ€” before finishing, write report.json at the sandbox root**: @@ -224,32 +222,8 @@ success_criteria: weight: 2.0 pass_threshold: 1.0 - - type: run_command - description: "the task-title is captured (caseplan on the resolved path, registry audit on the skeleton path)" - command: "grep -ri 'task-title\\|taskTitle\\|Task Title' tasks/registry-resolved.json VendorPOReviewCase/VendorPOReviewCase/caseplan.json" - timeout: 10 - expected_exit_code: 0 - weight: 2.0 - pass_threshold: 1.0 - - - type: run_command - description: "priority: High is captured (caseplan or registry audit)" - command: "grep -ri 'priority.*high' tasks/registry-resolved.json VendorPOReviewCase/VendorPOReviewCase/caseplan.json" - timeout: 10 - expected_exit_code: 0 - weight: 1.5 - pass_threshold: 1.0 - - - type: run_command - description: "the recipient email is captured (caseplan or registry audit)" - command: "grep -ri 'po-approvals@corp.com' tasks/registry-resolved.json VendorPOReviewCase/VendorPOReviewCase/caseplan.json" - timeout: 10 - expected_exit_code: 0 - weight: 1.5 - pass_threshold: 1.0 - - type: json_check - description: "report.json schema has at least 3 input fields, 1 output field, priority High" + description: "report.json preserves the HITL task contract" path: "report.json" assertions: - expression: "validation_passed" @@ -264,8 +238,14 @@ success_criteria: - expression: "priority" operator: equals expected: "High" - weight: 2.5 - pass_threshold: 0.75 + - expression: "task_title" + operator: equals + expected: "Please review this PO and approve or reject" + - expression: "recipient" + operator: equals + expected: "po-approvals@corp.com" + weight: 7.5 + pass_threshold: 0.85 run_limits: expected_turns: 66 diff --git a/tests/tasks/uipath-maestro-case/hitl/quality_03_boolean_decision.yaml b/tests/tasks/uipath-maestro-case/hitl/quality_03_boolean_decision.yaml index 8fdb57cae4..f601942991 100644 --- a/tests/tasks/uipath-maestro-case/hitl/quality_03_boolean_decision.yaml +++ b/tests/tasks/uipath-maestro-case/hitl/quality_03_boolean_decision.yaml @@ -258,12 +258,11 @@ success_criteria: weight: 1.5 pass_threshold: 1.0 - - type: file_contains - description: "Both branch expressions reference the approved boolean" - path: "VendorApprovalCase/VendorApprovalCase/caseplan.json" - includes: - - "vars.approved === true" - - "vars.approved === false" + - type: run_command + description: "Both semantic boolean branches reference approved" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-case/hitl/check_boolean_routing.py VendorApprovalCase/VendorApprovalCase/caseplan.json" + timeout: 30 + expected_exit_code: 0 weight: 3.0 pass_threshold: 1.0 diff --git a/tests/tasks/uipath-maestro-case/hitl/smoke_03_multi_outcome_routing.yaml b/tests/tasks/uipath-maestro-case/hitl/smoke_03_multi_outcome_routing.yaml index 7da10b3f8f..a221006a24 100644 --- a/tests/tasks/uipath-maestro-case/hitl/smoke_03_multi_outcome_routing.yaml +++ b/tests/tasks/uipath-maestro-case/hitl/smoke_03_multi_outcome_routing.yaml @@ -195,12 +195,11 @@ success_criteria: weight: 2.5 pass_threshold: 1.0 - - type: file_contains - description: "Both branch expressions present on Review's exits" - path: "LeaveRequestCase/LeaveRequestCase/caseplan.json" - includes: - - "vars.approved === true" - - "vars.approved === false" + - type: run_command + description: "Both semantic boolean branches are present on Review's exits" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-case/hitl/check_boolean_routing.py LeaveRequestCase/LeaveRequestCase/caseplan.json" + timeout: 30 + expected_exit_code: 0 weight: 2.0 pass_threshold: 1.0 diff --git a/tests/tasks/uipath-maestro-case/hitl/test_check_boolean_routing.py b/tests/tasks/uipath-maestro-case/hitl/test_check_boolean_routing.py new file mode 100644 index 0000000000..c98cd97abc --- /dev/null +++ b/tests/tasks/uipath-maestro-case/hitl/test_check_boolean_routing.py @@ -0,0 +1,28 @@ +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from check_boolean_routing import branch_values # noqa: E402 + + +def test_accepts_equivalent_case_expression_spellings(): + plan = { + "conditions": [ + {"conditionExpression": "=js:vars.approved === true"}, + {"conditionExpression": "approved == false"}, + ] + } + + assert branch_values(plan) == {True, False} + + +def test_does_not_accept_unrelated_boolean_expressions(): + plan = { + "conditions": [ + {"conditionExpression": "=js:vars.other === true"}, + {"conditionExpression": "=js:vars.approved !== false"}, + ] + } + + assert branch_values(plan) == set() diff --git a/tests/tasks/uipath-maestro-case/publish/publish_upload/publish_upload.yaml b/tests/tasks/uipath-maestro-case/publish/publish_upload/publish_upload.yaml index d69cbda021..aa439926e6 100644 --- a/tests/tasks/uipath-maestro-case/publish/publish_upload/publish_upload.yaml +++ b/tests/tasks/uipath-maestro-case/publish/publish_upload/publish_upload.yaml @@ -77,11 +77,11 @@ initial_prompt: | success_criteria: - type: skill_triggered - description: "Agent invoked the uipath-maestro-case skill" + description: "Advisory: agent invoked the uipath-maestro-case skill" skill_name: "uipath-maestro-case" expected_skill: "uipath-maestro-case" weight: 1.0 - pass_threshold: 1.0 + pass_threshold: 0.0 - type: run_command description: "uip maestro case validate passes on the generated caseplan.json" diff --git a/tests/tasks/uipath-maestro-case/safety_guard/artifact_safety_guard.yaml b/tests/tasks/uipath-maestro-case/safety_guard/artifact_safety_guard.yaml index d26718c1b7..4625d6a8a2 100644 --- a/tests/tasks/uipath-maestro-case/safety_guard/artifact_safety_guard.yaml +++ b/tests/tasks/uipath-maestro-case/safety_guard/artifact_safety_guard.yaml @@ -188,9 +188,9 @@ success_criteria: # --- Rule 13 guards: no interpreter/stream-editor/helper-script assembly of a skill artifact --- # crypto.randomUUID() UUID subprocess (no fs, no redirection) is NOT matched by either pattern. - type: command_not_executed - description: "Rule 13: no inline interpreter or stream-editor manipulation of a skill artifact (sed -i, perl -i, gawk in-place, jq on/into an artifact, python open(), node -e ...fs. / require('fs'))" + description: "Rule 13: no inline interpreter or stream-editor writes to a skill artifact (read-only inspection is allowed)" tool_name: "Bash" - command_pattern: '(sed\s+-i|perl\s+-i|awk\b[^\n]*-i\s*inplace|jq\b[^\n]*([<>]|caseplan\.json|registry-resolved\.json|bindings_v2\.json|entry-points\.json)|python[0-9]?\b[^\n]*open\s*\(|(node|nodejs)\s+(-e|--eval)\b[^\n]*(fs\.|require\s*\(\s*[''"](node:)?fs[''"]))' + command_pattern: '(sed\s+-i|perl\s+-i|awk\b[^\n]*-i\s*inplace|python[0-9]?\b[^\n]*(open\s*\([^)]*,[^)]*[wax]|\.write(_text|_bytes)?\s*\()|(node|nodejs)\s+(-e|--eval)\b[^\n]*(fs\.|require\s*\(\s*[''"](node:)?fs[''"]\s*\)\s*\.)(write|append|truncate|rm|unlink|rename))' weight: 2.0 pass_threshold: 1.0 diff --git a/tests/tasks/uipath-maestro-case/triggers/event_placeholder/event_placeholder.yaml b/tests/tasks/uipath-maestro-case/triggers/event_placeholder/event_placeholder.yaml index c9e4729b21..4f6f2a26a2 100644 --- a/tests/tasks/uipath-maestro-case/triggers/event_placeholder/event_placeholder.yaml +++ b/tests/tasks/uipath-maestro-case/triggers/event_placeholder/event_placeholder.yaml @@ -102,11 +102,11 @@ initial_prompt: | success_criteria: - type: skill_triggered - description: "Agent invoked the uipath-maestro-case skill" + description: "Advisory: agent invoked the uipath-maestro-case skill" skill_name: "uipath-maestro-case" expected_skill: "uipath-maestro-case" weight: 1.0 - pass_threshold: 1.0 + pass_threshold: 0.0 - type: run_command description: "uip maestro case validate passes on the generated caseplan.json" From 50e5d2879a4fe953cd05d6b236cfa63f3d35f44a Mon Sep 17 00:00:00 2001 From: Tao Ma <51425734+tmatup@users.noreply.github.com> Date: Sat, 29 Aug 2026 14:31:08 -0700 Subject: [PATCH 21/52] test(maestro-bpmn): accept message summary events (#2899) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ๐Ÿค– Generated with Claude Code Co-Authored-By: [Claude](mailto:noreply@anthropic.com) --- .../patterns/high_volume_batch/check_greenfield.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tests/tasks/uipath-maestro-bpmn/patterns/high_volume_batch/check_greenfield.py b/tests/tasks/uipath-maestro-bpmn/patterns/high_volume_batch/check_greenfield.py index bcf46baeeb..2583a0c58f 100755 --- a/tests/tasks/uipath-maestro-bpmn/patterns/high_volume_batch/check_greenfield.py +++ b/tests/tasks/uipath-maestro-bpmn/patterns/high_volume_batch/check_greenfield.py @@ -44,8 +44,19 @@ def main() -> None: fail(f"no gateway is reachable from {sub_id} โ€” the block never reaches a verdict") verdict = verdicts[0] + # A BPMN message throw is the native way to send a summary. Count it as + # work on the path alongside task-shaped activities; requiring a sendTask + # would grade an implementation detail rather than the requested behavior. + message_throws = [ + event + for event in elements(root, "intermediateThrowEvent") + if event.find(f"./{{{BPMN_NS}}}messageEventDefinition") is not None + ] work_ids = ids( - elements(root, "serviceTask") + elements(root, "scriptTask") + elements(root, "sendTask") + elements(root, "serviceTask") + + elements(root, "scriptTask") + + elements(root, "sendTask") + + message_throws ) on_path = { n for n in downstream & work_ids From 52652459cbab4aaf52a9ad1dd5e81b025191d0d5 Mon Sep 17 00:00:00 2001 From: Tao Ma <51425734+tmatup@users.noreply.github.com> Date: Sat, 29 Aug 2026 16:01:04 -0700 Subject: [PATCH 22/52] fix(maestro-bpmn): accept registry-native agent jobs (#2901) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Align the live skill, bundled registry fallback, and checker with the runtime-authored Orchestrator.StartAgentJob shape emitted by the CLI registry while retaining the nested type contract for other nodes. Add focused checker coverage for both supported declarations. ๐Ÿค– Generated with Claude Code Co-Authored-By: [Claude](mailto:noreply@anthropic.com) --- .github/workflows/test-helpers.yml | 17 +++++ skills/uipath-maestro-bpmn/SKILL.md | 12 ++- .../validator/bpmn-spec.json | 45 ++++------- .../uipath-maestro-bpmn/_shared/bpmn_check.py | 28 ++++++- .../_shared/test_bpmn_check.py | 75 +++++++++++++++++++ .../nodes/check_contract_variant_wrappers.py | 14 +--- 6 files changed, 145 insertions(+), 46 deletions(-) create mode 100644 tests/tasks/uipath-maestro-bpmn/_shared/test_bpmn_check.py diff --git a/.github/workflows/test-helpers.yml b/.github/workflows/test-helpers.yml index 3b77dcafa2..32abfe85e6 100644 --- a/.github/workflows/test-helpers.yml +++ b/.github/workflows/test-helpers.yml @@ -5,6 +5,7 @@ on: paths: - 'tests/tasks/uipath-maestro-flow/**' - 'tests/tasks/uipath-maestro-case/**' + - 'tests/tasks/uipath-maestro-bpmn/**' - 'tests/tasks/uipath-planner/**' - 'tests/tasks/uipath-agents/**' - 'tests/tasks/uipath-admin/**' @@ -15,6 +16,22 @@ on: workflow_dispatch: jobs: + pytest-bpmn-check: + runs-on: uipath-ubuntu-latest + name: maestro-bpmn checker unit tests + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: '3.13' + + - name: Install pytest + run: pip install pytest + + - name: Run pytest + run: pytest tests/tasks/uipath-maestro-bpmn/ -v + pytest-flow-check: runs-on: uipath-ubuntu-latest name: maestro-flow checker unit tests diff --git a/skills/uipath-maestro-bpmn/SKILL.md b/skills/uipath-maestro-bpmn/SKILL.md index b0e6f64809..3247fc88e5 100644 --- a/skills/uipath-maestro-bpmn/SKILL.md +++ b/skills/uipath-maestro-bpmn/SKILL.md @@ -289,10 +289,14 @@ and honestly surfaced to the user as gaps when asked. and the process structure with the user (AskUserQuestion). 5. **The diagram is mandatory.** Import is diagram-driven โ€” every node needs a `BPMNShape`, every flow a `BPMNEdge`, or it will not appear on the canvas. -6. **Node type is a child element, never an attribute.** Every `uipath:activity` - / `uipath:event` / `uipath:mapping` declares its type as - `` inside the wrapper. Never write - `` โ€” the canvas will not recognize the node. +6. **Preserve the registry's node-type shape.** Most `uipath:activity` / + `uipath:event` / `uipath:mapping` templates declare their type as a nested + ``. Some runtime-authored + templates use the payload's `type` attribute instead; notably, + `Orchestrator.StartAgentJob` is a direct child of `bpmn:serviceTask` with + ``. Both + declarations are supported. Paste the selected registry template literally + and do not normalize one form into the other. Event extension types (`Intsvc.WaitForEvent`, `Intsvc.EventTrigger`, `Maestro.ReceiveMessageEvent`, `Maestro.SendMessageEvent`) must use ``, including when the BPMN host is task-like such as diff --git a/skills/uipath-maestro-bpmn/validator/bpmn-spec.json b/skills/uipath-maestro-bpmn/validator/bpmn-spec.json index 6e6bd0db81..0d7a5d5325 100644 --- a/skills/uipath-maestro-bpmn/validator/bpmn-spec.json +++ b/skills/uipath-maestro-bpmn/validator/bpmn-spec.json @@ -165,46 +165,33 @@ "extensionTag": "uipath:activity", "contextFields": [ { - "name": "releaseKey", - "displayName": "Process", + "name": "name", + "displayName": "Agent", "type": "string", - "required": false, + "required": true, "hidden": true, "isPrimaryKey": false, - "binding": false, + "binding": true, "defaultValue": null, "bindingInfo": { "resource": "process", "resourceKeyPattern": "${metadata.key}", - "propertyAttribute": "Key" + "propertyAttribute": "name" } }, - { - "name": "folderId", - "type": "string", - "required": false, - "hidden": true, - "isPrimaryKey": false, - "binding": false, - "defaultValue": null - }, { "name": "folderPath", "type": "string", - "required": false, - "hidden": true, - "isPrimaryKey": false, - "binding": false, - "defaultValue": null - }, - { - "name": "name", - "type": "string", - "required": false, + "required": true, "hidden": true, "isPrimaryKey": false, - "binding": false, - "defaultValue": null + "binding": true, + "defaultValue": null, + "bindingInfo": { + "resource": "process", + "resourceKeyPattern": "${metadata.key}", + "propertyAttribute": "folderPath" + } } ], "inputPattern": "mergedBody", @@ -218,12 +205,12 @@ "bindingInfo": { "resource": "process", "resourceKeyPattern": "${metadata.key}", - "propertyAttribute": "Key", - "contextField": "releaseKey" + "propertyAttribute": "name", + "contextField": "name" }, "requiresDiscovery": true, "isDynamic": true, - "xmlTemplate": "\n \n \n \n \n \n \n \n \n \n \n \n \n \n {incomingEdge}\n {outgoingEdge}\n", + "xmlTemplate": "\n \n \n \n \n \n \n \n \n {incomingEdge}\n {outgoingEdge}\n", "discoveryNotes": "Uses getSupportedOrchestratorTasks('agent', search) to list available agents" }, "Orchestrator.BusinessRules": { diff --git a/tests/tasks/uipath-maestro-bpmn/_shared/bpmn_check.py b/tests/tasks/uipath-maestro-bpmn/_shared/bpmn_check.py index d4eeb3f0b6..9aff8ca0a8 100644 --- a/tests/tasks/uipath-maestro-bpmn/_shared/bpmn_check.py +++ b/tests/tasks/uipath-maestro-bpmn/_shared/bpmn_check.py @@ -71,7 +71,33 @@ def text_content(element: ET.Element) -> str: def has_uipath_extension(element: ET.Element, token: str) -> bool: ext = element.find("bpmn:extensionElements", NS) - return ext is not None and token in ET.tostring(ext, encoding="unicode") + if ext is not None and token in ET.tostring(ext, encoding="unicode"): + return True + return any( + token in ET.tostring(child, encoding="unicode") + for child in element + if child.tag.startswith(f"{{{NS['uipath']}}}") + ) + + +def has_typed_uipath_extension( + element: ET.Element, extension_name: str, type_value: str +) -> bool: + """Match both registry-supported UiPath payload type declarations.""" + payloads = list(element.findall(f"uipath:{extension_name}", NS)) + ext = element.find("bpmn:extensionElements", NS) + if ext is not None: + payloads.extend(ext.findall(f"uipath:{extension_name}", NS)) + + for payload in payloads: + if payload.attrib.get("type") == type_value: + return True + if any( + type_elem.attrib.get("value") == type_value + for type_elem in payload.findall("uipath:type", NS) + ): + return True + return False def require_di_for_visible_elements(root: ET.Element) -> None: diff --git a/tests/tasks/uipath-maestro-bpmn/_shared/test_bpmn_check.py b/tests/tasks/uipath-maestro-bpmn/_shared/test_bpmn_check.py new file mode 100644 index 0000000000..1df5a1925a --- /dev/null +++ b/tests/tasks/uipath-maestro-bpmn/_shared/test_bpmn_check.py @@ -0,0 +1,75 @@ +"""Unit tests for shared Maestro BPMN checker contracts.""" + +from __future__ import annotations + +import json +import os +import sys +import xml.etree.ElementTree as ET +from pathlib import Path + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from bpmn_check import ( # noqa: E402 + NS, + has_typed_uipath_extension, + has_uipath_extension, +) + + +def _service_task(payload: str) -> ET.Element: + return ET.fromstring( + f'{payload}' + ) + + +def test_typed_extension_accepts_nested_type_child() -> None: + task = _service_task( + """ + + + + + + """ + ) + + assert has_typed_uipath_extension(task, "activity", "Orchestrator.StartJob") + assert has_uipath_extension(task, "Orchestrator.StartJob") + + +def test_typed_extension_accepts_registry_direct_type_attribute() -> None: + task = _service_task( + '' + ) + + assert has_typed_uipath_extension(task, "activity", "Orchestrator.StartAgentJob") + assert has_uipath_extension(task, "Orchestrator.StartAgentJob") + + +def test_typed_extension_rejects_wrong_wrapper_or_type() -> None: + task = _service_task( + '' + ) + + assert not has_typed_uipath_extension(task, "activity", "Orchestrator.StartAgentJob") + assert not has_typed_uipath_extension(task, "event", "Orchestrator.StartJob") + + +def test_bundled_start_agent_contract_matches_runtime_registry() -> None: + repo_root = Path(__file__).parents[4] + spec_path = repo_root / "skills/uipath-maestro-bpmn/validator/bpmn-spec.json" + spec = json.loads(spec_path.read_text(encoding="utf-8")) + start_agent = spec["extensionTypes"]["Orchestrator.StartAgentJob"] + + assert [field["name"] for field in start_agent["contextFields"]] == [ + "name", + "folderPath", + ] + assert start_agent["bindingInfo"]["contextField"] == "name" + assert '" not in start_agent["xmlTemplate"] + assert "releaseKey" not in start_agent["xmlTemplate"] diff --git a/tests/tasks/uipath-maestro-bpmn/nodes/check_contract_variant_wrappers.py b/tests/tasks/uipath-maestro-bpmn/nodes/check_contract_variant_wrappers.py index 8e25ffba04..755ca8ec85 100644 --- a/tests/tasks/uipath-maestro-bpmn/nodes/check_contract_variant_wrappers.py +++ b/tests/tasks/uipath-maestro-bpmn/nodes/check_contract_variant_wrappers.py @@ -9,6 +9,7 @@ NS, elements, fail, + has_typed_uipath_extension, parse_bpmn, require_di_for_visible_elements, require_no_private_connector_values, @@ -16,22 +17,11 @@ ) -def has_typed_extension(element: ET.Element, extension_name: str, type_value: str) -> bool: - ext = element.find("bpmn:extensionElements", NS) - if ext is None: - return False - for payload in ext.findall(f"uipath:{extension_name}", NS): - type_elem = payload.find("uipath:type", NS) - if type_elem is not None and type_elem.attrib.get("value") == type_value: - return True - return False - - def require_wrapper(root: ET.Element, wrapper: str, extension_name: str, type_value: str) -> None: matches = [ elem for elem in elements(root, wrapper) - if has_typed_extension(elem, extension_name, type_value) + if has_typed_uipath_extension(elem, extension_name, type_value) ] if not matches: fail(f"missing bpmn:{wrapper} with {type_value} uipath:{extension_name} shell") From ee05a86cba1a218fd646d80f257befea91c4706c Mon Sep 17 00:00:00 2001 From: Tao Ma <51425734+tmatup@users.noreply.github.com> Date: Sat, 29 Aug 2026 16:17:54 -0700 Subject: [PATCH 23/52] Accept the SDK default for optional Case tasks (#2902) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Align the evaluator with the Case SDK contract: omitted isRequired and explicit false both mean non-required.\n\n๐Ÿค– Generated with Claude Code\nCo-Authored-By: [Claude](mailto:noreply@anthropic.com) --- .../check_task_dependency_chain.py | 8 +++++++- .../test_check_task_dependency_chain.py | 16 ++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 tests/tasks/uipath-maestro-case/multi_stages_tasks/task_dependency_chain/test_check_task_dependency_chain.py diff --git a/tests/tasks/uipath-maestro-case/multi_stages_tasks/task_dependency_chain/check_task_dependency_chain.py b/tests/tasks/uipath-maestro-case/multi_stages_tasks/task_dependency_chain/check_task_dependency_chain.py index 0e011c7b0b..66a7014626 100644 --- a/tests/tasks/uipath-maestro-case/multi_stages_tasks/task_dependency_chain/check_task_dependency_chain.py +++ b/tests/tasks/uipath-maestro-case/multi_stages_tasks/task_dependency_chain/check_task_dependency_chain.py @@ -61,6 +61,12 @@ def _task_entry_rule(task: dict) -> str | None: return None +def _task_is_non_required(task: dict) -> bool: + # The Case SDK omits isRequired for the default non-required state. + value = task.get("isRequired") + return value is None or value is False + + def main(): plan = read_caseplan() @@ -158,7 +164,7 @@ def main(): (first_rule_of_condition(condition) or {}).get("rule") for condition in (optional_audit.get("entryConditions") or []) ] - if adhoc_rules != ["adhoc"] or optional_audit.get("isRequired") is not False: + if adhoc_rules != ["adhoc"] or not _task_is_non_required(optional_audit): sys.exit( "FAIL: Optional Audit must be an adhoc-only, non-required task; " f"rules={adhoc_rules!r}, isRequired={optional_audit.get('isRequired')!r}" diff --git a/tests/tasks/uipath-maestro-case/multi_stages_tasks/task_dependency_chain/test_check_task_dependency_chain.py b/tests/tasks/uipath-maestro-case/multi_stages_tasks/task_dependency_chain/test_check_task_dependency_chain.py new file mode 100644 index 0000000000..8df39d3c90 --- /dev/null +++ b/tests/tasks/uipath-maestro-case/multi_stages_tasks/task_dependency_chain/test_check_task_dependency_chain.py @@ -0,0 +1,16 @@ +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from check_task_dependency_chain import _task_is_non_required # noqa: E402 + + +def test_non_required_accepts_the_sdk_default_or_explicit_false(): + assert _task_is_non_required({}) + assert _task_is_non_required({"isRequired": False}) + + +def test_non_required_rejects_required_or_invalid_values(): + assert not _task_is_non_required({"isRequired": True}) + assert not _task_is_non_required({"isRequired": "false"}) From 4bd97a7c2ba80fb17d3c96265b85dcca59f9c112 Mon Sep 17 00:00:00 2001 From: Tao Ma <51425734+tmatup@users.noreply.github.com> Date: Sat, 29 Aug 2026 16:51:45 -0700 Subject: [PATCH 24/52] Accept SDK-authored generic BPMN activities (#2905) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes #2904 ๐Ÿค– Generated with Claude Code Co-Authored-By: [Claude](mailto:noreply@anthropic.com) --- .../nodes/check_contract_variant_wrappers.py | 12 +++++- .../nodes/test_contract_variant_wrappers.py | 42 +++++++++++++++++++ 2 files changed, 53 insertions(+), 1 deletion(-) create mode 100644 tests/tasks/uipath-maestro-bpmn/nodes/test_contract_variant_wrappers.py diff --git a/tests/tasks/uipath-maestro-bpmn/nodes/check_contract_variant_wrappers.py b/tests/tasks/uipath-maestro-bpmn/nodes/check_contract_variant_wrappers.py index 755ca8ec85..881ae6a6f9 100644 --- a/tests/tasks/uipath-maestro-bpmn/nodes/check_contract_variant_wrappers.py +++ b/tests/tasks/uipath-maestro-bpmn/nodes/check_contract_variant_wrappers.py @@ -32,6 +32,15 @@ def require_uipath_element(root: ET.Element, local_name: str, description: str) fail(f"missing {description}") +def has_preserve_only_generic_activity(root: ET.Element) -> bool: + if root.findall(".//uipath:Activity", NS): + return True + return any( + has_typed_uipath_extension(task, "activity", "uipath:Activity") + for task in elements(root, "serviceTask") + ) + + def require_uipath_attr_value( root: ET.Element, local_name: str, attr_name: str, expected_value: str, description: str ) -> None: @@ -75,7 +84,8 @@ def main() -> None: root, "scriptVersion", "value", "v2", "preserved legacy scriptVersion" ) require_uipath_element(root, "caseManagement", "preserve-only uipath:caseManagement payload") - require_uipath_element(root, "Activity", "preserve-only generic uipath:Activity payload") + if not has_preserve_only_generic_activity(root): + fail("missing preserve-only generic uipath:Activity payload") require_no_private_connector_values(root) require_sequence_integrity(root) diff --git a/tests/tasks/uipath-maestro-bpmn/nodes/test_contract_variant_wrappers.py b/tests/tasks/uipath-maestro-bpmn/nodes/test_contract_variant_wrappers.py new file mode 100644 index 0000000000..59e3bd2033 --- /dev/null +++ b/tests/tasks/uipath-maestro-bpmn/nodes/test_contract_variant_wrappers.py @@ -0,0 +1,42 @@ +import os +import sys +import xml.etree.ElementTree as ET + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from check_contract_variant_wrappers import NS, has_preserve_only_generic_activity + + +def _root(payload: str) -> ET.Element: + return ET.fromstring( + f'' + f'{payload}' + "" + ) + + +def test_accepts_legacy_literal_generic_activity() -> None: + root = _root('synthetic') + + assert has_preserve_only_generic_activity(root) + + +def test_accepts_sdk_authored_generic_activity_type() -> None: + root = _root( + '' + '' + "" + ) + + assert has_preserve_only_generic_activity(root) + + +def test_rejects_unrelated_activity_type() -> None: + root = _root( + '' + '' + "" + ) + + assert not has_preserve_only_generic_activity(root) From a58344d4e244896fc55a9c391420a04dd3c6eb92 Mon Sep 17 00:00:00 2001 From: Tao Ma <51425734+tmatup@users.noreply.github.com> Date: Sun, 30 Aug 2026 12:04:08 -0700 Subject: [PATCH 25/52] Make BPMN evals author-loop neutral (#2909) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Accept equivalent live-project and SDK-authored BPMN layouts while preserving semantic assertions. ๐Ÿค– Generated with Claude Code Co-Authored-By: [Claude](mailto:noreply@anthropic.com) --- .../author/check_simple_approval_bpmn.py | 18 +++++--- .../author/simple_approval_bpmn.yaml | 14 ++++-- .../author/test_simple_approval_bpmn.py | 46 +++++++++++++++++++ .../integration_service_boundary.yaml | 6 ++- .../debug_workflow_mocked.yaml | 10 ++-- .../wiki_pageviews/wiki_pageviews.yaml | 6 +-- .../nodes/hitl_rpa_wrappers.yaml | 6 ++- 7 files changed, 82 insertions(+), 24 deletions(-) create mode 100644 tests/tasks/uipath-maestro-bpmn/author/test_simple_approval_bpmn.py diff --git a/tests/tasks/uipath-maestro-bpmn/author/check_simple_approval_bpmn.py b/tests/tasks/uipath-maestro-bpmn/author/check_simple_approval_bpmn.py index db48444c9a..7209a2c695 100644 --- a/tests/tasks/uipath-maestro-bpmn/author/check_simple_approval_bpmn.py +++ b/tests/tasks/uipath-maestro-bpmn/author/check_simple_approval_bpmn.py @@ -2,6 +2,7 @@ import os import sys +import xml.etree.ElementTree as ET sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from _shared.bpmn_check import ( # noqa: E402 @@ -18,6 +19,15 @@ ) +def root_variable_identifiers(process: ET.Element) -> set[str]: + return { + identifier + for var in process.findall("bpmn:extensionElements/uipath:variables/*", NS) + for identifier in (var.attrib.get("id"), var.attrib.get("name")) + if identifier + } + + def main() -> None: path, root = parse_bpmn("ExpenseApprovalBpmn") @@ -57,13 +67,9 @@ def main() -> None: process = root.find("bpmn:process", NS) if process is None: fail("missing bpmn:process element") - variable_names = { - var.attrib.get("name") - for var in process.findall("bpmn:extensionElements/uipath:variables/*", NS) - if var.attrib.get("name") - } + variable_identifiers = root_variable_identifiers(process) for required in ("expenseId", "amount", "decision"): - if required not in variable_names: + if required not in variable_identifiers: fail(f"missing root variable named {required!r}") migration_versions = { diff --git a/tests/tasks/uipath-maestro-bpmn/author/simple_approval_bpmn.yaml b/tests/tasks/uipath-maestro-bpmn/author/simple_approval_bpmn.yaml index 4cea88bdcf..1c6a7f4a78 100644 --- a/tests/tasks/uipath-maestro-bpmn/author/simple_approval_bpmn.yaml +++ b/tests/tasks/uipath-maestro-bpmn/author/simple_approval_bpmn.yaml @@ -39,15 +39,19 @@ initial_prompt: | - Do not upload, publish, deploy, debug, or run the process. success_criteria: - - type: file_exists + - type: run_command description: "BPMN source was created" - path: "ExpenseApprovalBpmn/ExpenseApprovalBpmn.bpmn" + command: "python3 -c \"from pathlib import Path; import sys; sys.exit(0 if any(Path('.').rglob('ExpenseApprovalBpmn.bpmn')) else 'ExpenseApprovalBpmn.bpmn missing')\"" + timeout: 30 + expected_exit_code: 0 weight: 1.0 pass_threshold: 1.0 - - type: file_exists - description: "Project metadata was created" - path: "ExpenseApprovalBpmn/project.uiproj" + - type: run_command + description: "Editable project source or metadata was created beside the BPMN" + command: "python3 -c \"from pathlib import Path; import sys; b=next(Path('.').rglob('ExpenseApprovalBpmn.bpmn'), None); sys.exit(0 if b and ((b.parent / 'project.uiproj').is_file() or b.with_suffix('.bpmn.ts').is_file()) else 'project.uiproj or ExpenseApprovalBpmn.bpmn.ts missing beside BPMN')\"" + timeout: 30 + expected_exit_code: 0 weight: 1.0 pass_threshold: 1.0 diff --git a/tests/tasks/uipath-maestro-bpmn/author/test_simple_approval_bpmn.py b/tests/tasks/uipath-maestro-bpmn/author/test_simple_approval_bpmn.py new file mode 100644 index 0000000000..801df09293 --- /dev/null +++ b/tests/tasks/uipath-maestro-bpmn/author/test_simple_approval_bpmn.py @@ -0,0 +1,46 @@ +import os +import sys +import xml.etree.ElementTree as ET + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from check_simple_approval_bpmn import NS, root_variable_identifiers # noqa: E402 + + +def _variable_identifiers(payload: str) -> set[str]: + root = ET.fromstring( + f'' + f"{payload}" + "" + ) + return root_variable_identifiers(root) + + +def test_variable_identity_accepts_sdk_ids_with_descriptive_names() -> None: + identifiers = _variable_identifiers( + '' + '' + '' + ) + + assert {"expenseId", "amount", "decision"} <= identifiers + + +def test_variable_identity_retains_legacy_name_compatibility() -> None: + identifiers = _variable_identifiers( + '' + '' + '' + ) + + assert {"expenseId", "amount", "decision"} <= identifiers + + +def test_variable_identity_rejects_unrelated_ids_and_names() -> None: + identifiers = _variable_identifiers( + '' + '' + ) + + assert not {"expenseId", "amount", "decision"} <= identifiers diff --git a/tests/tasks/uipath-maestro-bpmn/connector/integration_service_boundary.yaml b/tests/tasks/uipath-maestro-bpmn/connector/integration_service_boundary.yaml index 8f1c2d25fe..b7048acd79 100644 --- a/tests/tasks/uipath-maestro-bpmn/connector/integration_service_boundary.yaml +++ b/tests/tasks/uipath-maestro-bpmn/connector/integration_service_boundary.yaml @@ -31,9 +31,11 @@ initial_prompt: | - Do not upload, publish, deploy, debug, or run the process. success_criteria: - - type: file_exists + - type: run_command description: "BPMN source was created" - path: "SlackDigestBoundaryBpmn/SlackDigestBoundaryBpmn.bpmn" + command: "python3 -c \"from pathlib import Path; import sys; sys.exit(0 if any(Path('.').rglob('SlackDigestBoundaryBpmn.bpmn')) else 'SlackDigestBoundaryBpmn.bpmn missing')\"" + timeout: 30 + expected_exit_code: 0 weight: 1.0 pass_threshold: 1.0 diff --git a/tests/tasks/uipath-maestro-bpmn/debug/debug_workflow_mocked/debug_workflow_mocked.yaml b/tests/tasks/uipath-maestro-bpmn/debug/debug_workflow_mocked/debug_workflow_mocked.yaml index b09fd6d170..e060fae3b2 100644 --- a/tests/tasks/uipath-maestro-bpmn/debug/debug_workflow_mocked/debug_workflow_mocked.yaml +++ b/tests/tasks/uipath-maestro-bpmn/debug/debug_workflow_mocked/debug_workflow_mocked.yaml @@ -61,13 +61,11 @@ success_criteria: weight: 1.5 pass_threshold: 1.0 - - type: file_contains + - type: run_command description: "Summary reports the runtime variable values only obtainable from the debug session" - path: "summary.md" - includes: - - "1875" - - "AutoApproved" - - "Tier2" + command: "python3 -c \"from pathlib import Path; import sys; texts=[p.read_text(encoding='utf-8') for p in Path('.').rglob('summary.md')]; required=('1875', 'AutoApproved', 'Tier2'); sys.exit(0 if any(all(value in text for value in required) for text in texts) else 'no summary.md contains all runtime values')\"" + timeout: 30 + expected_exit_code: 0 weight: 3.0 pass_threshold: 1.0 diff --git a/tests/tasks/uipath-maestro-bpmn/multi_node/wiki_pageviews/wiki_pageviews.yaml b/tests/tasks/uipath-maestro-bpmn/multi_node/wiki_pageviews/wiki_pageviews.yaml index 75ee12fdd2..d4caddb060 100644 --- a/tests/tasks/uipath-maestro-bpmn/multi_node/wiki_pageviews/wiki_pageviews.yaml +++ b/tests/tasks/uipath-maestro-bpmn/multi_node/wiki_pageviews/wiki_pageviews.yaml @@ -53,7 +53,7 @@ initial_prompt: | success_criteria: - type: run_command description: "BPMN source file was created" - command: "python3 -c \"from pathlib import Path; import sys; paths=[Path('WikiPageviews/WikiPageviews.bpmn'), Path('WikiPageviewsSolution/WikiPageviews/WikiPageviews.bpmn')]; sys.exit(0 if any(p.is_file() for p in paths) else 'WikiPageviews.bpmn missing in direct or solution-wrapper layout')\"" + command: "python3 -c \"from pathlib import Path; import sys; sys.exit(0 if any(Path('.').rglob('WikiPageviews.bpmn')) else 'WikiPageviews.bpmn missing')\"" timeout: 30 expected_exit_code: 0 weight: 1.5 @@ -61,7 +61,7 @@ success_criteria: - type: run_command description: "Package operate metadata was created" - command: "python3 -c \"from pathlib import Path; import sys; paths=[Path('WikiPageviews/operate.json'), Path('WikiPageviewsSolution/WikiPageviews/operate.json')]; sys.exit(0 if any(p.is_file() for p in paths) else 'operate.json missing in direct or solution-wrapper layout')\"" + command: "python3 -c \"from pathlib import Path; import sys; sys.exit(0 if any(Path('.').rglob('operate.json')) else 'operate.json missing')\"" timeout: 30 expected_exit_code: 0 weight: 1.0 @@ -69,7 +69,7 @@ success_criteria: - type: run_command description: "The invalid-article path returns the fixed error string" - command: "python3 -c \"from pathlib import Path; import sys; paths=[Path('WikiPageviews/WikiPageviews.bpmn'), Path('WikiPageviewsSolution/WikiPageviews/WikiPageviews.bpmn')]; texts=[p.read_text(encoding='utf-8') for p in paths if p.is_file()]; sys.exit(0 if any('Article not found' in t for t in texts) else 'Article not found missing from WikiPageviews.bpmn')\"" + command: "python3 -c \"from pathlib import Path; import sys; texts=[p.read_text(encoding='utf-8') for p in Path('.').rglob('WikiPageviews.bpmn')]; sys.exit(0 if any('Article not found' in t for t in texts) else 'Article not found missing from WikiPageviews.bpmn')\"" timeout: 30 expected_exit_code: 0 weight: 1.0 diff --git a/tests/tasks/uipath-maestro-bpmn/nodes/hitl_rpa_wrappers.yaml b/tests/tasks/uipath-maestro-bpmn/nodes/hitl_rpa_wrappers.yaml index 6566a293b2..4775bce1b6 100644 --- a/tests/tasks/uipath-maestro-bpmn/nodes/hitl_rpa_wrappers.yaml +++ b/tests/tasks/uipath-maestro-bpmn/nodes/hitl_rpa_wrappers.yaml @@ -22,9 +22,11 @@ initial_prompt: | - Do not upload, publish, deploy, debug, or run the process. success_criteria: - - type: file_exists + - type: run_command description: "BPMN source was created" - path: "EmployeeAccessBpmn/EmployeeAccessBpmn.bpmn" + command: "python3 -c \"from pathlib import Path; import sys; sys.exit(0 if any(Path('.').rglob('EmployeeAccessBpmn.bpmn')) else 'EmployeeAccessBpmn.bpmn missing')\"" + timeout: 30 + expected_exit_code: 0 weight: 1.0 pass_threshold: 1.0 From dd71ed35e9670fc416be9e678c94ee18b8d44679 Mon Sep 17 00:00:00 2001 From: Tao Ma <51425734+tmatup@users.noreply.github.com> Date: Sun, 30 Aug 2026 12:04:25 -0700 Subject: [PATCH 26/52] Align Case evals across authoring loops (#2911) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Accept equivalent Case defaults, layouts, and audit shapes while keeping semantic requirements intact. ๐Ÿค– Generated with Claude Code Co-Authored-By: [Claude](mailto:noreply@anthropic.com) --- .../uipath-maestro-case/_shared/case_check.py | 101 +++++++++++++++++- .../_shared/test_case_check.py | 86 +++++++++++++++ .../connector_wait/check_connector_wait.py | 3 +- .../logging/logging_issue_log.yaml | 5 +- .../case_exit_full/check_case_exit_full.py | 4 +- .../check_linear_three_stages.py | 3 +- .../check_registry_handoff.py | 84 ++++++++------- .../check_at_risk_enter_stage.py | 3 +- .../check_case_breach_oversight.py | 3 +- .../check_case_breach_takeover.py | 3 +- 10 files changed, 244 insertions(+), 51 deletions(-) diff --git a/tests/tasks/uipath-maestro-case/_shared/case_check.py b/tests/tasks/uipath-maestro-case/_shared/case_check.py index 706374b937..73d5ab1a94 100644 --- a/tests/tasks/uipath-maestro-case/_shared/case_check.py +++ b/tests/tasks/uipath-maestro-case/_shared/case_check.py @@ -75,6 +75,35 @@ def read_caseplan(path: str | None = None) -> dict: return json.load(f) +def is_non_required(item: dict) -> bool: + """Accept the Case SDK's omitted default and the explicit false form.""" + value = item.get("isRequired") + return value is None or value is False + + +def registry_audit_entries(payload: object) -> list[dict]: + """Read equivalent flat and enveloped registry-audit representations.""" + if isinstance(payload, list): + entries = payload + elif isinstance(payload, dict): + entries = next( + ( + payload[key] + for key in ("resolutions", "resources") + if isinstance(payload.get(key), list) + ), + None, + ) + if entries is None: + _fail("registry-resolved.json has no resolutions/resources list") + else: + _fail("registry-resolved.json must be a list or object envelope") + + if not all(isinstance(entry, dict) for entry in entries): + _fail("registry-resolved.json entries must be objects") + return entries + + def iter_tasks(plan: dict): """Yield every task dict from every Stage node (and legacy ExceptionStage). @@ -456,7 +485,9 @@ def _get_ci(mapping: Any, *candidate_keys: str, default: Any = None) -> Any: def find_project_dir(pattern: str = "**/project.uiproj") -> str: """Return the directory holding the Case `project.uiproj`. Filters by ``ProjectType`` so a sibling Agent / RPA / Coded project in the same - solution does not collide with the Case project we want to debug. + solution does not collide with the Case project we want to debug. When a + preview author leaves both a substantive project and a generated + trigger-only scaffold, select the substantive project. """ candidates = sorted( p for p in glob.glob(pattern, recursive=True) if "/.venv/" not in p @@ -472,6 +503,12 @@ def find_project_dir(pattern: str = "**/project.uiproj") -> str: f"\n - {joined}" ) if len(case_projects) > 1: + selected, husks = _split_case_project_husks(case_projects) + if selected is not None: + print( + f"note: ignoring {len(husks)} abandoned Case project scaffold(s)" + ) + return os.path.dirname(selected) joined = "\n - ".join(case_projects) _fail( f"Multiple Case projects match {pattern!r} โ€” refusing to guess:" @@ -482,7 +519,9 @@ def find_project_dir(pattern: str = "**/project.uiproj") -> str: def find_solution_dir(pattern: str = "**/*.uipx") -> str: """Return the directory holding the ``*.uipx`` solution manifest. - Used as ``--solution-folder`` for ``uip solution resources refresh``. + Used as ``--solution-folder`` for ``uip solution resources refresh``. A + solution whose Case project is substantive wins over generated solutions + that contain only a trigger scaffold. """ matches = sorted( p for p in glob.glob(pattern, recursive=True) if "/.venv/" not in p @@ -490,11 +529,69 @@ def find_solution_dir(pattern: str = "**/*.uipx") -> str: if not matches: _fail(f"No solution manifest found matching {pattern}") if len(matches) > 1: + selected, husks = _split_case_solution_husks(matches) + if selected is not None: + print( + f"note: ignoring {len(husks)} abandoned Case solution scaffold(s)" + ) + return os.path.dirname(selected) joined = "\n - ".join(matches) _fail(f"Multiple solution manifests match {pattern!r}:\n - {joined}") return os.path.dirname(matches[0]) +def _split_case_project_husks( + project_uiprojs: list[str], +) -> tuple[str | None, list[str]]: + counts = [ + (path, _caseplan_node_count(os.path.dirname(path))) + for path in project_uiprojs + ] + return _split_case_husks(counts) + + +def _split_case_solution_husks( + solution_uipxs: list[str], +) -> tuple[str | None, list[str]]: + counts = [ + (path, _caseplan_node_count(os.path.dirname(path))) + for path in solution_uipxs + ] + return _split_case_husks(counts) + + +def _split_case_husks( + counts: list[tuple[str, int | None]], +) -> tuple[str | None, list[str]]: + substantive = [path for path, count in counts if count is None or count > 1] + husks = [path for path, count in counts if count is not None and count <= 1] + if len(substantive) != 1: + return None, [] + selected = substantive[0] + selected_count = next(count for path, count in counts if path == selected) + if selected_count is None: + return None, [] + return selected, husks + + +def _caseplan_node_count(root: str) -> int | None: + """Return the node count for one unambiguous Case plan under ``root``.""" + plans = sorted( + path + for path in glob.glob(os.path.join(root, "**/caseplan.json"), recursive=True) + if "/.venv/" not in path + ) + if len(plans) != 1: + return None + try: + with open(plans[0], encoding="utf-8") as f: + plan = json.load(f) + except (OSError, json.JSONDecodeError, UnicodeDecodeError): + return None + nodes = plan.get("nodes") if isinstance(plan, dict) else None + return len(nodes) if isinstance(nodes, list) else None + + def run_debug( *, timeout: int = 540, diff --git a/tests/tasks/uipath-maestro-case/_shared/test_case_check.py b/tests/tasks/uipath-maestro-case/_shared/test_case_check.py index 44c8b083c8..c7f412d6e8 100644 --- a/tests/tasks/uipath-maestro-case/_shared/test_case_check.py +++ b/tests/tasks/uipath-maestro-case/_shared/test_case_check.py @@ -22,7 +22,11 @@ _get_ci, collect_outputs, find_caseplan, + find_project_dir, + find_solution_dir, + is_non_required, partition_return_to_origin_conditions, + registry_audit_entries, run_debug, ) import case_check # noqa: E402 @@ -33,6 +37,16 @@ def _write_caseplan(path, nodes): path.write_text(json.dumps({"nodes": nodes}), encoding="utf-8") +def _write_case_project(root, solution, project, nodes): + project_dir = root / solution / project + _write_caseplan(project_dir / "caseplan.json", nodes) + (project_dir / "project.uiproj").write_text( + json.dumps({"ProjectType": "CaseManagement"}), encoding="utf-8" + ) + (root / solution / f"{solution}.uipx").write_text("{}", encoding="utf-8") + return project_dir + + def test_find_caseplan_prefers_one_substantive_plan_over_scaffold_husk(tmp_path, monkeypatch): authored = tmp_path / "Case" / "Case" / "caseplan.json" husk = tmp_path / "CaseSolution" / "Case" / "caseplan.json" @@ -63,6 +77,78 @@ def test_find_caseplan_rejects_distinct_substantive_plans(tmp_path, monkeypatch) find_caseplan() +def test_project_and_solution_discovery_prefer_substantive_case_over_husk( + tmp_path, monkeypatch, capsys +): + substantive = _write_case_project( + tmp_path, + "Case", + "Case", + [{"id": "trigger"}, {"id": "stage"}], + ) + _write_case_project( + tmp_path, + "CaseSolution", + "Case", + [{"id": "trigger"}], + ) + monkeypatch.chdir(tmp_path) + + assert find_project_dir() == os.path.relpath(substantive, tmp_path) + assert find_solution_dir() == "Case" + notes = capsys.readouterr().out + assert "ignoring 1 abandoned Case project scaffold(s)" in notes + assert "ignoring 1 abandoned Case solution scaffold(s)" in notes + + +def test_project_discovery_still_refuses_distinct_substantive_cases( + tmp_path, monkeypatch +): + _write_case_project( + tmp_path, + "FirstSolution", + "First", + [{"id": "trigger"}, {"id": "first"}], + ) + _write_case_project( + tmp_path, + "SecondSolution", + "Second", + [{"id": "trigger"}, {"id": "second"}], + ) + monkeypatch.chdir(tmp_path) + + with pytest.raises(SystemExit, match="Multiple Case projects match"): + find_project_dir() + with pytest.raises(SystemExit, match="Multiple solution manifests match"): + find_solution_dir() + + +def test_non_required_accepts_omitted_or_explicit_false(): + assert is_non_required({}) + assert is_non_required({"isRequired": False}) + + +def test_non_required_rejects_required_or_invalid_values(): + assert not is_non_required({"isRequired": True}) + assert not is_non_required({"isRequired": "false"}) + + +def test_registry_audit_entries_accepts_flat_and_named_envelopes(): + entries = [{"task": "Post Invoice"}] + + assert registry_audit_entries(entries) == entries + assert registry_audit_entries({"resolutions": entries}) == entries + assert registry_audit_entries({"resources": entries}) == entries + + +def test_registry_audit_entries_rejects_missing_or_non_object_entries(): + with pytest.raises(SystemExit, match="no resolutions/resources list"): + registry_audit_entries({"status": "resolved"}) + with pytest.raises(SystemExit, match="entries must be objects"): + registry_audit_entries({"resources": ["Post Invoice"]}) + + def test_get_ci_reads_camelcase_and_pascalcase(): assert _get_ci({"finalStatus": "Completed"}, "finalStatus", "FinalStatus") == "Completed" assert _get_ci({"FinalStatus": "Completed"}, "finalStatus", "FinalStatus") == "Completed" diff --git a/tests/tasks/uipath-maestro-case/connector_features/connector_wait/check_connector_wait.py b/tests/tasks/uipath-maestro-case/connector_features/connector_wait/check_connector_wait.py index a8ecfe9181..b994f177f1 100644 --- a/tests/tasks/uipath-maestro-case/connector_features/connector_wait/check_connector_wait.py +++ b/tests/tasks/uipath-maestro-case/connector_features/connector_wait/check_connector_wait.py @@ -18,6 +18,7 @@ from _shared.case_check import ( # noqa: E402 assert_task_type_present, first_rule_of_condition, + is_non_required, iter_tasks, read_caseplan, task_is_skeleton, @@ -94,7 +95,7 @@ def main(): f"FAIL: entry rule {slot}[{entry.get('name')!r}].elementId " f"must be {expected_element_id!r}; got {entry.get('elementId')!r}" ) - if event_task.get("isRequired") is not False: + if not is_non_required(event_task): sys.exit( "FAIL: event-triggered placeholder process should stay non-required; " f"got isRequired={event_task.get('isRequired')!r}" diff --git a/tests/tasks/uipath-maestro-case/logging/logging_issue_log.yaml b/tests/tasks/uipath-maestro-case/logging/logging_issue_log.yaml index 91dec2bafa..ad12278b8a 100644 --- a/tests/tasks/uipath-maestro-case/logging/logging_issue_log.yaml +++ b/tests/tasks/uipath-maestro-case/logging/logging_issue_log.yaml @@ -131,8 +131,9 @@ initial_prompt: | Run `uip maestro case validate TimerLogCase/TimerLogCase/caseplan.json --output json`. Validation is expected to fail due to the invalid exitType; that is not a reason to stop. The task is NOT complete until validate has run - and the build's issue log is on disk. Follow the skill's own logging contract - for when and how that log is written. + and `tasks/build-issues.md` is on disk. Keep that file as an append-only build + journal: include a heading naming TimerLogCase, a `## Journal` table with the + validation failure, and a completed summary of the recorded issue. success_criteria: - type: run_command diff --git a/tests/tasks/uipath-maestro-case/multi_stages_tasks/case_exit_full/check_case_exit_full.py b/tests/tasks/uipath-maestro-case/multi_stages_tasks/case_exit_full/check_case_exit_full.py index 95e44156de..26709d71d6 100644 --- a/tests/tasks/uipath-maestro-case/multi_stages_tasks/case_exit_full/check_case_exit_full.py +++ b/tests/tasks/uipath-maestro-case/multi_stages_tasks/case_exit_full/check_case_exit_full.py @@ -13,6 +13,7 @@ find_stages, first_rule_of_condition, get_case_exit_conditions, + is_non_required, iter_stage_entry_conditions, iter_stage_exit_conditions, read_caseplan, @@ -47,7 +48,8 @@ def main(): } for name, (stage, want) in expected_required.items(): got = _is_required(stage) - if got is not want: + matches = got is True if want else is_non_required(stage.get("data") or {}) + if not matches: sys.exit( f"FAIL: stage {name!r} isRequired should be {want}; got {got!r}" ) diff --git a/tests/tasks/uipath-maestro-case/multi_stages_tasks/linear_three_stages/check_linear_three_stages.py b/tests/tasks/uipath-maestro-case/multi_stages_tasks/linear_three_stages/check_linear_three_stages.py index 4f86856f06..1ee3634bcb 100644 --- a/tests/tasks/uipath-maestro-case/multi_stages_tasks/linear_three_stages/check_linear_three_stages.py +++ b/tests/tasks/uipath-maestro-case/multi_stages_tasks/linear_three_stages/check_linear_three_stages.py @@ -16,6 +16,7 @@ get_variables, iter_stage_entry_conditions, iter_tasks, + is_non_required, read_caseplan, start_debug, ) @@ -110,7 +111,7 @@ def _by_label(label: str) -> dict | None: f"FAIL: 'Hold For 1 Hour' skipCondition should reference skipReview; " f"got {skip!r}" ) - if notify.get("isRequired") is not False: + if not is_non_required(notify): sys.exit( f"FAIL: 'Notify Reviewer' should have isRequired=false (task-level " f"flag explicitly set); got {notify.get('isRequired')!r}" diff --git a/tests/tasks/uipath-maestro-case/registry_handoff_sdd_only/check_registry_handoff.py b/tests/tasks/uipath-maestro-case/registry_handoff_sdd_only/check_registry_handoff.py index 7d68fd5e64..8461f828be 100644 --- a/tests/tasks/uipath-maestro-case/registry_handoff_sdd_only/check_registry_handoff.py +++ b/tests/tasks/uipath-maestro-case/registry_handoff_sdd_only/check_registry_handoff.py @@ -1,6 +1,11 @@ import json +import os +import sys from pathlib import Path +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +from _shared.case_check import registry_audit_entries # noqa: E402 + EXPECTED = { "FinancialPostingFunction": { @@ -15,16 +20,6 @@ }, } STAGE = "Resolve Resources" -REQUIRED_KEYS = { - "stage", - "task", - "taskType", - "cacheFile", - "searchQuery", - "matches", - "selected", - "rationale", -} def load_json(path: Path): @@ -34,7 +29,7 @@ def load_json(path: Path): registry_path = Path("tasks/registry-resolved.json") entries = load_json(registry_path) -assert isinstance(entries, list), "registry-resolved.json must be a list" +entries = registry_audit_entries(entries) assert len(entries) == len(EXPECTED), ( f"expected one fresh audit entry per SDD task, got {len(entries)}" ) @@ -45,34 +40,36 @@ def load_json(path: Path): matching_entries = [ entry for entry in entries - if str(entry.get("searchQuery", "")).strip() == name + if str(entry.get("searchQuery") or entry.get("resolvedResource") or "").strip() + == name ] assert len(matching_entries) == 1, ( f"expected one registry audit entry for {name}, got {len(matching_entries)}" ) entry = matching_entries[0] - assert REQUIRED_KEYS <= entry.keys(), ( - f"{name} audit entry is missing {sorted(REQUIRED_KEYS - entry.keys())}" - ) - assert str(entry.get("rationale", "")).strip(), ( - f"{name} audit entry has no selection rationale" - ) - assert entry.get("stage") == STAGE, ( - f"{name} audit entry is associated with stage {entry.get('stage')!r}" - ) - assert entry.get("task") == expected["task"], ( - f"{name} audit entry is associated with task {entry.get('task')!r}" + if "stage" in entry: + assert entry["stage"] == STAGE, ( + f"{name} audit entry is associated with stage {entry['stage']!r}" + ) + task_name = entry.get("task") or entry.get("taskName") + assert task_name == expected["task"], ( + f"{name} audit entry is associated with task {task_name!r}" ) assert entry.get("taskType") == expected["task_type"], ( f"{name} used taskType {entry.get('taskType')!r}" ) - assert entry.get("cacheFile") == expected["cache_file"], ( - f"{name} did not record cacheFile {expected['cache_file']}" - ) + if "cacheFile" in entry: + assert entry["cacheFile"] == expected["cache_file"], ( + f"{name} did not record cacheFile {expected['cache_file']}" + ) + assert str(entry.get("rationale") or "").strip() or isinstance( + entry.get("resolution"), dict + ), f"{name} audit entry has no resolution evidence" - selected = entry.get("selected") + selected = entry.get("selected") or entry.get("resourceIdentity") assert isinstance(selected, dict), f"missing selected result for {name}" - assert selected.get("name") == name, f"selected the wrong resource for {name}" + selected_name = selected.get("name") or entry.get("resolvedResource") + assert selected_name == name, f"selected the wrong resource for {name}" cached_resources = load_json(cache_root / expected["cache_file"]) exact_matches = [ @@ -89,18 +86,18 @@ def load_json(path: Path): # Also tolerant of ephemeral debug-deploy churn: we require the SELECTED entry to be a # live exact-name match and every recorded candidate to be exact-name, not a byte-for- # byte copy of the whole cache set. - recorded = entry.get("matches") or [] - assert recorded, f"{name} audit recorded no matches" - # The skill's discovery search is substring-based, so `matches` may carry near-name - # candidates (e.g. a `V2` sibling) alongside the exact hit. Correctness is proven - # by `selected` below (exact-name + live entityKey); here we only require the exact-name - # match to be PRESENT among the recorded candidates, not that every candidate is exact. - assert any( - str(m.get("name", "")).strip().casefold() == name.casefold() for m in recorded - ), f"{name} audit recorded no exact-name match" - assert isinstance(selected, dict), f"missing selected result for {name}" + recorded = entry.get("matches") + if recorded is not None: + assert recorded, f"{name} audit recorded no matches" + # The skill's discovery search is substring-based, so `matches` may carry + # near-name candidates alongside the exact hit. Correctness is proven by + # `selected`; here we only require the exact-name match to be present. + assert any( + str(m.get("name", "")).strip().casefold() == name.casefold() + for m in recorded + ), f"{name} audit recorded no exact-name match" assert ( - str(selected.get("name", "")).strip().casefold() == name.casefold() + str(selected_name).strip().casefold() == name.casefold() and str(selected.get("entityKey")) in cache_keys ), f"selected result for {name} is not a live exact-name cache entry" @@ -110,10 +107,15 @@ def load_json(path: Path): # flat (folder/folderType) or nested (folders[0].fullyQualifiedName/.type). fol = selected.get("folders") or [] sel_folder = str( - selected.get("folder") or (fol[0].get("fullyQualifiedName") if fol else "") + selected.get("folder") + or selected.get("folderPath") + or entry.get("folderPath") + or (fol[0].get("fullyQualifiedName") if fol else "") ).casefold() sel_ftype = str( - selected.get("folderType") or (fol[0].get("type") if fol else "") + selected.get("folderType") + or selected.get("organizationUnitType") + or (fol[0].get("type") if fol else "") ).casefold() assert sel_ftype != "debugsolution" and "debug_" not in sel_folder, ( f"{name} resolved to an ephemeral debug-solution deploy " diff --git a/tests/tasks/uipath-maestro-case/sla_response/at_risk_enter_stage/check_at_risk_enter_stage.py b/tests/tasks/uipath-maestro-case/sla_response/at_risk_enter_stage/check_at_risk_enter_stage.py index eef4531a94..a5a36d0d4d 100644 --- a/tests/tasks/uipath-maestro-case/sla_response/at_risk_enter_stage/check_at_risk_enter_stage.py +++ b/tests/tasks/uipath-maestro-case/sla_response/at_risk_enter_stage/check_at_risk_enter_stage.py @@ -13,6 +13,7 @@ assert_stage_count, fail, iter_sla_status_change, + is_non_required, label_of, read_plan, secondary_stages, @@ -33,7 +34,7 @@ def main() -> None: lane = lanes[0] lane_data = lane["data"] - if lane_data.get("isRequired") is not False: + if not is_non_required(lane_data): fail( f"lane {label_of(lane)!r} has isRequired={lane_data.get('isRequired')!r}; an SLA lane " "must stay out of the happy-path required-stages-completed set" diff --git a/tests/tasks/uipath-maestro-case/sla_response/case_breach_oversight/check_case_breach_oversight.py b/tests/tasks/uipath-maestro-case/sla_response/case_breach_oversight/check_case_breach_oversight.py index c94f5dd9ef..0ecd0ddc8e 100644 --- a/tests/tasks/uipath-maestro-case/sla_response/case_breach_oversight/check_case_breach_oversight.py +++ b/tests/tasks/uipath-maestro-case/sla_response/case_breach_oversight/check_case_breach_oversight.py @@ -19,6 +19,7 @@ fail, is_secondary, iter_sla_status_change, + is_non_required, label_of, read_plan, ) @@ -46,7 +47,7 @@ def main() -> None: "non-interrupting SLA oversight lane still stays `secondary` โ€” promoting it to a " "regular stage would make it required for case completion" ) - if lane["data"].get("isRequired") is not False: + if not is_non_required(lane["data"]): fail( f"lane {label_of(lane)!r} has isRequired={lane['data'].get('isRequired')!r}; an " "oversight lane must stay out of the required-stages-completed set" diff --git a/tests/tasks/uipath-maestro-case/sla_response/case_breach_takeover/check_case_breach_takeover.py b/tests/tasks/uipath-maestro-case/sla_response/case_breach_takeover/check_case_breach_takeover.py index 2ab719570f..6787afa7d0 100644 --- a/tests/tasks/uipath-maestro-case/sla_response/case_breach_takeover/check_case_breach_takeover.py +++ b/tests/tasks/uipath-maestro-case/sla_response/case_breach_takeover/check_case_breach_takeover.py @@ -13,6 +13,7 @@ assert_stage_count, fail, iter_sla_status_change, + is_non_required, label_of, read_plan, secondary_stages, @@ -31,7 +32,7 @@ def main() -> None: f"{[label_of(n) for n in lanes]}" ) lane = lanes[0] - if lane["data"].get("isRequired") is not False: + if not is_non_required(lane["data"]): fail(f"lane {label_of(lane)!r} must be isRequired False") hits = [(n, c, r) for n, c, r in iter_sla_status_change(plan) if n is lane] From f0eb75ac3e3c178b8eec776d8fcaeeacd68019ec Mon Sep 17 00:00:00 2001 From: Tao Ma <51425734+tmatup@users.noreply.github.com> Date: Sun, 30 Aug 2026 12:04:41 -0700 Subject: [PATCH 27/52] fix(maestro-flow): make artifact discovery author-loop neutral (#2913) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Resolve one substantive Flow project across equivalent live and SDK authoring layouts while retaining ambiguity checks for genuinely distinct artifacts. ๐Ÿค– Generated with Claude Code Co-Authored-By: [Claude](mailto:noreply@anthropic.com) --- .../uipath-maestro-flow/_shared/flow_check.py | 110 +++++++++++++++--- .../_shared/test_flow_check_discovery.py | 47 ++++++++ 2 files changed, 138 insertions(+), 19 deletions(-) diff --git a/tests/tasks/uipath-maestro-flow/_shared/flow_check.py b/tests/tasks/uipath-maestro-flow/_shared/flow_check.py index 82ad7728e8..e190601bc2 100644 --- a/tests/tasks/uipath-maestro-flow/_shared/flow_check.py +++ b/tests/tasks/uipath-maestro-flow/_shared/flow_check.py @@ -1523,17 +1523,46 @@ def find_flow_files( ) -> list[str]: """Return flow artifacts from the selected solution, or one root emit. - A Flow solution project wins whenever one exists. This keeps a root-level - SDK scratch file from competing with the linked solution copy. - When no project exists, validate and source-structure checks may consume a - root-level SDK emit. Multiple genuinely distinct root emits are ambiguous - and fail with their paths rather than selecting one by glob order. + A substantive Flow solution project wins whenever one exists. This keeps a + root-level SDK scratch file from competing with the linked solution copy. + When no project exists, or every project is only an abandoned scaffold, + validate and source-structure checks may consume a substantive root-level + SDK emit. Multiple genuinely distinct root emits are ambiguous and fail + with their paths rather than selecting one by glob order. Debug callers intentionally do not use this helper: ``uip maestro flow debug`` is project-scoped and must continue through :func:`find_project_dir`. """ project_candidates = sorted(glob.glob(project_pattern, recursive=True)) - if any(_is_flow_project(path) for path in project_candidates): + flow_projects = [path for path in project_candidates if _is_flow_project(path)] + root_matches = _dedupe_flow_candidates( + sorted(glob.glob(os.path.basename(flow_glob))) + ) + + # A preview author may compile a complete root-level SDK source before a + # later CLI command leaves an untouched trigger-only project scaffold. For + # file-scoped checks the substantive artifact is the build; the scaffold is + # not allowed to shadow it merely because it carries project.uiproj. + if flow_projects and len(root_matches) == 1: + project_counts = [ + _flow_node_count(os.path.dirname(path)) for path in flow_projects + ] + root_count = _flow_file_node_count(root_matches[0]) + if ( + root_count is not None + and root_count > _HUSK_MAX_NODES + and all( + count is not None and count <= _HUSK_MAX_NODES + for count in project_counts + ) + ): + print( + "note: ignoring abandoned project scaffold(s) in favor of " + f"substantive root Flow emit: {root_matches[0]}" + ) + return root_matches + + if flow_projects: project_dir = _find_project(project_pattern) matches = sorted( glob.glob( @@ -1548,9 +1577,6 @@ def find_flow_files( ) return matches - root_matches = _dedupe_flow_candidates( - sorted(glob.glob(os.path.basename(flow_glob))) - ) if not root_matches: _fail( f"No Flow project found matching {project_pattern!r}, and no " @@ -1676,8 +1702,9 @@ def _find_project(pattern: str) -> str: Filtering by manifest avoids a 1-of-N glob collision the symptom of MST-9734. - Two Flow projects can also mean one build plus one abandoned scaffold โ€” - see :func:`_split_off_scaffold_husks`. Anything else stays a refusal. + Two Flow projects can also mean byte-identical copies, or one build plus one + abandoned scaffold โ€” see :func:`_dedupe_flow_projects` and + :func:`_split_off_scaffold_husks`. Anything else stays a refusal. """ candidates = sorted(glob.glob(pattern, recursive=True)) if not candidates: @@ -1690,6 +1717,15 @@ def _find_project(pattern: str) -> str: f'candidates exist but none declare ProjectType="Flow":\n - {joined}' ) if len(flow_projects) > 1: + original_count = len(flow_projects) + flow_projects = _dedupe_flow_projects(flow_projects) + if len(flow_projects) == 1: + print( + "note: ignoring " + f"{original_count - 1} byte-identical Flow project duplicate(s)" + ) + return os.path.dirname(flow_projects[0]) + counts = [(p, _flow_node_count(os.path.dirname(p))) for p in flow_projects] selected, husks = _split_off_scaffold_husks(counts) if selected is not None: @@ -1736,18 +1772,54 @@ def _flow_node_count(project_dir: str) -> int | None: return None total = 0 for path in flows: - try: - with open(path, encoding="utf-8") as f: - flow = json.load(f) - except (OSError, json.JSONDecodeError, UnicodeDecodeError): + count = _flow_file_node_count(path) + if count is None: return None - nodes = flow.get("nodes") if isinstance(flow, dict) else None - if not isinstance(nodes, list): - return None - total += len(nodes) + total += count return total +def _flow_file_node_count(path: str) -> int | None: + """Return one Flow file's node count, or ``None`` when unreadable.""" + try: + with open(path, encoding="utf-8") as f: + flow = json.load(f) + except (OSError, json.JSONDecodeError, UnicodeDecodeError): + return None + nodes = flow.get("nodes") if isinstance(flow, dict) else None + return len(nodes) if isinstance(nodes, list) else None + + +def _dedupe_flow_projects(project_uiprojs: list[str]) -> list[str]: + """Collapse projects whose relative Flow files are byte-for-byte equal. + + Missing or unreadable Flow files stay distinct so project-scoped debug never + guesses across candidates whose equivalence cannot be proved. + """ + by_content: dict[tuple[tuple[str, str], ...] | tuple[str, str], str] = {} + for project_uiproj in project_uiprojs: + project_dir = os.path.dirname(project_uiproj) + flows = sorted( + glob.glob(os.path.join(project_dir, "**/*.flow"), recursive=True) + ) + fingerprints: list[tuple[str, str]] = [] + try: + for path in flows: + with open(path, "rb") as f: + digest = hashlib.sha256(f.read()).hexdigest() + fingerprints.append((os.path.relpath(path, project_dir), digest)) + except OSError: + fingerprints = [] + + key: tuple[tuple[str, str], ...] | tuple[str, str] + if fingerprints: + key = tuple(fingerprints) + else: + key = ("unknown", project_uiproj) + by_content.setdefault(key, project_uiproj) + return sorted(by_content.values()) + + def _describe_candidate(project_uiproj: str, node_count: int | None) -> str: project_dir = os.path.dirname(project_uiproj) if node_count is None: diff --git a/tests/tasks/uipath-maestro-flow/_shared/test_flow_check_discovery.py b/tests/tasks/uipath-maestro-flow/_shared/test_flow_check_discovery.py index 52c3547f76..fdc1a3a38a 100644 --- a/tests/tasks/uipath-maestro-flow/_shared/test_flow_check_discovery.py +++ b/tests/tasks/uipath-maestro-flow/_shared/test_flow_check_discovery.py @@ -92,6 +92,25 @@ def test_refuses_when_two_candidates_are_substantive(tmp_path, monkeypatch): _find_project(PATTERN) +def test_collapses_byte_identical_substantive_projects( + tmp_path, monkeypatch, capsys +): + monkeypatch.chdir(tmp_path) + body = json.dumps( + {"nodes": [{"id": f"n{i}", "type": "core.action.script"} for i in range(3)]} + ) + first = _make_flow_project( + tmp_path, "SolutionA", "Build", 3, flow_body=body + ) + _make_flow_project(tmp_path, "SolutionB", "Build", 3, flow_body=body) + + assert _find_project(PATTERN) == os.path.relpath(first, tmp_path) + assert ( + "ignoring 1 byte-identical Flow project duplicate(s)" + in capsys.readouterr().out + ) + + def test_refuses_when_a_candidate_flow_is_unreadable(tmp_path, monkeypatch): """Unknown node count is not a husk โ€” stay conservative and refuse.""" monkeypatch.chdir(tmp_path) @@ -180,6 +199,34 @@ def test_solution_flow_beats_root_scratch_emit(tmp_path, monkeypatch): assert project.name == "Build" +def test_substantive_root_emit_beats_abandoned_project_scaffold( + tmp_path, monkeypatch, capsys +): + monkeypatch.chdir(tmp_path) + _make_flow_project(tmp_path, "GeneratedSolution", "Build", 1) + root = tmp_path / "Build.flow" + root.write_text( + json.dumps( + {"nodes": [{"id": f"n{i}", "type": "core.action.script"} for i in range(4)]} + ) + ) + + assert find_flow_files() == ["Build.flow"] + assert find_flow_file() == "Build.flow" + assert "ignoring abandoned project scaffold(s)" in capsys.readouterr().out + + +def test_root_husk_does_not_override_project_husk(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + project = _make_flow_project(tmp_path, "Solution", "Build", 1) + (tmp_path / "Build.flow").write_text( + json.dumps({"nodes": [{"id": "start", "type": "core.trigger.manual"}]}) + ) + + assert find_flow_file() == os.path.join("Solution", "Build", "Build.flow") + assert project.name == "Build" + + def test_root_fallback_refuses_distinct_candidates_and_lists_them( tmp_path, monkeypatch ): From 15b9ddc0e8e81a272969180c533887ad9b040628 Mon Sep 17 00:00:00 2001 From: Tao Ma <51425734+tmatup@users.noreply.github.com> Date: Sun, 30 Aug 2026 13:05:18 -0700 Subject: [PATCH 28/52] fix(maestro-flow): remove remaining arm-specific checks (#2915) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Normalize Flow validation and checker contracts across the live and SDK authoring loops, including offline connector tasks. ๐Ÿค– Generated with Claude Code Co-Authored-By: [Claude](mailto:noreply@anthropic.com) --- .../connector_features/ceql_where.yaml | 18 +++--- .../connector_features/enhanced_enum.yaml | 15 ++--- .../check_webhook_waitfor_parallel.py | 35 +++++++---- .../test_check_webhook_waitfor_parallel.py | 58 +++++++++++++++++++ .../webhook_waitfor_parallel.yaml | 7 ++- .../escalation_slack_alert.yaml | 4 +- 6 files changed, 100 insertions(+), 37 deletions(-) create mode 100644 tests/tasks/uipath-maestro-flow/connector_trigger/test_check_webhook_waitfor_parallel.py diff --git a/tests/tasks/uipath-maestro-flow/connector_features/ceql_where.yaml b/tests/tasks/uipath-maestro-flow/connector_features/ceql_where.yaml index 6a96f7c287..4d1d1909f7 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/ceql_where.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_features/ceql_where.yaml @@ -9,8 +9,10 @@ description: > operation with a `displayName = "active"` filter, following the canonical shape documented in the Filter Trees (CEQL) section of the uipath-platform skill. The flow must reference the registered connector - key (`uipath-microsoft-azureactivedirectory`), use Decision + Terminate - for routing, and pass `uip maestro flow validate`. + key (`uipath-microsoft-azureactivedirectory`) and use Decision + Terminate + for routing. This is an offline structure test: product validation is not + graded because the two authoring loops persist an unconfigured connector + differently. tags: [uipath-maestro-flow, integration, connector, ceql, filter, uipath-microsoft-azureactivedirectory, ipe, mode:build] run_limits: @@ -30,7 +32,9 @@ initial_prompt: | Branch on the result: if the call fails, stop the flow immediately; if it succeeds, log "CeqlWhere test passed". - Validate the final flow file. + Compile the final flow file. Do not fabricate a tenant connection merely to + make product validation pass; the generated structure and the separately + reviewable filter are the deliverables in this offline task. success_criteria: - type: run_command @@ -40,11 +44,3 @@ success_criteria: expected_exit_code: 0 weight: 8.0 pass_threshold: 1.0 - - - type: run_command - description: "The completed flow validates" - command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py" - timeout: 30 - expected_exit_code: 0 - weight: 2.0 - pass_threshold: 1.0 diff --git a/tests/tasks/uipath-maestro-flow/connector_features/enhanced_enum.yaml b/tests/tasks/uipath-maestro-flow/connector_features/enhanced_enum.yaml index 13189c7b6f..29d1680003 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/enhanced_enum.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_features/enhanced_enum.yaml @@ -19,10 +19,11 @@ initial_prompt: | Build a flow that retrieves WooCommerce product reviews and lets the caller choose whether they come back oldest-first or newest-first, picking from friendly labels rather than raw codes. Discover the get-product-reviews operation and set - its sort-order field. Validate the final flow file. + its sort-order field. Compile the final flow file. If the WooCommerce operation exists in the registry but the tenant has no - live connection, keep the real WooCommerce operation node with its inputs left unset, - confirm that shape validates, and stop there. + live connection, keep the real WooCommerce operation node with its inputs left unset + and stop there. Do not fabricate a tenant connection merely to make product + validation pass; the connector shape is the deliverable in this offline task. success_criteria: - type: run_command @@ -40,11 +41,3 @@ success_criteria: expected_exit_code: 0 weight: 3.0 pass_threshold: 1.0 - - - type: run_command - description: "The completed flow validates" - command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py" - timeout: 30 - expected_exit_code: 0 - weight: 2.0 - pass_threshold: 1.0 diff --git a/tests/tasks/uipath-maestro-flow/connector_trigger/check_webhook_waitfor_parallel.py b/tests/tasks/uipath-maestro-flow/connector_trigger/check_webhook_waitfor_parallel.py index 1cfcdd5436..c8a894c895 100644 --- a/tests/tasks/uipath-maestro-flow/connector_trigger/check_webhook_waitfor_parallel.py +++ b/tests/tasks/uipath-maestro-flow/connector_trigger/check_webhook_waitfor_parallel.py @@ -5,7 +5,8 @@ manual start trigger โ”œโ”€ branch 1: Wait-for-event node (HTTP Webhook, uipath-http-webhook) -> End - โ””โ”€ branch 2: Managed HTTP Request (core.action.http.v2, manual GET to the + โ””โ”€ branch 2: Managed HTTP Request (core.action.http or + core.action.http.v2, manual GET to the HTTP Webhook connection's webhook URL, no headers / no query) -> End Branch 2's GET hits the webhook URL, which delivers the event that completes @@ -17,8 +18,9 @@ branches (the event wait and the HTTP request). 2. A Wait-for-event node of the HTTP Webhook connector event exists (`uipath.connector.event.uipath-http-webhook.http-webhook`). - 3. A `core.action.http.v2` node is a MANUAL `GET` whose `url` is the webhook - URL, with NOTHING in headers or query (per the prompt). + 3. A `core.action.http` or `core.action.http.v2` node is a MANUAL `GET` whose + `url` is the webhook URL, with NOTHING in headers or query (per the + prompt). 4. Both branch tails reach an End node (`core.control.end`). """ @@ -33,7 +35,7 @@ FLOW_GLOB = "**/WebhookSelfTest*.flow" EVENT_MARKERS = ("uipath.connector.event", "uipath-http-webhook") -HTTP_TYPE = "core.action.http.v2" +HTTP_TYPES = ("core.action.http", "core.action.http.v2") END_TYPE = "core.control.end" TRIGGER_PREFIX = "core.trigger." @@ -96,20 +98,33 @@ def main() -> None: print("OK: HTTP Webhook wait-for-event node present") # 3. Manual GET HTTP node to the webhook URL, no headers / no query. - http_nodes = [n for n in nodes if str(n.get("type", "")) == HTTP_TYPE] + http_nodes = [n for n in nodes if str(n.get("type", "")) in HTTP_TYPES] if not http_nodes: - _fail(f"no {HTTP_TYPE} node. Types seen: {types_seen}") + _fail( + f"no managed HTTP node ({' or '.join(HTTP_TYPES)}). " + f"Types seen: {types_seen}" + ) good_http = None for n in http_nodes: - body = ((n.get("inputs") or {}).get("detail") or {}).get("bodyParameters") or {} + inputs = n.get("inputs") or {} + body = ((inputs.get("detail") or {}).get("bodyParameters") or inputs) if not isinstance(body, dict): continue - auth = str(body.get("authentication", "")).lower() + auth = str( + body.get("authentication") + or body.get("authenticationType") + or body.get("mode") + or "" + ).lower() method = str(body.get("method", "")).lower() url = str(body.get("url", "")) headers = body.get("headers") - query = body.get("query") or body.get("queryParameters") + query = ( + body.get("query") + or body.get("queryParameters") + or body.get("queryParams") + ) if auth != "manual" or method != "get": continue if "webhook" not in url.lower(): @@ -123,7 +138,7 @@ def main() -> None: if good_http is None: _fail( - "no core.action.http.v2 node configured as a manual GET to the " + "no managed HTTP node configured as a manual GET to the " "webhook URL (authentication=manual, method=GET, url contains 'webhook')" ) print("OK: manual GET HTTP node to webhook URL, no headers / no query") diff --git a/tests/tasks/uipath-maestro-flow/connector_trigger/test_check_webhook_waitfor_parallel.py b/tests/tasks/uipath-maestro-flow/connector_trigger/test_check_webhook_waitfor_parallel.py new file mode 100644 index 0000000000..b68cb49e9c --- /dev/null +++ b/tests/tasks/uipath-maestro-flow/connector_trigger/test_check_webhook_waitfor_parallel.py @@ -0,0 +1,58 @@ +import importlib.util +import json +from pathlib import Path + +import pytest + + +CHECKER_PATH = Path(__file__).with_name("check_webhook_waitfor_parallel.py") +SPEC = importlib.util.spec_from_file_location("check_webhook_waitfor_parallel", CHECKER_PATH) +assert SPEC and SPEC.loader +CHECKER = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(CHECKER) + + +@pytest.mark.parametrize("http_type", ["core.action.http", "core.action.http.v2"]) +def test_accepts_both_managed_http_representations(tmp_path, monkeypatch, http_type): + monkeypatch.chdir(tmp_path) + if http_type.endswith(".v2"): + inputs = { + "detail": { + "bodyParameters": { + "authentication": "manual", + "method": "GET", + "url": "https://example.test/webhook/events/1", + "headers": {}, + "query": {}, + } + } + } + else: + inputs = { + "authenticationType": "manual", + "method": "GET", + "url": "https://example.test/webhook/events/1", + "headers": {}, + "queryParams": {}, + } + + flow = { + "nodes": [ + {"id": "start", "type": "core.trigger.manual"}, + { + "id": "wait", + "type": "uipath.connector.event.uipath-http-webhook.http-webhook", + }, + {"id": "get", "type": http_type, "inputs": inputs}, + {"id": "end", "type": "core.control.end"}, + ], + "edges": [ + {"sourceNodeId": "start", "targetNodeId": "wait"}, + {"sourceNodeId": "start", "targetNodeId": "get"}, + {"sourceNodeId": "wait", "targetNodeId": "end"}, + {"sourceNodeId": "get", "targetNodeId": "end"}, + ], + } + (tmp_path / "WebhookSelfTest.flow").write_text(json.dumps(flow)) + + CHECKER.main() diff --git a/tests/tasks/uipath-maestro-flow/connector_trigger/webhook_waitfor_parallel.yaml b/tests/tasks/uipath-maestro-flow/connector_trigger/webhook_waitfor_parallel.yaml index fdc25b7ac4..521c211ec8 100644 --- a/tests/tasks/uipath-maestro-flow/connector_trigger/webhook_waitfor_parallel.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_trigger/webhook_waitfor_parallel.yaml @@ -7,9 +7,10 @@ description: > E2E self-testing flow: a manual start trigger fans out into two parallel branches. Branch 1 is a mid-flow Wait-for-event node bound to the HTTP Webhook connector (`uipath.connector.event.uipath-http-webhook.http-webhook`). Branch 2 - is a Managed HTTP Request (`core.action.http.v2`, manual GET) whose URL is the - webhook URL of that same HTTP Webhook connection, with nothing in headers or - query. The GET self-delivers the event that completes the wait at runtime. + is a Managed HTTP Request (`core.action.http` or `core.action.http.v2`, manual + GET) whose URL is the webhook URL of that same HTTP Webhook connection, with + nothing in headers or query. The GET self-delivers the event that completes + the wait at runtime. Exercises the connector-trigger plugin's "Wait for events" variant, parallel branch fan-out from the start trigger, manual-mode HTTP, and the CLI webhook-URL wiring. Validates the static two-branch shape. diff --git a/tests/tasks/uipath-maestro-flow/e2e/escalation_slack_alert/escalation_slack_alert.yaml b/tests/tasks/uipath-maestro-flow/e2e/escalation_slack_alert/escalation_slack_alert.yaml index 0a8076c810..b7efd19bef 100644 --- a/tests/tasks/uipath-maestro-flow/e2e/escalation_slack_alert/escalation_slack_alert.yaml +++ b/tests/tasks/uipath-maestro-flow/e2e/escalation_slack_alert/escalation_slack_alert.yaml @@ -73,8 +73,8 @@ success_criteria: # โ”€โ”€ The delivered flow validates โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ - type: run_command - description: "Generated EscalationSlackAlert Flow validates without errors" - command: "uip maestro flow validate EscalationSlackAlert/EscalationSlackAlert/EscalationSlackAlert.flow --output json" + description: "Generated EscalationSlackAlert Flow validates without errors, independent of solution layout" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py" timeout: 180 expected_exit_code: 0 weight: 3.0 From fd9500f08e94a8cef0690ababdf4fbe67da3c1b2 Mon Sep 17 00:00:00 2001 From: Tao Ma <51425734+tmatup@users.noreply.github.com> Date: Sun, 30 Aug 2026 21:55:23 -0700 Subject: [PATCH 29/52] fix(maestro-flow): accept equivalent connector artifacts (#2919) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Teach the affected Flow checkers to recognize the equivalent live-authoring and SDK persistence forms while preserving negative controls. Fixes #2917 ๐Ÿค– Generated with Claude Code Co-Authored-By: [Claude](mailto:noreply@anthropic.com) --- ...connector_checker_representation_parity.py | 248 ++++++++++++++++++ .../check_ceql_where_flow.py | 24 +- .../check_smoke_query_filter.py | 34 ++- .../check_trigger_lifecycle.py | 10 +- 4 files changed, 295 insertions(+), 21 deletions(-) create mode 100644 tests/tasks/uipath-maestro-flow/_shared/test_connector_checker_representation_parity.py diff --git a/tests/tasks/uipath-maestro-flow/_shared/test_connector_checker_representation_parity.py b/tests/tasks/uipath-maestro-flow/_shared/test_connector_checker_representation_parity.py new file mode 100644 index 0000000000..458902ac5e --- /dev/null +++ b/tests/tasks/uipath-maestro-flow/_shared/test_connector_checker_representation_parity.py @@ -0,0 +1,248 @@ +"""Regression tests for semantic parity across live and SDK Flow artifacts.""" + +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path + +import pytest + + +TASK_ROOT = Path(__file__).resolve().parent.parent + + +def _write_json(path: Path, value: object) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(value), encoding="utf-8") + + +def _run(relative: str, cwd: Path) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, str(TASK_ROOT / relative)], + cwd=cwd, + capture_output=True, + text=True, + check=False, + ) + + +def _filter_tree() -> dict: + return { + "groupOperator": 1, + "index": 0, + "filters": [ + {"id": "active", "operator": "Equals", "value": {"value": True}}, + {"id": "score", "operator": "GreaterThanOrEqual", "value": {"value": 8.5}}, + {"id": "viewCount", "operator": "GreaterThanOrEqual", "value": {"value": 1000}}, + {"id": "title", "operator": "Equals", "value": {"value": "FilterFixture-Matrix"}}, + {"id": "description", "operator": "Contains", "value": {"value": "sci-fi"}}, + {"id": "releaseDate", "operator": "LessThan", "value": {"value": "2025-01-01"}}, + {"id": "lastUpdated", "operator": "GreaterThanOrEqual", "value": {"value": "2024-01-01"}}, + {"id": "externalId", "operator": "Equals", "value": {"value": "7f4d8f1e-5c2b-4a6e-9d31-2b7c8e0f1a45"}}, + {"id": "description", "operator": "IsNull", "value": {"value": None}}, + ], + "groups": [], + } + + +def _query_detail(representation: str, *, start: int, ascending: bool) -> dict: + detail = { + "bodyParameters": {"_sortFieldName": "score"}, + "queryParameters": { + "queryExpression": ( + "active = true OR score >= 8.5 OR viewCount >= 1000 OR " + "title = 'FilterFixture-Matrix' OR description LIKE '%sci-fi%' OR " + "releaseDate < '2025-01-01' OR lastUpdated >= '2024-01-01' OR " + "externalId = '7f4d8f1e-5c2b-4a6e-9d31-2b7c8e0f1a45' OR description IS NULL" + ), + "start": start, + "limit": 2, + "isAscending": ascending, + }, + } + if representation == "sdk": + detail["filter"] = _filter_tree() + elif representation == "live": + detail["configuration"] = "=jsonString:" + json.dumps( + {"essentialConfiguration": {"savedFilterTrees": {"queryExpression": _filter_tree()}}} + ) + return detail + + +@pytest.mark.parametrize("representation", ["live", "sdk"]) +def test_smoke_query_accepts_both_filter_tree_locations( + tmp_path: Path, representation: str +) -> None: + details = [ + _query_detail(representation, start=0, ascending=True), + _query_detail(representation, start=2, ascending=True), + _query_detail(representation, start=0, ascending=False), + ] + _write_json( + tmp_path / "queries.flow", + { + "nodes": [ + { + "type": "uipath.connector.uipath-uipath-dataservice.query-entity-records", + "inputs": {"detail": detail}, + } + for detail in details + ] + }, + ) + + result = _run("connector_features/datafabric_connector/check_smoke_query_filter.py", tmp_path) + + assert result.returncode == 0, result.stderr or result.stdout + + +def test_smoke_query_rejects_runtime_text_without_contains_operator(tmp_path: Path) -> None: + details = [ + _query_detail("runtime-only", start=0, ascending=True), + _query_detail("runtime-only", start=2, ascending=True), + _query_detail("runtime-only", start=0, ascending=False), + ] + _write_json( + tmp_path / "queries.flow", + { + "nodes": [ + { + "type": "uipath.connector.uipath-uipath-dataservice.query-entity-records", + "inputs": {"detail": detail}, + } + for detail in details + ] + }, + ) + + result = _run("connector_features/datafabric_connector/check_smoke_query_filter.py", tmp_path) + + assert result.returncode != 0 + assert "multiline" in result.stderr + + +def _lifecycle_flow(representation: str, *, query_entity: str = "ContractRegistry") -> dict: + def entity_detail(name: str, extra: dict | None = None) -> dict: + value = ({"entityName": name} if representation == "sdk" + else {"pathParameters": {"entityName": name}}) + value.update(extra or {}) + return value + + return { + "nodes": [ + { + "id": "created", + "type": "uipath.connector.trigger.uipath-uipath-dataservice.record-created", + "inputs": {"detail": { + "objectName": "ContractRegistry", + "filterExpression": "dueDate < '2026-08-04'", + }}, + }, + { + "id": "query", + "type": "uipath.connector.uipath-uipath-dataservice.query-entity-records", + "inputs": {"detail": entity_detail(query_entity, {"queryParameters": { + "queryExpression": "dueDate < '2026-08-04'", "limit": 100, + }})}, + }, + { + "id": "updated", + "type": "uipath.connector.trigger.uipath-uipath-dataservice.record-updated", + "inputs": {"detail": {"objectName": "FileUploadVerify_20260618"}}, + }, + { + "id": "get", + "type": "uipath.connector.uipath-uipath-dataservice.get-entity-record-by-id", + "inputs": {"detail": entity_detail("FileUploadVerify_20260618", { + "queryParameters": {"recordId": "=js:$vars.updated.output.Id"}, + })}, + }, + { + "id": "delete", + "type": "uipath.connector.uipath-uipath-dataservice.delete-entity-record", + "inputs": {"detail": entity_detail("FileUploadVerify_20260618", { + "queryParameters": {"recordId": "=js:$vars.get.output.Id"}, + })}, + }, + ] + } + + +@pytest.mark.parametrize("representation", ["live", "sdk"]) +def test_trigger_lifecycle_accepts_both_entity_locations( + tmp_path: Path, representation: str +) -> None: + _write_json(tmp_path / "lifecycle.flow", _lifecycle_flow(representation)) + + result = _run("connector_features/datafabric_connector/check_trigger_lifecycle.py", tmp_path) + + assert result.returncode == 0, result.stderr or result.stdout + + +def test_trigger_lifecycle_rejects_wrong_query_entity(tmp_path: Path) -> None: + _write_json(tmp_path / "lifecycle.flow", _lifecycle_flow("sdk", query_entity="Other")) + + result = _run("connector_features/datafabric_connector/check_trigger_lifecycle.py", tmp_path) + + assert result.returncode != 0 + assert "Query Entity Records" in result.stderr + + +def _where_plan() -> dict: + return { + "filter": { + "groupOperator": 0, + "filters": [ + {"id": "displayName", "operator": "Equals", "value": {"value": "active"}} + ], + } + } + + +@pytest.mark.parametrize("representation", ["live", "sdk"]) +def test_ceql_where_accepts_curated_and_generic_groups_operations( + tmp_path: Path, representation: str +) -> None: + if representation == "live": + connector = { + "type": "uipath.connector.uipath-microsoft-azureactivedirectory.list-groups", + "inputs": {"detail": {}}, + } + else: + connector = { + "type": "uipath.connector.uipath-microsoft-azureactivedirectory.list-all-records", + "inputs": {"detail": {"objectName": "groups", "endpoint": "/groups"}}, + } + _write_json(tmp_path / "where_detail.json", _where_plan()) + _write_json( + tmp_path / "CeqlWhereTest.flow", + {"nodes": [connector, {"type": "core.logic.terminate"}], "edges": []}, + ) + + result = _run("connector_features/check_ceql_where_flow.py", tmp_path) + + assert result.returncode == 0, result.stderr or result.stdout + + +def test_ceql_where_rejects_generic_operation_for_another_object(tmp_path: Path) -> None: + _write_json(tmp_path / "where_detail.json", _where_plan()) + _write_json( + tmp_path / "CeqlWhereTest.flow", + { + "nodes": [ + { + "type": "uipath.connector.uipath-microsoft-azureactivedirectory.list-all-records", + "inputs": {"detail": {"objectName": "users", "endpoint": "/users"}}, + }, + {"type": "core.logic.terminate"}, + ], + "edges": [], + }, + ) + + result = _run("connector_features/check_ceql_where_flow.py", tmp_path) + + assert result.returncode != 0 + assert "list-groups" in result.stderr diff --git a/tests/tasks/uipath-maestro-flow/connector_features/check_ceql_where_flow.py b/tests/tasks/uipath-maestro-flow/connector_features/check_ceql_where_flow.py index 3c38307f80..1470d2ab0b 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/check_ceql_where_flow.py +++ b/tests/tasks/uipath-maestro-flow/connector_features/check_ceql_where_flow.py @@ -137,6 +137,23 @@ def _find_flow() -> str: return find_flow_file(flow_glob=os.path.basename(FLOW_GLOB)) +def _is_groups_operation(node: dict) -> bool: + node_type = str(node.get("type") or "").lower() + if CONNECTOR_KEY not in node_type: + return False + if "group" in node_type: + return True + + detail = node.get("inputs", {}).get("detail", {}) or {} + object_name = detail.get("objectName") or (detail.get("pathParameters") or {}).get("objectName") + endpoint = str(detail.get("endpoint") or "").rstrip("/").lower() + return ( + node_type.endswith(".list-all-records") + and str(object_name or "").lower() == "groups" + and (not endpoint or endpoint.endswith("/groups")) + ) + + def _check_flow_structure() -> None: flow_path = _find_flow() raw = open(flow_path).read() @@ -155,12 +172,7 @@ def _check_flow_structure() -> None: "the registered key with `uip maestro flow registry search`." ) - found_groups = False - for node in flow.get("nodes", []): - node_type = node.get("type", "") - if CONNECTOR_KEY in node_type and "group" in node_type.lower(): - found_groups = True - break + found_groups = any(_is_groups_operation(node) for node in flow.get("nodes", [])) if not found_groups: sys.exit( f"FAIL: No connector node of type " diff --git a/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/check_smoke_query_filter.py b/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/check_smoke_query_filter.py index f9a7308582..16fc78d6c6 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/check_smoke_query_filter.py +++ b/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/check_smoke_query_filter.py @@ -23,17 +23,6 @@ def structured_filter_leaves(detail): - config = detail.get("configuration") - if not isinstance(config, str): - return [] - raw = config.removeprefix("=jsonString:") - try: - parsed = json.loads(raw) - except (TypeError, ValueError): - return [] - trees = (parsed.get("essentialConfiguration", {}) - .get("savedFilterTrees", {})) - tree = trees.get("queryExpression", {}) if isinstance(trees, dict) else {} leaves = [] def walk(group): @@ -43,7 +32,28 @@ def walk(group): for child in group.get("groups") or []: walk(child) - walk(tree) + # The live authoring loop stores the design-time tree in the encoded + # configuration. The SDK emit-only loop preserves the same semantic tree + # directly at detail.filter until product tooling normalizes the artifact. + trees = [detail.get("filter")] + direct_saved = detail.get("savedFilterTrees") or {} + if isinstance(direct_saved, dict): + trees.append(direct_saved.get("queryExpression")) + + config = detail.get("configuration") + if isinstance(config, str): + raw = config.removeprefix("=jsonString:") + try: + parsed = json.loads(raw) + except (TypeError, ValueError): + parsed = {} + saved = (parsed.get("essentialConfiguration", {}) + .get("savedFilterTrees", {})) + if isinstance(saved, dict): + trees.append(saved.get("queryExpression")) + + for tree in trees: + walk(tree) return leaves diff --git a/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/check_trigger_lifecycle.py b/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/check_trigger_lifecycle.py index 709310ffe2..a12165e56c 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/check_trigger_lifecycle.py +++ b/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/check_trigger_lifecycle.py @@ -21,6 +21,10 @@ def text(value): return str(value or "").lower() +def entity_name(d): + return d.get("entityName") or (d.get("pathParameters") or {}).get("entityName") + + def has_due_filter(d): values = [d.get("filterExpression"), d.get("filter")] config = d.get("configuration") @@ -62,7 +66,7 @@ def main(): contract_queries = [] for _, n in queries: - if (detail(n).get("pathParameters") or {}).get("entityName") == "ContractRegistry": + if entity_name(detail(n)) == "ContractRegistry": contract_queries.append(n) if not contract_queries: print("FAIL: Query Entity Records after ContractRegistry trigger missing", file=sys.stderr) @@ -80,8 +84,8 @@ def main(): print("FAIL: Record Updated trigger for FileUploadVerify_20260618 missing", file=sys.stderr) return 1 trigger_ids = {n.get("id", "") for n in file_updated} - file_gets = [n for _, n in gets if (detail(n).get("pathParameters") or {}).get("entityName") == "FileUploadVerify_20260618"] - file_deletes = [n for _, n in deletes if (detail(n).get("pathParameters") or {}).get("entityName") == "FileUploadVerify_20260618"] + file_gets = [n for _, n in gets if entity_name(detail(n)) == "FileUploadVerify_20260618"] + file_deletes = [n for _, n in deletes if entity_name(detail(n)) == "FileUploadVerify_20260618"] if not file_gets or not file_deletes: print("FAIL: FileUploadVerify updated flow must contain Get and Delete activities", file=sys.stderr) return 1 From 2b27394386061f6855b91e1bdca6daca96bf89a7 Mon Sep 17 00:00:00 2001 From: Tao Ma <51425734+tmatup@users.noreply.github.com> Date: Sun, 30 Aug 2026 21:55:26 -0700 Subject: [PATCH 30/52] fix(maestro-flow): clarify task resource scope (#2920) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit State the tenant-wide connection scope and required Slack connector identity that the existing criteria grade. Fixes #2918 ๐Ÿค– Generated with Claude Code Co-Authored-By: [Claude](mailto:noreply@anthropic.com) --- .../_shared/test_same_ground_corpus.py | 19 +++++++++++++++++++ .../reconfigure_different_connection.yaml | 7 ++++--- .../connector_features/multiselect.yaml | 2 +- 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/tests/tasks/uipath-maestro-flow/_shared/test_same_ground_corpus.py b/tests/tasks/uipath-maestro-flow/_shared/test_same_ground_corpus.py index 5be99f5fb9..cf2f786ec2 100644 --- a/tests/tasks/uipath-maestro-flow/_shared/test_same_ground_corpus.py +++ b/tests/tasks/uipath-maestro-flow/_shared/test_same_ground_corpus.py @@ -68,6 +68,10 @@ def _tagged_tasks() -> list[tuple[str, Path, str]]: return tasks +def _task_text(relative: str) -> str: + return (FLOW_TASKS / relative).read_text() + + def _referenced_python_files(task_path: Path, criterion: str) -> list[Path]: paths = [] for token in re.findall( @@ -191,3 +195,18 @@ def test_external_graders_use_package_qualified_shared_imports() -> None: if module in shared_modules: offenders.add(relative.as_posix()) assert offenders == set() + + +def test_reconfigure_prompt_requires_a_tenant_wide_connection_inventory() -> None: + prompt = _prompt_text( + _task_text("bindings/reconfigure_different_connection.yaml") + ).lower() + + assert "tenant-wide" in prompt + assert "every folder" in prompt + + +def test_multiselect_prompt_names_slack_as_the_required_connector() -> None: + prompt = _prompt_text(_task_text("connector_features/multiselect.yaml")).lower() + + assert "slack group direct message" in prompt diff --git a/tests/tasks/uipath-maestro-flow/bindings/reconfigure_different_connection.yaml b/tests/tasks/uipath-maestro-flow/bindings/reconfigure_different_connection.yaml index 708b00d987..23986756b8 100644 --- a/tests/tasks/uipath-maestro-flow/bindings/reconfigure_different_connection.yaml +++ b/tests/tasks/uipath-maestro-flow/bindings/reconfigure_different_connection.yaml @@ -19,9 +19,10 @@ run_limits: initial_prompt: | PRECONDITION CHECK โ€” do this FIRST, before any flow work: - Inspect the available Integration Service connections once. Group the enabled - connections by connector key and find a connector with two or more enabled - connections. That connector supplies A and B. + Inspect the tenant-wide Integration Service connection inventory once, + including connections in every folder. Group the enabled connections by + connector key and find a connector with two or more enabled connections. + That connector supplies A and B. If no connector has two or more enabled connections, the precondition is not met for this tenant. Stop IMMEDIATELY: write diff --git a/tests/tasks/uipath-maestro-flow/connector_features/multiselect.yaml b/tests/tasks/uipath-maestro-flow/connector_features/multiselect.yaml index cf89896445..b16551315c 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/multiselect.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_features/multiselect.yaml @@ -16,7 +16,7 @@ run_limits: turn_timeout: 1200 initial_prompt: | - Create a flow file to create a group direct message between + Create a flow file to create a Slack group direct message between Coder Evals, IS-sandboxes@uipath.com and E2E-testing. Use the connection present in Shared/uipath-maestro-flow. From 9e6e7d863d5bee5a152f2d01bdd49f3a2d13b1e2 Mon Sep 17 00:00:00 2001 From: tmatup <51425734+tmatup@users.noreply.github.com> Date: Mon, 31 Aug 2026 22:22:46 +0000 Subject: [PATCH 31/52] test(skills): stabilize npm publish dry runs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ๐Ÿค– Generated with Claude Code Co-Authored-By: [Claude](mailto:noreply@anthropic.com) --- tests/scripts/compose-skill-flavor.test.mjs | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/tests/scripts/compose-skill-flavor.test.mjs b/tests/scripts/compose-skill-flavor.test.mjs index b5ed73be0c..dbfedf7fa2 100644 --- a/tests/scripts/compose-skill-flavor.test.mjs +++ b/tests/scripts/compose-skill-flavor.test.mjs @@ -1147,7 +1147,7 @@ test("pack builds complete, marker-free default and custom npm packages", (t) => ); const customPublishDryRun = invokeNpm( repo, - ["publish", selectedStudioWeb, "--dry-run"], + ["publish", selectedStudioWeb, "--dry-run", "--tag", "preview"], { npm_config_userconfig: forcedPublicUserConfig }, ); assert.equal( @@ -1448,7 +1448,16 @@ test("root npm publish dry-run keeps the old default-only behavior and restores addPackageManifest(repo); const canonicalBefore = treeFileBytes(join(repo, "skills")); - const result = runNpm(repo, "publish", "--dry-run", "--json", "--access", "public"); + const result = runNpm( + repo, + "publish", + "--dry-run", + "--json", + "--access", + "public", + "--tag", + "test", + ); assert.equal(result.status, 0, result.stderr || result.stdout); assert.match(`${result.stdout}\n${result.stderr}`, /@uipath\/skills|"name"\s*:\s*"@uipath\/skills"/); assert.doesNotMatch(`${result.stdout}\n${result.stderr}`, /skills-studioweb/); From 712cadea27c4c32e9bc296e8699736eb57e66ebd Mon Sep 17 00:00:00 2001 From: Tao Ma <51425734+tmatup@users.noreply.github.com> Date: Tue, 1 Sep 2026 10:02:39 -0700 Subject: [PATCH 32/52] fix(maestro-flow): accept supported Slack lookup routes (#2945) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Grade the resolved Slack channel outcome across both manual resource collections and the SDK lookup resolver, with a regression test for each route. ๐Ÿค– Generated with Claude Code Co-Authored-By: [Claude](mailto:noreply@anthropic.com) --- .../_shared/test_same_ground_corpus.py | 20 ++++++++++++++ .../paginated_reference_lookup.yaml | 26 +++++++++---------- 2 files changed, 32 insertions(+), 14 deletions(-) diff --git a/tests/tasks/uipath-maestro-flow/_shared/test_same_ground_corpus.py b/tests/tasks/uipath-maestro-flow/_shared/test_same_ground_corpus.py index cf2f786ec2..8a5b5902ee 100644 --- a/tests/tasks/uipath-maestro-flow/_shared/test_same_ground_corpus.py +++ b/tests/tasks/uipath-maestro-flow/_shared/test_same_ground_corpus.py @@ -210,3 +210,23 @@ def test_multiselect_prompt_names_slack_as_the_required_connector() -> None: prompt = _prompt_text(_task_text("connector_features/multiselect.yaml")).lower() assert "slack group direct message" in prompt + + +def test_paginated_lookup_accepts_both_supported_resolution_routes() -> None: + text = _task_text("connector_features/paginated_reference_lookup.yaml") + patterns = re.findall(r"(?m)^\s+command_pattern:\s*'([^']+)'", text) + resolve_pattern, pagination_pattern = map(re.compile, patterns[:2]) + + manual_commands = ( + "uip is resources run list uipath-salesforce-slack conversations " + "--connection-id id --query nextPage=token" + ) + sdk_command = ( + "uip maestro flow registry prepare uipath-salesforce-slack " + "send-message-to-channel --resolve channel:name=simple" + ) + + assert resolve_pattern.search(manual_commands) + assert pagination_pattern.search(manual_commands) + assert resolve_pattern.search(sdk_command) + assert pagination_pattern.search(sdk_command) diff --git a/tests/tasks/uipath-maestro-flow/connector_features/paginated_reference_lookup.yaml b/tests/tasks/uipath-maestro-flow/connector_features/paginated_reference_lookup.yaml index bcdcbdb0d5..711df8b599 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/paginated_reference_lookup.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_features/paginated_reference_lookup.yaml @@ -5,10 +5,10 @@ task_id: skill-flow-paginated-reference-lookup description: > Build a Flow with a Slack `Send Message to Channel` node targeting the `simple` channel. The channel lives on a later page of the `is-sandboxes` - curated_channels list, so resolving the Slack channel id forces the agent - to paginate `uip is resources run list`. Triggered by the `resources.md` - read-before-call rule added in #1059 (ENGCE-58198) โ€” old behavior was to - abandon pagination after page 1 and bail to AskUserQuestion. + Slack channel resources, so resolving the Slack channel id requires either + explicit pagination or the SDK's lookup resolver. Triggered by the + `resources.md` read-before-call rule added in #1059 (ENGCE-58198) โ€” old + behavior was to abandon pagination after page 1 and bail to AskUserQuestion. tags: - uipath-maestro-flow - integration @@ -38,23 +38,21 @@ initial_prompt: | planning and implementation. Build the complete flow end-to-end in a single pass. success_criteria: +# TWO ROUTES ARE CORRECT, and these criteria accept either. Manual discovery +# may expose the channel collection as `conversations` or `curated_channels`; +# the final artifact criterion below is the authority on the resolved id. - type: command_executed - description: "Agent invoked `uip is resources run list ... curated_channels` more than once (pagination loop ran)" + description: "Agent resolved the channel through the library โ€” either a paged Slack channel-resource loop or `registry prepare --resolve`" tool_name: "Bash" - # Commands are recorded as the raw `bash -lc '...'`/`bash -lc "..."` wrapper. When the - # command carries an --output-filter with an embedded single-quoted JMESPath - # (items[?name=='simple']), the wrapper switches to double quotes and backslash-escapes the - # inner quotes (\"uipath-salesforce-slack\"). Allow an optional escaping backslash before each - # quote so both quoting styles match; otherwise paginated calls are silently uncounted. - command_pattern: 'uip\s+is\s+resources\s+run\s+list\s+\\?"?uipath-salesforce-slack\\?"?\s+\\?"?curated_channels' - min_count: 2 + command_pattern: '(uip\s+is\s+resources\s+run\s+list\s+\\?"?uipath-salesforce-slack\\?"?\s+\\?"?(curated_channels|conversations)|registry\s+prepare\s+\\?"?uipath-salesforce-slack)' + min_count: 1 weight: 3.0 pass_threshold: 1.0 - type: command_executed - description: "Agent used `nextPage=` to advance pagination at least once" + description: "Agent went past page 1 โ€” `nextPage=` by hand, or `--resolve channel:` which pages inside `prepare`" tool_name: "Bash" - command_pattern: 'nextPage=' + command_pattern: '(nextPage=|--resolve\s+\\?"?channel:)' min_count: 1 weight: 3.0 pass_threshold: 1.0 From 00423c2129fb653614694debc4aec337629331b0 Mon Sep 17 00:00:00 2001 From: tmatup <51425734+tmatup@users.noreply.github.com> Date: Tue, 1 Sep 2026 18:39:11 +0000 Subject: [PATCH 33/52] fix(maestro-flow): restore the routing tasks' loop-neutral scope clause MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The same-ground sweep (#2751) removed the whole "Important:" block from `ixp/routing.yaml` and `ixp/routing_negative.yaml`. Most of that block was loop coaching and belonged out (`--output json`, "validate at most once"), but one sentence was the task's own scope โ€” "a skeleton whose nodes cover the steps is enough; leave a node unconfigured rather than leaving a step out" โ€” and nothing replaced it. Every other task in the sweep kept a loop-neutral "Do NOT ask for approvalโ€ฆ build in a single pass" line; the two routing YAMLs lost theirs and got nothing back. Without it the routing rows turned from a routing smoke into "build a working flow against live tenant connections", on an unchanged 25-turn budget. In adhoc-2026-09-01_06-35-33 (gpt-5.6-luna, both arms): - live-v1: `sf-update` and `stripe-http` ended their only turn asking the user a question ("Which Salesforce stageโ€ฆ?", "Which should I use?") โ€” the skill's dropdown rule #5 in a headless run โ€” and produced no artifact; 5 of the 8 negative rows that did pass ran into the 25-turn cap first. - SDK-v2: `sf-update` spent its budget chasing the tenant-specific Salesforce `update-record` schema (a real SDK `prepare` defect, fixed in flow-builder-sdk) and `teams-decision` hit the cap the same way. - The official baseline (`2026-09-01_04-16-00`, gpt-5.6-terra, skills main with the block) passed all 8 rows in 8-18 tool calls, leaving nodes unconfigured exactly as told. This restores the scope sentence in loop-neutral words: skeleton is enough, leave unconfigured rather than omit or ask, do not ask for approval or missing details, do not iterate to make it runnable. No CLI or SDK coaching, no debug/deploy instruction (the corpus test forbids it, and the budget already precludes debug). The campaign checker's stricter requirement that a `.flow` must exist (`flow_contains.py --absent-regex`, replacing a `! grep` that passed vacuously with no artifact) is kept โ€” that was a checker bug on main, not something to restore. Limits are unchanged. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01G1JCRWy18Fgw4q8m5Lx6V6 --- tests/tasks/uipath-maestro-flow/ixp/routing.yaml | 7 +++++++ tests/tasks/uipath-maestro-flow/ixp/routing_negative.yaml | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/tests/tasks/uipath-maestro-flow/ixp/routing.yaml b/tests/tasks/uipath-maestro-flow/ixp/routing.yaml index 46b7bd6e91..7a4f27e407 100644 --- a/tests/tasks/uipath-maestro-flow/ixp/routing.yaml +++ b/tests/tasks/uipath-maestro-flow/ixp/routing.yaml @@ -54,6 +54,13 @@ dataset: initial_prompt: | ${row.prompt} + Scope: this is a routing check, not a build-to-run task. A skeleton whose + nodes cover the steps of the request is enough โ€” leave a node unconfigured + rather than leaving a step out or stopping to ask. Do NOT ask for approval, + confirmation, or missing details; pick a sensible default and state it in + your final report. Do not iterate to make the flow runnable โ€” a broken + skeleton is acceptable. + success_criteria: # What this checks: did the agent search the registry for IxP nodes? # Passes (case-insensitive) when the agent ran either: diff --git a/tests/tasks/uipath-maestro-flow/ixp/routing_negative.yaml b/tests/tasks/uipath-maestro-flow/ixp/routing_negative.yaml index 5c967d0230..1cdeb58edb 100644 --- a/tests/tasks/uipath-maestro-flow/ixp/routing_negative.yaml +++ b/tests/tasks/uipath-maestro-flow/ixp/routing_negative.yaml @@ -59,6 +59,13 @@ dataset: initial_prompt: | ${row.prompt} + Scope: this is a routing check, not a build-to-run task. A skeleton whose + nodes cover the steps of the request is enough โ€” leave a node unconfigured + rather than leaving a step out or stopping to ask. Do NOT ask for approval, + confirmation, or missing details; pick a sensible default and state it in + your final report. Do not iterate to make the flow runnable โ€” a broken + skeleton is acceptable. + success_criteria: # Inverted twin of the positive routing.yaml artifact check. - type: run_command From 94cf2589bc404f66894a0a6587e2ee7f453ed3de Mon Sep 17 00:00:00 2001 From: tmatup <51425734+tmatup@users.noreply.github.com> Date: Tue, 1 Sep 2026 19:03:42 +0000 Subject: [PATCH 34/52] fix(eval-image): read the connector library location from the CLI, not from a snapshot layout it stopped writing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Dockerfile's registry step hard-coded the layout `uip maestro registry pull` used to produce: a snapshot under $UIP_MAESTRO_REGISTRY_HOME plus a `current.json` pointer. From @uipath/maestro-tool 1.202.0-dev.8461 (published 2026-09-01 18:13Z, carrying cli#3823) the command delegates to the Flow SDK, which caches the library under ~/.uipath/cache/flow-sdk/library// and writes no current.json โ€” so every fresh `docker build` of this image has failed since that publish: jq: error: Could not open file /opt/preview-flow-sdk-assets/registry/current.json The nightly and ADO builds have not noticed yet only because they do not pass --no-cache: the `npm install @uipath/maestro-tool@dev` layer is keyed on the command string, not on what the floating tag resolves to, so they keep replaying the 08-27 layer. Any cache eviction reproduces the failure. Fix at the layer where the coupling lives: the image now asks the CLI where the library is (`uip maestro registry path --library-json | --sha`), keeps the old pointer as the source when a pinned older CLI still writes it, and then records the image's OWN provenance pointer at $UIP_MAESTRO_REGISTRY_HOME/ current.json (libraryHash + snapshotPath). The per-task stage-preview-sdk-workspace.sh and the run scripts read that one contract unchanged, whichever layout the installed CLI produced. Verified by running the step's shell against the new-layout host CLI: pointer written, asset links resolve, stage script reports the library hash. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01G1JCRWy18Fgw4q8m5Lx6V6 --- tests/docker/Dockerfile | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/tests/docker/Dockerfile b/tests/docker/Dockerfile index deeb4df4c2..546f4976a1 100644 --- a/tests/docker/Dockerfile +++ b/tests/docker/Dockerfile @@ -153,15 +153,42 @@ ENV DOTNET_ROOT=/usr/share/dotnet \ # Install the released connector catalog through the product CLI. The stable # links keep the authoring paths independent of the catalog's content hash. +# +# Where the pull lands is the CLI's business, and it moved: up to +# maestro-tool 1.202.0-dev.8425 `uip maestro registry pull` unpacked a snapshot +# under $UIP_MAESTRO_REGISTRY_HOME and wrote `current.json` there; from +# dev.8461 (cli#3823) it delegates to the Flow SDK, which caches the library +# under ~/.uipath/cache/flow-sdk/library// and writes no current.json. +# This step reads BOTH: the old pointer when the CLI still writes it, otherwise +# `uip maestro registry path` (the CLI's own answer to "where is it") โ€” and then +# records the image's provenance pointer itself, so the per-task +# stage-preview-sdk-workspace.sh and the run scripts keep one contract +# ($UIP_MAESTRO_REGISTRY_HOME/current.json: libraryHash + snapshotPath) no matter +# which layout the installed CLI produced. Hard-coding either layout is what +# broke the 2026-09-01 image build the moment the @dev tool moved. RUN cd "$PREVIEW_FLOW_SDK_ROOT" && \ uip maestro registry pull --force --output json && \ - registry_snapshot="$(jq -er '.snapshotPath' "$UIP_MAESTRO_REGISTRY_HOME/current.json")" && \ + if [ -f "$UIP_MAESTRO_REGISTRY_HOME/current.json" ]; then \ + registry_snapshot="$(jq -er '.snapshotPath' "$UIP_MAESTRO_REGISTRY_HOME/current.json")"; \ + registry_connectors="$registry_snapshot/connectors"; \ + else \ + library_json="$(readlink -f "$(uip maestro registry path --library-json)")" && \ + registry_snapshot="$(dirname "$library_json")" && \ + library_hash="$(uip maestro registry path --sha)" && \ + test -n "$library_hash" && \ + mkdir -p "$UIP_MAESTRO_REGISTRY_HOME" && \ + jq -n --arg libraryHash "$library_hash" --arg snapshotPath "$registry_snapshot" \ + '{libraryHash: $libraryHash, snapshotPath: $snapshotPath}' \ + > "$UIP_MAESTRO_REGISTRY_HOME/current.json" && \ + registry_connectors="$PREVIEW_FLOW_SDK_ROOT/connectors" && \ + mkdir -p "$registry_connectors"; \ + fi && \ test -f "$registry_snapshot/library-json/index.json" && \ test -f "$registry_snapshot/library-md/index.json" && \ mkdir -p "$PREVIEW_FLOW_SDK_ASSETS_ROOT" && \ ln -s "$registry_snapshot/library-json" "$PREVIEW_FLOW_SDK_ASSETS_ROOT/library-json" && \ ln -s "$registry_snapshot/library-md" "$PREVIEW_FLOW_SDK_ASSETS_ROOT/library-md" && \ - ln -s "$registry_snapshot/connectors" "$PREVIEW_FLOW_SDK_ASSETS_ROOT/connectors" + ln -s "$registry_connectors" "$PREVIEW_FLOW_SDK_ASSETS_ROOT/connectors" ENV FLOW_SDK_LIBRARY_JSON=/opt/preview-flow-sdk-assets/library-json \ FLOW_SDK_LIBRARY_MD=/opt/preview-flow-sdk-assets/library-md \ From e1ce801be80e1239da0540e7ccf3e51b16c692e0 Mon Sep 17 00:00:00 2001 From: tmatup <51425734+tmatup@users.noreply.github.com> Date: Tue, 1 Sep 2026 19:15:12 +0000 Subject: [PATCH 35/52] =?UTF-8?q?fix(maestro-flow):=20name=20the=20routing?= =?UTF-8?q?=20tasks'=20deliverable=20=E2=80=94=20the=20.flow=20artifact?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The restored scope clause said "a skeleton is enough" without saying what the skeleton IS. The two loops read that differently: for the live-v1 loop the skeleton is the `.flow` the CLI writes, so 8/8 rows passed; for the SDK loop the skeleton is `.flow.ts`, and 7/8 rows stopped there โ€” "No runnable validation was performed", "left unset per scope" โ€” so no `.flow` existed for the artifact check (local rerun 2026-09-01 19:12, preview arm 1/8). Name the outcome the checker grades, which both loops produce: the `.flow` file. Still loop-neutral โ€” no command, no mechanism โ€” and the corpus tests stay green. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01G1JCRWy18Fgw4q8m5Lx6V6 --- tests/tasks/uipath-maestro-flow/ixp/routing.yaml | 12 +++++++----- .../uipath-maestro-flow/ixp/routing_negative.yaml | 12 +++++++----- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/tests/tasks/uipath-maestro-flow/ixp/routing.yaml b/tests/tasks/uipath-maestro-flow/ixp/routing.yaml index 7a4f27e407..033a54d279 100644 --- a/tests/tasks/uipath-maestro-flow/ixp/routing.yaml +++ b/tests/tasks/uipath-maestro-flow/ixp/routing.yaml @@ -54,12 +54,14 @@ dataset: initial_prompt: | ${row.prompt} - Scope: this is a routing check, not a build-to-run task. A skeleton whose - nodes cover the steps of the request is enough โ€” leave a node unconfigured - rather than leaving a step out or stopping to ask. Do NOT ask for approval, + Scope: this is a routing check, not a build-to-run task. The deliverable is + the flow artifact itself โ€” the `.flow` file. A skeleton whose nodes cover the + steps of the request is enough: leave a node unconfigured rather than + leaving a step out or stopping to ask. Do NOT ask for approval, confirmation, or missing details; pick a sensible default and state it in - your final report. Do not iterate to make the flow runnable โ€” a broken - skeleton is acceptable. + your final report. Do not iterate on validation errors to make the flow + runnable โ€” a broken skeleton is acceptable, as long as the `.flow` file has + been produced. success_criteria: # What this checks: did the agent search the registry for IxP nodes? diff --git a/tests/tasks/uipath-maestro-flow/ixp/routing_negative.yaml b/tests/tasks/uipath-maestro-flow/ixp/routing_negative.yaml index 1cdeb58edb..7286b450a5 100644 --- a/tests/tasks/uipath-maestro-flow/ixp/routing_negative.yaml +++ b/tests/tasks/uipath-maestro-flow/ixp/routing_negative.yaml @@ -59,12 +59,14 @@ dataset: initial_prompt: | ${row.prompt} - Scope: this is a routing check, not a build-to-run task. A skeleton whose - nodes cover the steps of the request is enough โ€” leave a node unconfigured - rather than leaving a step out or stopping to ask. Do NOT ask for approval, + Scope: this is a routing check, not a build-to-run task. The deliverable is + the flow artifact itself โ€” the `.flow` file. A skeleton whose nodes cover the + steps of the request is enough: leave a node unconfigured rather than + leaving a step out or stopping to ask. Do NOT ask for approval, confirmation, or missing details; pick a sensible default and state it in - your final report. Do not iterate to make the flow runnable โ€” a broken - skeleton is acceptable. + your final report. Do not iterate on validation errors to make the flow + runnable โ€” a broken skeleton is acceptable, as long as the `.flow` file has + been produced. success_criteria: # Inverted twin of the positive routing.yaml artifact check. From 47ad96bb09738682129bb48ef3cf4e44063318ea Mon Sep 17 00:00:00 2001 From: Tao Ma <51425734+tmatup@users.noreply.github.com> Date: Tue, 1 Sep 2026 14:12:58 -0700 Subject: [PATCH 36/52] fix(flow-check): classify a Completed debug with unreadable outputs as INFRA, keep the CLI stderr (#2967) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `run_debug` graded whatever `uip maestro flow debug` returned on exit 0. When the CLI's post-completion fetch of the output variables failed, the CLI (before UiPath/cli#3929) printed `Result: Success` with no `variables` key at all and the failure reason only on stderr โ€” which `run_debug` discarded on a zero exit. `assert_outputs_contain` then reported "Outputs missing โ€ฆ; present=[]" and the task was graded as a flow defect. Observed on coder-eval `skill-flow-move-node` v2 on 2026-08-31 and 2026-09-01: every element Completed, `Outputs: []`, score 0.375. The same artifact debugged 7/7 with outputs from a host and from inside the eval image. - A Completed payload whose `Data` has no `variables` key, or carries the new `variablesError`, is now classified as "outputs UNREADABLE" โ€” the debug is retried once (its own 2-attempt budget, independent of the 3 attempts for 5xx/RetryLater), and if it persists the check fails with an INFRA message, never "Outputs missing". The condition is key-absent, not empty: a flow that declares no outputs still returns `variables` with an empty `globals` map. - The CLI's stderr is kept for every attempt and printed (tail) in the FLOW_DEBUG_RAW_CAPTURE dump, so the next occurrence self-explains. Tests: 7 new (retry-then-grade, variablesError, persistent โ†’ INFRA with capture, present-but-empty not retried, budgets independent, classifier table, stderr in capture). Suite: 340 passed. Rebase note (2026-09-02, onto main 88d10d747): re-applied over main's #2960 (`run_debug` deadline/budget loop). The unreadable-outputs retry now extends `max_attempts` by one instead of assuming a third loop iteration, and still honours the deadline. Main's new budget guard (`test_criterion_budgets.py`) priced only `$TASK_DIR/...` commands; the campaign addresses checkers as `$SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/...`, so the guard now resolves both forms (686 pass, every campaign criterion clears its budget). Suite after rebase: 723 passed. ๐Ÿค– Generated with Claude Code Co-Authored-By: [Claude](mailto:noreply@anthropic.com) Claude-Session: https://claude.ai/code/session_0193r8XwXyqZqZwbBXXuwiHA Claude-Session: https://claude.ai/code/session_01SMAwQ1iQQSUUPi3oatMqgG Co-authored-by: Claude Fable 5.1 --- .../uipath-maestro-flow/_shared/flow_check.py | 98 +++++++++++++- .../_shared/test_criterion_budgets.py | 28 +++- .../_shared/test_flow_check.py | 125 ++++++++++++++++++ 3 files changed, 239 insertions(+), 12 deletions(-) diff --git a/tests/tasks/uipath-maestro-flow/_shared/flow_check.py b/tests/tasks/uipath-maestro-flow/_shared/flow_check.py index e190601bc2..3202bd67fd 100644 --- a/tests/tasks/uipath-maestro-flow/_shared/flow_check.py +++ b/tests/tasks/uipath-maestro-flow/_shared/flow_check.py @@ -54,6 +54,7 @@ # the exact payload (finalStatus, elementExecutions, globals, incidents) that the # checker saw โ€” which is otherwise ephemeral and unrecoverable post-run. _LAST_DEBUG_RAW: str | None = None +_LAST_DEBUG_STDERR: str | None = None # Variable ids the most recent :func:`run_debug` bound as INPUTS (via ``--inputs`` # / ``--attachment``). Stashed because the runtime returns every global โ€” `in` and @@ -110,6 +111,20 @@ _DEFAULT_RETRIES = 2 _DEFAULT_BACKOFF_SECONDS = 5.0 +# A run that reports ``finalStatus: Completed`` but whose ``Data`` carries no +# ``variables`` key at all did NOT produce "no outputs": the CLI's +# post-completion GET of ``/debug-instances/{id}/variables`` failed and (before +# UiPath/cli fixed it) the failure was reduced to a stderr warning. A flow that +# genuinely declares no outputs still returns ``variables`` (with an empty +# ``globals`` map), so "key absent" is exact, not heuristic. Newer CLIs keep the +# key absent and add ``variablesError`` explaining why. Both shapes mean +# "outputs UNREADABLE" โ€” an infrastructure failure to retry, never a flow +# result to grade. Observed live on coder-eval ``skill-flow-move-node`` (v2, +# 2026-08-31 and 2026-09-01): every element Completed, ``Outputs: []``, task +# graded as a flow defect. +_VARIABLES_UNREADABLE_ATTEMPTS = 2 +_STDERR_CAPTURE_TAIL_CHARS = 4000 + # A `uip maestro flow debug` run can die on a transient server-side error โ€” a # gateway timeout / 5xx while polling the debug instance, which the CLI reports # as `Result:Failure`, `ErrorCode:server_error`, `Retry:RetryLater` (the CLI's @@ -155,6 +170,36 @@ def _is_poll_timeout(result: subprocess.CompletedProcess) -> bool: ) +def _variables_unreadable(data: dict | None) -> str | None: + """Return a reason when a *completed* debug payload's outputs could not be + read (see :data:`_VARIABLES_UNREADABLE_ATTEMPTS`), else ``None``. + + Exact by construction: only the ``variables`` key being absent from ``Data`` + (or an explicit ``variablesError``) counts. A present-but-empty + ``variables`` is a real result and is returned to the caller as-is.""" + payload = _get_ci(data or {}, "Data") or {} + if not isinstance(payload, dict): + return None + status = _get_ci(payload, "finalStatus", "FinalStatus") + if status != "Completed": + return None + err = _get_ci(payload, "variablesError", "VariablesError") + if isinstance(err, dict): + attempts = _get_ci(err, "attempts", "Attempts") + message = _get_ci(err, "message", "Message") or "unknown error" + return ( + f"the CLI could not read the output variables (variablesError after " + f"{attempts} attempt(s): {message})" + ) + has_key = any(str(k).lower() == "variables" for k in payload) + if not has_key: + return ( + "the CLI returned no `variables` key at all โ€” the post-completion " + "fetch of /debug-instances/{id}/variables failed and was dropped" + ) + return None + + def _as_text(raw: bytes | str | None) -> str: """Decode captured child output. ``subprocess.TimeoutExpired`` carries it as bytes even under ``text=True``, unlike ``CompletedProcess``.""" @@ -232,7 +277,8 @@ def run_debug( cmd.extend(["--attachment", f"{var_id}={local_path}"]) env = dict(os.environ) env.setdefault("UIP_LOG_LEVEL", _DEBUG_LOG_LEVEL) - global _LAST_DEBUG_RAW, _LAST_DEBUG_INPUT_IDS, _LAST_DEBUG_PROJECT_DIR + global _LAST_DEBUG_RAW, _LAST_DEBUG_STDERR, _LAST_DEBUG_INPUT_IDS + global _LAST_DEBUG_PROJECT_DIR # Record what we bound as input, and where the project lives, so output # assertions can discount echoes of our own inputs. _LAST_DEBUG_INPUT_IDS = {str(k) for k in (inputs or {})} | { @@ -276,7 +322,15 @@ def run_debug( # Mirrors debug_budget: pricing an attempt the loop never makes left `r` unbound. attempts = max(1, retries) - for attempt in range(attempts): + unreadable: str | None = None + unreadable_attempts = 0 + # `max_attempts` starts at the transient allowance and is extended by one + # for an unreadable-outputs retry (see _VARIABLES_UNREADABLE_ATTEMPTS): that + # retry has its own allowance, never the 5xx/RetryLater one. The deadline + # below still bounds every attempt, so the criterion budget holds. + max_attempts = attempts + attempt = 0 + while attempt < max_attempts: attempt_cap = min(timeout, int(deadline - time.monotonic())) cli_timeout = max( _MIN_CLI_TIMEOUT_SECONDS, attempt_cap - _CLI_TIMEOUT_HEADROOM_SECONDS @@ -293,6 +347,7 @@ def run_debug( # The CLI's own --timeout never fired, so the stall is upstream of # polling. Keep the partial output rather than dying on a traceback. _LAST_DEBUG_RAW = _as_text(exc.stdout) + _LAST_DEBUG_STDERR = _as_text(exc.stderr) _fail_with_capture( f"flow debug exceeded the {attempt_cap}s subprocess cap without returning " f"(CLI --timeout was {cli_timeout}s, so the stall is upstream of " @@ -301,16 +356,29 @@ def run_debug( f"stdout: {_as_text(exc.stdout)}\nstderr: {_as_text(exc.stderr)}" ) _LAST_DEBUG_RAW = r.stdout - if r.returncode == 0 or not _is_transient_debug_error(r): + # Keep the CLI's stderr too: it is where `flow debug` reports what it + # could not do (e.g. the output-variables fetch). Dropping it on a zero + # exit is how the 2026-08-31/09-01 move-node failures lost their cause. + _LAST_DEBUG_STDERR = r.stderr + if r.returncode == 0: + unreadable = _variables_unreadable(_parse_json(r.stdout)) + if unreadable is None: + break + unreadable_attempts += 1 + if unreadable_attempts >= _VARIABLES_UNREADABLE_ATTEMPTS: + break + max_attempts = max(max_attempts, attempt + 2) + elif not _is_transient_debug_error(r): break - if _is_poll_timeout(r) and attempt + 1 >= _POLL_TIMEOUT_ATTEMPTS: + elif _is_poll_timeout(r) and attempt + 1 >= _POLL_TIMEOUT_ATTEMPTS: break - if attempt + 1 < attempts: + if attempt + 1 < max_attempts: left = deadline - time.monotonic() - backoff_seconds if left < _MIN_RETRY_BUDGET_SECONDS: out_of_budget = True break time.sleep(backoff_seconds) + attempt += 1 if r.returncode != 0: spent = ( @@ -330,6 +398,16 @@ def run_debug( status = _get_ci(payload, "finalStatus", "FinalStatus") if status != "Completed": _fail(f"Flow did not complete (finalStatus={status})\n{r.stdout}") + if unreadable is not None: + # Every attempt completed, and every attempt's outputs were unreadable. + # This is an infrastructure failure (INFRA), not a verdict on the flow: + # the run reached its End node, we just never saw what it returned. + _fail_with_capture( + f"flow debug completed but its outputs could not be read on " + f"{unreadable_attempts} attempt(s): {unreadable}. " + "Classify as INFRA (outputs unknown), not as a flow defect; the " + "elements all ran. See the captured STDERR for the CLI's own account." + ) return payload @@ -783,6 +861,11 @@ def assert_outputs_contain( ) +def get_last_debug_stderr() -> str | None: + """Return the stderr of the most recent ``run_debug`` call, or ``None``.""" + return _LAST_DEBUG_STDERR + + def get_last_debug_raw() -> str | None: """Return the raw stdout of the most recent ``run_debug`` call (the full ``uip maestro flow debug`` JSON envelope), or ``None`` if none ran yet. @@ -1894,6 +1977,11 @@ def _dump_debug_capture(context: str = "") -> None: except Exception as exc: # noqa: BLE001 โ€” diagnostics must never mask the real failure lines.append(f"SUMMARY: ") lines.append("RAW: " + raw.strip()) + stderr = (_LAST_DEBUG_STDERR or "").strip() + if stderr: + # The CLI's own account of the run (progress, warnings such as a failed + # output-variables fetch). Tail only: the head is login/upload noise. + lines.append("STDERR (tail): " + stderr[-_STDERR_CAPTURE_TAIL_CHARS:]) lines.append("=== FLOW_DEBUG_RAW_CAPTURE END ===") print("\n".join(lines), file=sys.stderr) diff --git a/tests/tasks/uipath-maestro-flow/_shared/test_criterion_budgets.py b/tests/tasks/uipath-maestro-flow/_shared/test_criterion_budgets.py index 08d39fcbfe..045f2310ec 100644 --- a/tests/tasks/uipath-maestro-flow/_shared/test_criterion_budgets.py +++ b/tests/tasks/uipath-maestro-flow/_shared/test_criterion_budgets.py @@ -45,6 +45,24 @@ _COMMAND = re.compile(r"command:\s*(.*)", re.S) _TIMEOUT = re.compile(r"^\s*timeout:\s*(\d+)", re.M) _TASK_DIR_SCRIPT = re.compile(r"\$TASK_DIR/(\S+\.py)") +# The same-ground campaign addresses checkers through the mounted repo instead of +# the task dir (`python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow//check_x.py`); +# both forms name one script under this suite, and both are priced. +_SUITE_SCRIPT = re.compile(r"\$SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/(\S+\.py)") + + +def _script_path(root: str, command_text: str) -> str | None: + """The checker script a run_command criterion executes, resolved on disk, or + None when the command is not one of the two forms this guard prices.""" + m = _TASK_DIR_SCRIPT.search(command_text) + if m: + resolved = os.path.join(root, m.group(1)) + else: + m = _SUITE_SCRIPT.search(command_text) + if not m: + return None + resolved = os.path.join(_SUITE_ROOT, m.group(1)) + return resolved if os.path.exists(resolved) else None _TASK_TIMEOUT = re.compile(r"^\s*task_timeout:\s*(\d+)", re.M) # Only these timeouts run inside the watchdog. _SUCCESS_CRITERIA = re.compile( @@ -562,13 +580,10 @@ def _criteria(): command = _COMMAND.search(block) if not command: continue - script = _TASK_DIR_SCRIPT.search(command.group(1)) - if not script: + resolved = _script_path(root, command.group(1)) + if not resolved: continue first_line = command.group(1).split("\n")[0].strip().strip("'\"") - resolved = os.path.join(root, script.group(1)) - if not os.path.exists(resolved): - continue timeout_line = next( (ln for ln in block.split("\n") if _TIMEOUT.search(ln)), "" ) @@ -654,8 +669,7 @@ def _eligible(): tasks += 1 for block in _criterion_blocks(text, "run_command"): command = _COMMAND.search(block) - script = _TASK_DIR_SCRIPT.search(command.group(1)) if command else None - if script and os.path.exists(os.path.join(root, script.group(1))): + if command and _script_path(root, command.group(1)): criteria += 1 return criteria, tasks diff --git a/tests/tasks/uipath-maestro-flow/_shared/test_flow_check.py b/tests/tasks/uipath-maestro-flow/_shared/test_flow_check.py index 79079e3755..b86e6c6203 100644 --- a/tests/tasks/uipath-maestro-flow/_shared/test_flow_check.py +++ b/tests/tasks/uipath-maestro-flow/_shared/test_flow_check.py @@ -883,6 +883,131 @@ def test_is_transient_debug_error(cp, expected): assert flow_check._is_transient_debug_error(cp) is expected +# โ”€โ”€ run_debug: completed but outputs unreadable (variables fetch failed) โ”€โ”€โ”€โ”€โ”€ + +# Verbatim shape of the 2026-09-01 move-node/v2 payload: every element +# Completed, no `variables` key anywhere in Data. +_COMPLETED_NO_VARIABLES = ( + '{\n "Result": "Success",\n "Code": "FlowDebug",\n' + ' "Data": {"jobKey": "j", "instanceId": "609c57fa", "finalStatus": "Completed",\n' + ' "elementExecutions": [{"elementId": "end", "status": "Completed"}]},\n' + ' "Instructions": "Debug completed with status: Completed"\n}' +) +# Shape emitted by the fixed CLI: still no `variables`, plus the structured reason. +_COMPLETED_VARIABLES_ERROR = ( + '{\n "Result": "Success",\n "Code": "FlowDebug",\n' + ' "Data": {"instanceId": "609c57fa", "finalStatus": "Completed", "elementExecutions": [],\n' + ' "variablesError": {"message": "pims API request failed: 403 Forbidden", ' + '"httpStatus": 403, "attempts": 4, "traceIds": ["t1"], ' + '"endpoint": "/api/v1/debug-instances/609c57fa/variables"}}\n}' +) +# A run whose flow declares no outputs: `variables` is PRESENT and empty. Real result. +_COMPLETED_EMPTY_VARIABLES = ( + '{\n "Result": "Success",\n' + ' "Data": {"finalStatus": "Completed", "variables": {"globals": {}, "elements": []}}\n}' +) +_CLI_STDERR = ( + "Polling for completion...\nStatus: Completed (9/9 elements completed)\n" + "Fetching output variables...\n" + "[WARN] Could not fetch variables: pims API request failed: 403 Forbidden on GET " + "/api/v1/debug-instances/609c57fa/variables\n" +) + + +def test_run_debug_retries_completed_without_variables_then_returns_outputs(monkeypatch): + """A Completed payload with no `variables` key is a failed fetch, not a + result โ€” retry the debug once; the second run's outputs are graded.""" + calls = _stub_debug( + monkeypatch, + [_cp(0, _COMPLETED_NO_VARIABLES, _CLI_STDERR), _cp(0, _COMPLETED)], + ) + payload = run_debug() + assert calls["n"] == 2 + assert flow_check.collect_outputs(payload) == ["Sev1"] + + +def test_run_debug_retries_variables_error_payload(monkeypatch): + calls = _stub_debug( + monkeypatch, [_cp(0, _COMPLETED_VARIABLES_ERROR), _cp(0, _COMPLETED)] + ) + payload = run_debug() + assert calls["n"] == 2 + assert flow_check.collect_outputs(payload) == ["Sev1"] + + +def test_run_debug_persistent_unreadable_outputs_fails_as_infra(monkeypatch, capsys): + """Both attempts complete with unreadable outputs: fail with an INFRA + message and the capture (RAW + the CLI's STDERR), never as + 'Outputs missing' โ€” that verdict would blame the flow.""" + calls = _stub_debug( + monkeypatch, + [ + _cp(0, _COMPLETED_NO_VARIABLES, _CLI_STDERR), + _cp(0, _COMPLETED_NO_VARIABLES, _CLI_STDERR), + ], + ) + with pytest.raises(SystemExit) as exc: + run_debug() + assert calls["n"] == flow_check._VARIABLES_UNREADABLE_ATTEMPTS == 2 + msg = str(exc.value) + assert "could not be read" in msg + assert "INFRA" in msg + assert "Outputs missing" not in msg + err = capsys.readouterr().err + assert "FLOW_DEBUG_RAW_CAPTURE BEGIN" in err + assert "STDERR (tail):" in err + assert "Could not fetch variables" in err + + +def test_run_debug_does_not_retry_present_but_empty_variables(monkeypatch): + """`variables` present and empty is a genuine 'no outputs' result โ€” return + it on the first attempt so the caller's assertion grades it.""" + calls = _stub_debug(monkeypatch, [_cp(0, _COMPLETED_EMPTY_VARIABLES)]) + payload = run_debug() + assert calls["n"] == 1 + assert flow_check.collect_outputs(payload) == [] + + +def test_run_debug_unreadable_retry_does_not_burn_the_transient_budget(monkeypatch): + """The unreadable-outputs retry is its own budget (2), independent of the + 3 attempts reserved for 5xx/RetryLater.""" + calls = _stub_debug( + monkeypatch, + [_cp(1, _TRANSIENT_504), _cp(0, _COMPLETED_NO_VARIABLES), _cp(0, _COMPLETED)], + ) + payload = run_debug() + assert calls["n"] == 3 + assert flow_check.collect_outputs(payload) == ["Sev1"] + + +@pytest.mark.parametrize( + "stdout,expected", + [ + (_COMPLETED, None), + (_COMPLETED_EMPTY_VARIABLES, None), + ('{"Result": "Success", "Data": {"finalStatus": "Faulted"}}', None), # not Completed: other path + (_COMPLETED_NO_VARIABLES, "no `variables` key"), + (_COMPLETED_VARIABLES_ERROR, "variablesError after 4 attempt(s)"), + ], +) +def test_variables_unreadable_classifier(stdout, expected): + reason = flow_check._variables_unreadable(flow_check._parse_json(stdout)) + if expected is None: + assert reason is None + else: + assert expected in reason + + +def test_capture_includes_cli_stderr_tail(capsys, _reset_debug_raw, monkeypatch): + monkeypatch.setattr(flow_check, "_LAST_DEBUG_STDERR", _CLI_STDERR) + flow_check._LAST_DEBUG_RAW = _COMPLETED_NO_VARIABLES + with pytest.raises(SystemExit): + flow_check._fail_with_capture("boom") + err = capsys.readouterr().err + assert "STDERR (tail): Polling for completion..." in err + assert "Could not fetch variables" in err + + # โ”€โ”€ run_debug timeout budget + diagnostics โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ # # Regression cover for skill-flow-wiki-pageviews: the subprocess cap fired below From d2bd00eac9b378f34f393df4152386f2abbf73e2 Mon Sep 17 00:00:00 2001 From: Tao Ma <51425734+tmatup@users.noreply.github.com> Date: Tue, 1 Sep 2026 17:46:37 -0700 Subject: [PATCH 37/52] fix(flow-check): name an unconfigured connector node instead of "no create-entity-record" (#2972) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(flow-check): name an unconfigured connector node instead of "no create-entity-record" skill-flow-datafabric-smoke-file-activities, SDK arm, 2026-09-01: the agent placed `create-entity-record` through `rawNode`, so the node existed but carried its inputs flat with no `inputs.detail`. The checker reads entity/body/connection from `detail` (that is what the platform reads), skipped the node, and said "no create-entity-record on FlowCodeEvalEntity" โ€” which reads as "the agent forgot the step". It did not; the step can never run. The verdict is unchanged (same nodes pass, same nodes fail โ€” verified on the 09-01 v1 pass, the 09-01 v2 fail and the 08-31 22:43 v2 fail artifacts). The FAIL line now says where entityName is expected and lists connector nodes that have no `inputs.detail`, so the next reader lands on the enabling defect (flow-builder-sdk#663 refuses that shape at compile) instead of the prompt. Tests: 3 new (configured pass, raw-unconfigured fail + message, nested-body fail); pytest tests/tasks/uipath-maestro-flow โ†’ 343 passed; check-task-driver OK. Measurement contract: no prompt, criterion, weight or run_limits change. ๐Ÿค– Generated with Claude Code Co-Authored-By: [Claude](mailto:noreply@anthropic.com) Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01SMAwQ1iQQSUUPi3oatMqgG * fix(flow-check): render the unconfigured-node list without Python repr brackets (review nit) ๐Ÿค– Generated with Claude Code Co-Authored-By: [Claude](mailto:noreply@anthropic.com) Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01SMAwQ1iQQSUUPi3oatMqgG --------- Co-authored-by: Claude Fable 5.1 --- .../check_smoke_file_activities.py | 17 ++- .../test_check_smoke_file_activities.py | 100 ++++++++++++++++++ 2 files changed, 116 insertions(+), 1 deletion(-) create mode 100644 tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/test_check_smoke_file_activities.py diff --git a/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/check_smoke_file_activities.py b/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/check_smoke_file_activities.py index c0b41be1b9..ab25123adb 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/check_smoke_file_activities.py +++ b/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/check_smoke_file_activities.py @@ -90,9 +90,19 @@ def main() -> int: upload_node = None delete_node = None create_node = None + # Connector nodes that carry no `inputs.detail` at all. The platform reads + # a connector's entity/body/connection from `detail`, so such a node can + # never run โ€” it is an unconfigured node (SDK `rawNode`, a hand-written + # node, or `node add` without `node configure`), not a missing one. + # Naming it keeps "no create-entity-record" from being read as "the + # agent forgot the step" (2026-09-01 v2: the step was there, raw). + unconfigured = [] for n in doc.get("nodes", []): t = n.get("type", "") detail = n.get("inputs", {}).get("detail", {}) + if t.startswith("uipath.connector.") and (not isinstance(detail, dict) or not detail): + unconfigured.append(f"{n.get('id')} ({t.rsplit('.', 1)[-1]})") + detail = {} pp = detail.get("pathParameters") or {} query = detail.get("queryParameters") or {} body = detail.get("bodyParameters") or {} @@ -111,6 +121,11 @@ def main() -> int: delete_node = n if not REQUIRED.issubset(seen.keys()): continue + unconfigured_note = ( + f" Unconfigured connector node(s) with no inputs.detail: {', '.join(unconfigured)} โ€” " + f"the platform reads entity, body and connection from detail, so these can never run." + if unconfigured else "" + ) resolved = {s: resolve_field(v, globals_by_id) for s, v in seen.items()} wrong = [(s, seen[s], resolved[s]) for s in REQUIRED if resolved[s] != FIELD] if wrong: @@ -135,7 +150,7 @@ def main() -> int: # create-entity-record on ENTITY must exist and upload+delete recordId must reference its output if not create_node: - print(f"FAIL: {path} โ€” no create-entity-record on {ENTITY}", file=sys.stderr) + print(f"FAIL: {path} โ€” no create-entity-record on {ENTITY} with inputs.detail.pathParameters.entityName={ENTITY!r}.{unconfigured_note}", file=sys.stderr) return 1 create_body = (create_node.get("inputs", {}).get("detail", {}).get("bodyParameters") or {}) required_body = {"title", "description", "score"} diff --git a/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/test_check_smoke_file_activities.py b/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/test_check_smoke_file_activities.py new file mode 100644 index 0000000000..b9a46518f6 --- /dev/null +++ b/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/test_check_smoke_file_activities.py @@ -0,0 +1,100 @@ +"""Unit tests for check_smoke_file_activities.py โ€” purely structural, no CLI. + +Run with ``pytest tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/test_check_smoke_file_activities.py``. + +The shapes come from real eval artifacts of ``skill-flow-datafabric-smoke-file-activities``: +the v1 arm's configured connector nodes (pass), and the 2026-09-01 SDK arm, whose +``create-entity-record`` node was placed through ``rawNode`` and so carried its inputs +flat with no ``inputs.detail`` (fail โ€” and the message must say the node is unconfigured, +not absent). +""" + +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path +from typing import Any + +CHECKER = Path(__file__).resolve().parent / "check_smoke_file_activities.py" +ENTITY = "FlowCodeEvalEntity" +DS = "uipath.connector.uipath-uipath-dataservice." + + +def _run(cwd: Path) -> subprocess.CompletedProcess[str]: + return subprocess.run([sys.executable, str(CHECKER)], cwd=str(cwd), capture_output=True, text=True) + + +def _configured(node_id: str, action: str, *, path: dict[str, Any] | None = None, + query: dict[str, Any] | None = None, body: dict[str, Any] | None = None, + multipart: list[dict[str, Any]] | None = None) -> dict[str, Any]: + detail: dict[str, Any] = { + "connector": "uipath-uipath-dataservice", + "connectionId": "d61e5d0e-04af-4f93-95cc-151d81fa08dc", + "connectionFolderKey": "c4359cde-55f0-4f0e-9322-c6cdce74ab4c", + "pathParameters": path or {"entityName": ENTITY}, + } + if query is not None: + detail["queryParameters"] = query + if body is not None: + detail["bodyParameters"] = body + if multipart is not None: + detail["multipartParameters"] = multipart + return {"id": node_id, "type": DS + action, "typeVersion": "1.0.0", "inputs": {"detail": detail}} + + +def _file_nodes(create_node: dict[str, Any]) -> list[dict[str, Any]]: + return [ + {"id": "start", "type": "core.trigger.manual"}, + _configured("downloadFile", "download-file-from-record-field", + query={"recordId": "11111111-1111-4111-8111-111111111111", "_fieldName": "file1"}), + create_node, + _configured("uploadFile", "upload-file-to-record-field", + query={"recordId": "=js:$vars.createRecord.output.Id", "_fieldName": "file1"}, + multipart=[{"name": "file", "dataType": "file", "value": "=js:$vars.downloadedFile"}]), + _configured("deleteFile", "delete-file-from-record-field", + query={"recordId": "=js:$vars.createRecord.output.Id", "_fieldName": "file1"}), + {"id": "end", "type": "core.control.end"}, + ] + + +def _write(tmp_path: Path, nodes: list[dict[str, Any]]) -> None: + doc = { + "id": "t", "version": "1.0.0", "name": "T", "nodes": nodes, "edges": [], + "variables": {"globals": [{"id": "downloadedFile", "name": "downloadedFile", + "direction": "inout", "type": "file"}], + "variableUpdates": {}}, + } + (tmp_path / "TransferFile.flow").write_text(json.dumps(doc)) + + +def test_configured_create_with_flat_body_passes(tmp_path: Path) -> None: + create = _configured("createRecord", "create-entity-record", + body={"title": "t", "description": "d", "score": 42.5}) + _write(tmp_path, _file_nodes(create)) + r = _run(tmp_path) + assert r.returncode == 0, r.stderr + assert "OK:" in r.stdout + + +def test_raw_unconfigured_create_fails_and_is_named_as_unconfigured(tmp_path: Path) -> None: + # The 2026-09-01 SDK artifact: inputs flat, no `detail` โ€” an unconfigured node. + raw_create = {"id": "createRecord", "type": DS + "create-entity-record", "typeVersion": "1.0.0", + "inputs": {"entityName": ENTITY, "body": {"title": "t", "description": "d", "score": 42.5}}} + _write(tmp_path, _file_nodes(raw_create)) + r = _run(tmp_path) + assert r.returncode == 1 + assert "no create-entity-record" in r.stderr + assert "Unconfigured connector node(s) with no inputs.detail" in r.stderr + assert "createRecord (create-entity-record)" in r.stderr + + +def test_nested_body_object_still_fails_on_required_fields(tmp_path: Path) -> None: + # The 2026-08-31 22:43 SDK artifact: a forged overlay nested the body under `body`. + create = _configured("createRecord", "create-entity-record", + body={"body": {"title": "t", "description": "d", "score": 42.5}}) + _write(tmp_path, _file_nodes(create)) + r = _run(tmp_path) + assert r.returncode == 1 + assert "create body missing required fields" in r.stderr From e470389ef653d63b6caacd043162b9a544e184a3 Mon Sep 17 00:00:00 2001 From: Tao Ma <51425734+tmatup@users.noreply.github.com> Date: Tue, 1 Sep 2026 22:56:11 -0700 Subject: [PATCH 38/52] fix(flow-check): carry the backend's incident details when a debug run faults (#2981) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `flow debug` reports a fault as one line per incident โ€” `[102003] Integration Services bad request (element createIssue)` โ€” and defers the provider's message to `uip maestro flow debug-instance incidents `. The criterion output never carried that message, so six identical `[102003]` failures of skill-flow-e2e-escalation-jira-ticket across three arms and two weeks (08-19 โ†’ 09-01) read as an opaque IS error. The instance still had the answer on 09-02: `errors - {reporter=Specify a valid value for Reporter}` โ€” the flow had bound the sender's e-mail into `fields.reporter.id`, an Atlassian account-id slot. run_debug now fetches the incidents for a faulted instance (exitโ‰ 0 with an instanceId, or exit 0 with finalStatusโ‰ Completed) and appends `element= code= message: ErrorDetails` lines to the failure text. Best-effort, 60 s cap, never masks the original failure; grading is unchanged. RCA: workspaces/reports/2026-09-02-maestro-flow-escalation-jira-ticket/rca-escalation-jira-ticket.md ๐Ÿค– Generated with Claude Code Co-Authored-By: [Claude](mailto:noreply@anthropic.com) Claude-Session: https://claude.ai/code/session_01TkcZFmSDDFpBghDWatA9gv Co-authored-by: Claude Fable 5.1 --- .../uipath-maestro-flow/_shared/flow_check.py | 42 ++++++++++++- .../_shared/test_flow_check.py | 61 +++++++++++++++++++ 2 files changed, 102 insertions(+), 1 deletion(-) diff --git a/tests/tasks/uipath-maestro-flow/_shared/flow_check.py b/tests/tasks/uipath-maestro-flow/_shared/flow_check.py index 3202bd67fd..e993f109e6 100644 --- a/tests/tasks/uipath-maestro-flow/_shared/flow_check.py +++ b/tests/tasks/uipath-maestro-flow/_shared/flow_check.py @@ -108,6 +108,8 @@ # 5xx, matching the CLI's own "retry once before reporting") that keeps a third # full-length attempt out of every criterion ceiling. `retries=3` still works. _DEFAULT_RETRIES_TIMEOUT = 240 +# One backend read of a faulted instance's incidents; never worth more of the criterion budget. +_INCIDENTS_TIMEOUT_SECONDS = 60 _DEFAULT_RETRIES = 2 _DEFAULT_BACKOFF_SECONDS = 5.0 @@ -390,6 +392,7 @@ def run_debug( _fail( f"flow debug exit {r.returncode}{spent}\n" f"stdout: {r.stdout}\nstderr: {r.stderr}" + + _incident_details(_parse_json(r.stdout)) ) data = _parse_json(r.stdout) if data is None: @@ -397,7 +400,7 @@ def run_debug( payload = _get_ci(data, "Data") or {} status = _get_ci(payload, "finalStatus", "FinalStatus") if status != "Completed": - _fail(f"Flow did not complete (finalStatus={status})\n{r.stdout}") + _fail(f"Flow did not complete (finalStatus={status})\n{r.stdout}" + _incident_details(data)) if unreadable is not None: # Every attempt completed, and every attempt's outputs were unreadable. # This is an infrastructure failure (INFRA), not a verdict on the flow: @@ -1701,6 +1704,43 @@ def assert_no_flow_files() -> None: # โ”€โ”€ Internals โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +def _incident_details(envelope: dict | None) -> str: + """The backend's own account of a faulted debug run, for the failure text. + + `flow debug` reports a fault as one line per incident (`[102003] Integration + Services bad request (element createIssue)`) and defers the provider's + message to `uip maestro flow debug-instance incidents `. That message is + the cause โ€” 2026-09-01: `errors - {reporter=Specify a valid value for + Reporter}` behind six identical `[102003]` failures across three arms and two + weeks, none of whose task.json said so. Fetch it while the instance is still + on the tenant (it is gone within days) so the criterion output carries it. + + Best-effort and side-effect-free: returns "" when there is no instance id, + the CLI is unavailable, or the tenant answers with anything but a list. + """ + data = _get_ci(envelope, "Data") or {} + instance_id = _get_ci(data, "instanceId", "InstanceId") + if not instance_id: + return "" + try: + r = subprocess.run( + ["uip", "maestro", "flow", "debug-instance", "incidents", str(instance_id), "--output", "json"], + capture_output=True, text=True, timeout=_INCIDENTS_TIMEOUT_SECONDS, + ) + parsed = _parse_json(r.stdout) or {} + incidents = _get_ci(parsed, "Data") + if not isinstance(incidents, list) or not incidents: + return "" + lines = [ + f" - element={_get_ci(i, 'elementId', 'ElementId')} code={_get_ci(i, 'errorCode', 'ErrorCode')} " + f"{_get_ci(i, 'errorMessage', 'ErrorMessage')}: {_get_ci(i, 'errorDetails', 'ErrorDetails')}" + for i in incidents if isinstance(i, dict) + ] + return "\nincidents (uip maestro flow debug-instance incidents):\n" + "\n".join(lines) + except Exception: # noqa: BLE001 โ€” diagnostics must never mask the real failure + return "" + + def _parse_json(stdout: str) -> dict | None: try: return json.loads(stdout) diff --git a/tests/tasks/uipath-maestro-flow/_shared/test_flow_check.py b/tests/tasks/uipath-maestro-flow/_shared/test_flow_check.py index b86e6c6203..2d6e77b28e 100644 --- a/tests/tasks/uipath-maestro-flow/_shared/test_flow_check.py +++ b/tests/tasks/uipath-maestro-flow/_shared/test_flow_check.py @@ -883,6 +883,67 @@ def test_is_transient_debug_error(cp, expected): assert flow_check._is_transient_debug_error(cp) is expected +# โ”€โ”€ run_debug: a faulted run carries the backend's incident details โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + +_FAULTED_102003 = ( + '{\n "Result": "Failure",\n "Code": "FlowDebug",\n' + ' "Message": "Debug session Faulted. [102003] Integration Services bad request (element createIssue)",\n' + ' "Context": {"ErrorCode": "102003"},\n' + ' "Data": {"instanceId": "d007648d-8341-460c-92f8-c0fe4b8bde59", "finalStatus": "Faulted"}\n}' +) +_INCIDENTS = ( + '{\n "Result": "Success",\n "Code": "DebugInstanceIncidents",\n "Data": [{\n' + ' "ElementId": "createIssue", "ErrorCode": "102003",\n' + ' "ErrorMessage": "Integration Services bad request",\n' + ' "ErrorDetails": "Request to Integration Services failed with status code \'400\', message: ' + '{\\"providerMessage\\":\\"errors - {reporter=Specify a valid value for Reporter}\\"}"\n }]\n}' +) + + +def test_run_debug_fault_appends_incident_details(monkeypatch): + """2026-09-01 escalation-jira-ticket: the CLI said `[102003]` and deferred the + provider's message to `debug-instance incidents`; the criterion output never + carried it. Now the failure text does, fetched while the instance still exists.""" + calls = _stub_debug(monkeypatch, [_cp(1, _FAULTED_102003), _cp(0, _INCIDENTS)]) + with pytest.raises(SystemExit) as exc: + run_debug() + text = str(exc.value) + assert "flow debug exit 1" in text + assert "incidents (uip maestro flow debug-instance incidents):" in text + assert "element=createIssue code=102003" in text + assert "reporter=Specify a valid value for Reporter" in text + assert calls["n"] == 2 + assert calls["cmd"][:5] == ["uip", "maestro", "flow", "debug-instance", "incidents"] + assert calls["cmd"][5] == "d007648d-8341-460c-92f8-c0fe4b8bde59" + + +def test_run_debug_fault_without_instance_id_is_unchanged(monkeypatch): + bad = '{\n "Result": "Failure",\n "ErrorCode": "invalid_argument",\n "Retry": "RetryWillNotFix"\n}' + calls = _stub_debug(monkeypatch, [_cp(1, bad)]) + with pytest.raises(SystemExit) as exc: + run_debug() + assert "incidents" not in str(exc.value) + assert calls["n"] == 1 # no incidents call without an instance to ask about + + +def test_run_debug_fault_incidents_read_failure_never_masks_the_fault(monkeypatch): + calls = _stub_debug(monkeypatch, [_cp(1, _FAULTED_102003), _cp(1, "", "Not logged in")]) + with pytest.raises(SystemExit) as exc: + run_debug() + assert "flow debug exit 1" in str(exc.value) + assert "incidents (" not in str(exc.value) + assert calls["n"] == 2 + + +def test_run_debug_exit0_faulted_appends_incident_details(monkeypatch): + faulted = '{\n "Result": "Success",\n "Data": {"finalStatus": "Faulted", "instanceId": "abc"}\n}' + _stub_debug(monkeypatch, [_cp(0, faulted), _cp(0, _INCIDENTS)]) + with pytest.raises(SystemExit) as exc: + run_debug() + assert "Flow did not complete (finalStatus=Faulted)" in str(exc.value) + assert "reporter=Specify a valid value for Reporter" in str(exc.value) + + # โ”€โ”€ run_debug: completed but outputs unreadable (variables fetch failed) โ”€โ”€โ”€โ”€โ”€ # Verbatim shape of the 2026-09-01 move-node/v2 payload: every element From fd3381a9047d6ea0b9ccd738e017036cd0644926 Mon Sep 17 00:00:00 2001 From: Tao Ma <51425734+tmatup@users.noreply.github.com> Date: Tue, 1 Sep 2026 23:29:06 -0700 Subject: [PATCH 39/52] =?UTF-8?q?fix(jira-teardown):=20confirm=20the=20irr?= =?UTF-8?q?eversible=20delete=20=E2=80=94=20every=20Jira=20e2e=20teardown?= =?UTF-8?q?=20since=2008-19=20leaked=20its=20ticket=20(#2983)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `uip is resources run delete` never prompts and refuses without `--yes` ("Confirmation required: this will delete resource 'issue' and cannot be undone. Re-run with --yes to confirm."). The five task-local `jira_is.py` helpers (escalation_jira_ticket, jira_create_issue, jira_get_issue, jira_lifecycle, jira_search_triage) called it without the flag, so every teardown in the 08-19 โ†’ 09-01 run archive printed `WARN: could NOT confirm deletion of CE-โ€ฆ` (escalation) or silently returned (the others), and the CE project accumulated the tickets: CE-1150 โ†’ CE-1256 are ALL eval-created (search-triage, lifecycle, flow e2e, ESC-JIRA), none deleted. The helper's refusal-is-not-a-deletion logic was right; the invocation was missing one word. Verified on the host 09-02: with `--yes` the delete returns Success and an independent re-read of the key answers 404 (CE-1257, CE-1258 โ€” this session's own reruns). Tests: jira_create_issue/test_jira_is.py +1 (the command carries --yes); escalation_jira_ticket/test_jira_is.py new, 3 cases (confirmed delete โ†’ True; "Confirmation required" Failure โ†’ False; structured 404 โ†’ True). pytest tests/tasks/uipath-maestro-flow/: 1073 passed. Historical leaks left for Tao (not deleted here): CE-1169, CE-1172, CE-1173, CE-1183, CE-1186, CE-1222, CE-1225, CE-1247, CE-1248 (ESC-JIRA-โ€ฆ summaries), plus the search-triage / lifecycle / jira-flow-e2e tickets in the same range. RCA: workspaces/reports/2026-09-02-maestro-flow-escalation-jira-ticket/rca-escalation-jira-ticket.md ยง6 ๐Ÿค– Generated with Claude Code Co-Authored-By: [Claude](mailto:noreply@anthropic.com) Claude-Session: https://claude.ai/code/session_01TkcZFmSDDFpBghDWatA9gv Co-authored-by: Claude Fable 5.1 --- .../e2e/escalation_jira_ticket/jira_is.py | 4 ++ .../escalation_jira_ticket/test_jira_is.py | 71 +++++++++++++++++++ .../e2e/jira_create_issue/jira_is.py | 4 ++ .../e2e/jira_create_issue/test_jira_is.py | 19 +++++ .../e2e/jira_get_issue/jira_is.py | 4 ++ .../e2e/jira_lifecycle/jira_is.py | 4 ++ .../e2e/jira_search_triage/jira_is.py | 4 ++ 7 files changed, 110 insertions(+) create mode 100644 tests/tasks/uipath-maestro-flow/e2e/escalation_jira_ticket/test_jira_is.py diff --git a/tests/tasks/uipath-maestro-flow/e2e/escalation_jira_ticket/jira_is.py b/tests/tasks/uipath-maestro-flow/e2e/escalation_jira_ticket/jira_is.py index f017e61484..046ed14d16 100644 --- a/tests/tasks/uipath-maestro-flow/e2e/escalation_jira_ticket/jira_is.py +++ b/tests/tasks/uipath-maestro-flow/e2e/escalation_jira_ticket/jira_is.py @@ -125,6 +125,10 @@ def delete_issue(conn_id: str, key: str) -> bool: env = _run( "is", "resources", "run", "delete", CONNECTOR, "issue", "--connection-id", conn_id, "--query", f"issueId={key}", + # The CLI never prompts and REFUSES an irreversible delete without this + # flag ("Confirmation required โ€ฆ Re-run with --yes"). Without it every + # teardown since 08-19 printed WARN and left its ticket in the CE project. + "--yes", ) if str(env.get("Result", "")).lower() != "failure": return True diff --git a/tests/tasks/uipath-maestro-flow/e2e/escalation_jira_ticket/test_jira_is.py b/tests/tasks/uipath-maestro-flow/e2e/escalation_jira_ticket/test_jira_is.py new file mode 100644 index 0000000000..5dfb339f03 --- /dev/null +++ b/tests/tasks/uipath-maestro-flow/e2e/escalation_jira_ticket/test_jira_is.py @@ -0,0 +1,71 @@ +"""Teardown contract for the escalation task's Jira helper.""" + +from __future__ import annotations + +import importlib.util +import json +from pathlib import Path +from types import ModuleType + +import pytest + +HERE = Path(__file__).parent + + +def _load_module() -> ModuleType: + spec = importlib.util.spec_from_file_location("escalation_jira_is", HERE / "jira_is.py") + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class Result: + def __init__(self, envelope: object, returncode: int = 0) -> None: + self.stdout = json.dumps(envelope) + self.stderr = "" + self.returncode = returncode + + +def _stub(monkeypatch: pytest.MonkeyPatch, jira_is: ModuleType, envelope: object) -> list[list[str]]: + seen: list[list[str]] = [] + + def fake_run(cmd, **kwargs): + seen.append(list(cmd)) + return Result(envelope) + + monkeypatch.setattr(jira_is.subprocess, "run", fake_run) + return seen + + +def test_delete_issue_confirms_and_reports_success(monkeypatch: pytest.MonkeyPatch) -> None: + """The CLI refuses an unconfirmed delete with a Failure envelope + ("Confirmation required โ€ฆ Re-run with --yes"), which `delete_issue` + correctly reports as NOT confirmed โ€” so every run 08-19 โ†’ 09-01 printed + `WARN: could NOT confirm deletion` and left its ticket in CE. `--yes` is + the missing word.""" + jira_is = _load_module() + seen = _stub(monkeypatch, jira_is, {"Result": "Success", "Data": {"Value": ""}}) + assert jira_is.delete_issue("conn", "CE-1257") is True + (cmd,) = seen + assert cmd[:5] == ["uip", "is", "resources", "run", "delete"] + assert "--yes" in cmd and "issueId=CE-1257" in cmd + + +def test_delete_issue_unconfirmed_failure_is_not_a_deletion(monkeypatch: pytest.MonkeyPatch) -> None: + jira_is = _load_module() + _stub(monkeypatch, jira_is, { + "Result": "Failure", + "Message": "Confirmation required: this will delete resource 'issue' and cannot be undone.", + "Instructions": "Re-run with --yes to confirm.", + }) + assert jira_is.delete_issue("conn", "CE-1257") is False + + +def test_delete_issue_structured_404_counts_as_gone(monkeypatch: pytest.MonkeyPatch) -> None: + jira_is = _load_module() + _stub(monkeypatch, jira_is, { + "Result": "Failure", + "Message": "Request failed with status code '404': Issue does not exist or you do not have permission to see it.", + }) + assert jira_is.delete_issue("conn", "CE-1257") is True diff --git a/tests/tasks/uipath-maestro-flow/e2e/jira_create_issue/jira_is.py b/tests/tasks/uipath-maestro-flow/e2e/jira_create_issue/jira_is.py index 960562ae42..90dd603348 100644 --- a/tests/tasks/uipath-maestro-flow/e2e/jira_create_issue/jira_is.py +++ b/tests/tasks/uipath-maestro-flow/e2e/jira_create_issue/jira_is.py @@ -106,4 +106,8 @@ def delete_issue(conn_id: str, key: str) -> None: _run( "is", "resources", "run", "delete", CONNECTOR, "issue", "--connection-id", conn_id, "--query", f"issueId={key}", + # The CLI never prompts and REFUSES an irreversible delete without this + # flag ("Confirmation required โ€ฆ Re-run with --yes"). Without it every + # teardown since 08-19 printed WARN and left its ticket in the CE project. + "--yes", ) diff --git a/tests/tasks/uipath-maestro-flow/e2e/jira_create_issue/test_jira_is.py b/tests/tasks/uipath-maestro-flow/e2e/jira_create_issue/test_jira_is.py index 4ff92e1598..340d4c8d9c 100644 --- a/tests/tasks/uipath-maestro-flow/e2e/jira_create_issue/test_jira_is.py +++ b/tests/tasks/uipath-maestro-flow/e2e/jira_create_issue/test_jira_is.py @@ -68,3 +68,22 @@ def test_invalid_json_does_not_echo_cli_output(monkeypatch: pytest.MonkeyPatch) assert "invalid JSON" in message assert "exit 1" in message assert "must-not-appear" not in message + + +def test_delete_issue_confirms_the_irreversible_delete(monkeypatch: pytest.MonkeyPatch) -> None: + """`uip is resources run delete` never prompts and refuses without `--yes` + ("Confirmation required: this will delete resource 'issue' โ€ฆ"). Every + teardown in the 08-19 โ†’ 09-01 run archive omitted it and leaked its ticket.""" + jira_is = _load_module() + seen: list[list[str]] = [] + + def fake_run(cmd, **kwargs): + seen.append(list(cmd)) + return Result({"Result": "Success", "Data": {"Value": ""}}) + + monkeypatch.setattr(jira_is.subprocess, "run", fake_run) + jira_is.delete_issue("conn", "CE-1") + (cmd,) = seen + assert cmd[:5] == ["uip", "is", "resources", "run", "delete"] + assert "--yes" in cmd + assert "issueId=CE-1" in cmd diff --git a/tests/tasks/uipath-maestro-flow/e2e/jira_get_issue/jira_is.py b/tests/tasks/uipath-maestro-flow/e2e/jira_get_issue/jira_is.py index 40a9b2acff..9e6edb53c0 100644 --- a/tests/tasks/uipath-maestro-flow/e2e/jira_get_issue/jira_is.py +++ b/tests/tasks/uipath-maestro-flow/e2e/jira_get_issue/jira_is.py @@ -59,4 +59,8 @@ def delete_issue(conn_id: str, key: str) -> None: _run( "is", "resources", "run", "delete", CONNECTOR, "issue", "--connection-id", conn_id, "--query", f"issueId={key}", + # The CLI never prompts and REFUSES an irreversible delete without this + # flag ("Confirmation required โ€ฆ Re-run with --yes"). Without it every + # teardown since 08-19 printed WARN and left its ticket in the CE project. + "--yes", ) diff --git a/tests/tasks/uipath-maestro-flow/e2e/jira_lifecycle/jira_is.py b/tests/tasks/uipath-maestro-flow/e2e/jira_lifecycle/jira_is.py index da3f617715..e5e2b601ee 100644 --- a/tests/tasks/uipath-maestro-flow/e2e/jira_lifecycle/jira_is.py +++ b/tests/tasks/uipath-maestro-flow/e2e/jira_lifecycle/jira_is.py @@ -60,4 +60,8 @@ def delete_issue(conn_id: str, key: str) -> None: _run( "is", "resources", "run", "delete", CONNECTOR, "issue", "--connection-id", conn_id, "--query", f"issueId={key}", + # The CLI never prompts and REFUSES an irreversible delete without this + # flag ("Confirmation required โ€ฆ Re-run with --yes"). Without it every + # teardown since 08-19 printed WARN and left its ticket in the CE project. + "--yes", ) diff --git a/tests/tasks/uipath-maestro-flow/e2e/jira_search_triage/jira_is.py b/tests/tasks/uipath-maestro-flow/e2e/jira_search_triage/jira_is.py index 188986d2f6..d734b13884 100644 --- a/tests/tasks/uipath-maestro-flow/e2e/jira_search_triage/jira_is.py +++ b/tests/tasks/uipath-maestro-flow/e2e/jira_search_triage/jira_is.py @@ -58,4 +58,8 @@ def delete_issue(conn_id: str, key: str) -> None: _run( "is", "resources", "run", "delete", CONNECTOR, "issue", "--connection-id", conn_id, "--query", f"issueId={key}", + # The CLI never prompts and REFUSES an irreversible delete without this + # flag ("Confirmation required โ€ฆ Re-run with --yes"). Without it every + # teardown since 08-19 printed WARN and left its ticket in the CE project. + "--yes", ) From 4c151dd18710ae217b0e46399532a6c7b0b8b428 Mon Sep 17 00:00:00 2001 From: Tao Ma <51425734+tmatup@users.noreply.github.com> Date: Wed, 2 Sep 2026 00:32:33 -0700 Subject: [PATCH 40/52] fix(eval-checker): discover the orchestrator-paths flow for validate instead of hardcoding the v1 solution path (#2986) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `escalation_orchestrator_paths.yaml` was the only e2e task whose "validates without errors" criterion ran `uip maestro flow validate CustomerEscalationOrchestrator/CustomerEscalationOrchestrator/CustomerEscalationOrchestrator.flow` by literal path. The SDK loop's own SKILL.md scaffolds `uip solution init Sol`, so an agent following it emits `CustomerEscalationOrchestratorSol/CustomerEscalationOrchestrator/โ€ฆflow` and the criterion fails "File not found" on a flow that is otherwise green โ€” adhoc 2026-08-31 22:43 v2 scored 0.68: every seeded path case passed, validate 0.0. Two of the task's six archived v2 failures carry this signature (08-29: `โ€ฆSolution/`, 08-31 22:43: `โ€ฆSol/`). Every sibling (escalation_slack_alert, escalation_jira_ticket, jira_*) already validates through `_shared/validate_flow.py`, which discovers the project's `.flow` files the same way the checker does. Same weight, threshold and timeout. Criterion edit only โ€” the prompt (fenced by FORBIDDEN_PROMPT_EXCEPTIONS) is untouched. ๐Ÿค– Generated with Claude Code Co-Authored-By: [Claude](mailto:noreply@anthropic.com) Claude-Session: https://claude.ai/code/session_013cn8thdbhgRMqTehFhnAWW Co-authored-by: Claude Fable 5.1 --- .../escalation_orchestrator_paths.yaml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/tasks/uipath-maestro-flow/e2e/escalation_orchestrator_paths/escalation_orchestrator_paths.yaml b/tests/tasks/uipath-maestro-flow/e2e/escalation_orchestrator_paths/escalation_orchestrator_paths.yaml index bb191b3e1a..10954e51d9 100644 --- a/tests/tasks/uipath-maestro-flow/e2e/escalation_orchestrator_paths/escalation_orchestrator_paths.yaml +++ b/tests/tasks/uipath-maestro-flow/e2e/escalation_orchestrator_paths/escalation_orchestrator_paths.yaml @@ -94,9 +94,14 @@ success_criteria: weight: 1.5 pass_threshold: 1.0 + # โ”€โ”€ The delivered flow validates โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + # Discovered, not hardcoded: the SDK loop scaffolds `Sol//.flow` + # (its SKILL.md says `uip solution init Sol`), so a fixed + # `//.flow` path failed "File not found" on flows that were + # otherwise green (adhoc 2026-08-31 22:43 v2: seeded paths passed, validate 0.0). - type: run_command - description: "Generated CustomerEscalationOrchestrator Flow validates without errors" - command: "uip maestro flow validate CustomerEscalationOrchestrator/CustomerEscalationOrchestrator/CustomerEscalationOrchestrator.flow --output json" + description: "Generated CustomerEscalationOrchestrator Flow validates without errors, independent of solution layout" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py" timeout: 180 expected_exit_code: 0 weight: 3.0 From 20ae144196b7f7fa24feb95cf685485934430e1b Mon Sep 17 00:00:00 2001 From: Tao Ma <51425734+tmatup@users.noreply.github.com> Date: Wed, 2 Sep 2026 01:11:36 -0700 Subject: [PATCH 41/52] =?UTF-8?q?fix(flow-check):=20INTERIM=20=E2=80=94=20?= =?UTF-8?q?retry=20a=20debug=20once=20as=20a=20new=20import=20when=20Studi?= =?UTF-8?q?o=20Web's=20Overwrite=20answers=20400/1001=20(UiPath/cli#3938)?= =?UTF-8?q?=20(#2990)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since 2026-09-02 (last known good 2026-09-01 07:30Z) Studio Web's `Solution/{id}/Overwrite` returns `400 {"code":"1001","message":"An argument had an invalid value."}` for a solution that EXISTS. `uip maestro flow debug` imports a project on its first run and writes the server id back into the `.uipx` (`updateSolutionFile`), so every later debug of the same project is an overwrite, and the CLI treats a non-404 Overwrite as fatal. Every multi-case checker therefore died at solution upload on case 2, in all three arms: orchestrator_paths (7 cases), escalation_slack_alert, billing_invoice_lookup, decision, wiki_pageviews, canary. Reproduced on the host with the current CLI and yesterday's image (1.202.0-dev.8414), and with `uip solution upload --force`; a fresh bundled SolutionId imports as new and Completes with a real Slack post โ€” only overwrite is broken. The one earlier archived Overwrite 400 (2026-08-26) was code 20001, a content error, and still fails. `run_debug` now: on exactly `Overwrite failed (400)` + `"code":"1001"`, rotate the `.uipx` SolutionId to a fresh uuid4 and retry ONCE (its own allowance, inside the existing deadline; `debug_budget()` untouched). Prints an INTERIM line naming the issue, both ids, and after success the server id of the new solution and the orphaned previous ones โ€” `cleanup_solutions.py` reads only the current id. Never fires on 20001, on 404, or on exit 0. Self-disabling: once Overwrite recovers the signature stops matching; delete the block then. Tests: +6 (rotation then import; rotate once only; 20001 untouched; no .uipx โ†’ plain failure; classifier table). test_flow_check + test_criterion_budgets: 830 passed. The full tree's 3 failures in test_same_ground_corpus are pre-existing on the campaign tip (new ixp tasks from main's rebase, not allowlisted). ๐Ÿค– Generated with Claude Code Co-Authored-By: [Claude](mailto:noreply@anthropic.com) Claude-Session: https://claude.ai/code/session_013cn8thdbhgRMqTehFhnAWW Co-authored-by: Claude Fable 5.1 --- .../uipath-maestro-flow/_shared/flow_check.py | 98 +++++++++++++++++++ .../_shared/test_flow_check.py | 91 +++++++++++++++++ 2 files changed, 189 insertions(+) diff --git a/tests/tasks/uipath-maestro-flow/_shared/flow_check.py b/tests/tasks/uipath-maestro-flow/_shared/flow_check.py index e993f109e6..f0e97d51ff 100644 --- a/tests/tasks/uipath-maestro-flow/_shared/flow_check.py +++ b/tests/tasks/uipath-maestro-flow/_shared/flow_check.py @@ -148,6 +148,77 @@ def _output_blob(result: subprocess.CompletedProcess) -> str: return f"{result.stdout}\n{result.stderr}".lower() +# INTERIM โ€” Studio Web `Solution/{id}/Overwrite` regression (UiPath/cli#3938). +# +# Since 2026-09-02 (last known good: 2026-09-01 07:30Z) Studio Web answers +# `400 {"code":"1001","message":"An argument had an invalid value."}` when a +# solution that EXISTS is overwritten. `uip maestro flow debug` imports the +# project on its first run and writes the server's solution id back into the +# `.uipx`, so every LATER debug of the same project is an overwrite โ€” and the +# CLI treats any non-404 Overwrite as fatal. Result: the second seeded case of +# every multi-case checker died at solution upload, in all three arms +# (orchestrator_paths, escalation_slack_alert, billing_invoice_lookup, +# decision, wiki_pageviews, canary). Reproduced on the host with two CLI +# vintages and with `uip solution upload --force`; a fresh bundled SolutionId +# imports as new and Completes (Slack message posted), so ONLY overwrite is +# broken. The only earlier archived Overwrite 400 (2026-08-26) was code 20001, a +# content error that must still fail. +# +# So: on exactly that signature, rotate the bundled SolutionId to a fresh uuid4 +# and retry ONCE โ€” the retry is a new import. Self-limiting: once Overwrite +# recovers the signature never fires and nothing here runs. Cost while it does: +# one extra tenant solution per affected case, which `cleanup_solutions.py` +# cannot see (it reads the `.uipx`'s CURRENT id) โ€” the ids are printed so they +# are not lost. The platform fix and a CLI-side fallback are asked for in the +# issue above; delete this block when Overwrite works again. +_OVERWRITE_STUCK_ISSUE = "UiPath/cli#3938" +_OVERWRITE_STUCK_MARKER = "overwrite failed (400)" +_OVERWRITE_STUCK_CODE = re.compile(r'\\?"code\\?":\s*\\?"1001\\?"') +_OVERWRITE_STUCK_ATTEMPTS = 2 # the failed overwrite + one import-as-new retry +_SOLUTION_ID_RE = re.compile(r'("SolutionId"\s*:\s*")([0-9a-fA-F-]{36})(")') + + +def _is_overwrite_stuck(result: subprocess.CompletedProcess) -> bool: + """True iff ``flow debug`` died on the Studio Web Overwrite 400/1001 regression + (see :data:`_OVERWRITE_STUCK_ISSUE`) โ€” never on any other overwrite failure.""" + if result.returncode == 0: + return False + blob = _output_blob(result) + return _OVERWRITE_STUCK_MARKER in blob and _OVERWRITE_STUCK_CODE.search(blob) is not None + + +def _rotate_solution_id(project_dir: str) -> tuple[str, str] | None: + """Give the project's solution a fresh bundled ``SolutionId`` so the next + ``flow debug`` imports it as a NEW Studio Web solution instead of overwriting. + + Walks up from ``project_dir`` to the nearest ancestor holding exactly one + ``*.uipx`` (a JSON text file; ``uip solution init`` writes it beside the + project directory). Returns ``(old, new)``, or ``None`` when no unambiguous + ``.uipx`` is found โ€” the caller then fails the plain way.""" + import uuid + + here = os.path.abspath(project_dir) + for _ in range(4): + uipx = sorted(glob.glob(os.path.join(glob.escape(here), "*.uipx"))) + if len(uipx) == 1: + try: + text = open(uipx[0], encoding="utf-8").read() + except OSError: + return None + match = _SOLUTION_ID_RE.search(text) + if match is None: + return None + new = str(uuid.uuid4()) + with open(uipx[0], "w", encoding="utf-8") as handle: + handle.write(text[: match.start(2)] + new + text[match.end(2):]) + return match.group(2), new + parent = os.path.dirname(here) + if parent == here: + break + here = parent + return None + + def _is_transient_debug_error(result: subprocess.CompletedProcess) -> bool: """True iff a failed ``flow debug`` invocation looks like a transient server-side error (5xx / RetryLater / poll-budget expiry) worth retrying, @@ -326,6 +397,8 @@ def run_debug( unreadable: str | None = None unreadable_attempts = 0 + overwrite_rotations = 0 + rotated_solution_ids: list[str] = [] # `max_attempts` starts at the transient allowance and is extended by one # for an unreadable-outputs retry (see _VARIABLES_UNREADABLE_ATTEMPTS): that # retry has its own allowance, never the 5xx/RetryLater one. The deadline @@ -370,6 +443,23 @@ def run_debug( if unreadable_attempts >= _VARIABLES_UNREADABLE_ATTEMPTS: break max_attempts = max(max_attempts, attempt + 2) + elif _is_overwrite_stuck(r) and overwrite_rotations < _OVERWRITE_STUCK_ATTEMPTS - 1: + # INTERIM (see _OVERWRITE_STUCK_ISSUE): its own single allowance, like + # the unreadable-outputs retry; the deadline below still bounds it. + rotated = _rotate_solution_id(project_dir) + if rotated is None: + break + overwrite_rotations += 1 + rotated_solution_ids.append(rotated[0]) + print( + f"INTERIM ({_OVERWRITE_STUCK_ISSUE}: Studio Web Overwrite โ†’ 400/1001 for an " + f"existing solution since 2026-09-02): rotated the bundled SolutionId " + f"{rotated[0]} โ†’ {rotated[1]}; retrying as a new import. The solution " + f"{rotated[0]} stays on the tenant (cleanup reads only the current id).", + file=sys.stderr, + flush=True, + ) + max_attempts = max(max_attempts, attempt + 2) elif not _is_transient_debug_error(r): break elif _is_poll_timeout(r) and attempt + 1 >= _POLL_TIMEOUT_ATTEMPTS: @@ -398,6 +488,14 @@ def run_debug( if data is None: _fail(f"Could not parse JSON from flow debug\n{r.stdout}") payload = _get_ci(data, "Data") or {} + if rotated_solution_ids: + print( + f"INTERIM ({_OVERWRITE_STUCK_ISSUE}): debug ran as a new import; Studio Web " + f"solution {_get_ci(payload, 'solutionId', 'SolutionId')} (previous: " + f"{', '.join(rotated_solution_ids)}).", + file=sys.stderr, + flush=True, + ) status = _get_ci(payload, "finalStatus", "FinalStatus") if status != "Completed": _fail(f"Flow did not complete (finalStatus={status})\n{r.stdout}" + _incident_details(data)) diff --git a/tests/tasks/uipath-maestro-flow/_shared/test_flow_check.py b/tests/tasks/uipath-maestro-flow/_shared/test_flow_check.py index 2d6e77b28e..8cb48e7409 100644 --- a/tests/tasks/uipath-maestro-flow/_shared/test_flow_check.py +++ b/tests/tasks/uipath-maestro-flow/_shared/test_flow_check.py @@ -788,6 +788,23 @@ def test_collect_outputs_pascalcase_matches_camelcase(): ) +# Verbatim from `uip maestro flow debug` on 2026-09-02 07:48Z: Studio Web's +# Overwrite answering 400/1001 for an EXISTING solution (UiPath/cli#3938). +_OVERWRITE_400_1001 = ( + '{\n "Result": "Failure",\n' + ' "Message": "Overwrite failed (400): {\\"code\\":\\"1001\\",\\"message\\":' + '\\"An argument had an invalid value.\\",\\"translatedMessage\\":null}",\n' + ' "Instructions": "Check that the flow project is valid, the selected folder ' + 'is accessible, and Studio Web debug is available, then retry.",\n' + ' "ErrorCode": "unknown_error",\n "Retry": "RetryWillNotFix"\n}' +) +# The 2026-08-26 shape: a real content error that must NOT be rotated around. +_OVERWRITE_400_20001 = _OVERWRITE_400_1001.replace('1001', '20001').replace( + 'An argument had an invalid value.', + 'Archive is missing project directories referenced by solution metadata: [X/temp/project.uiproj].', +) + + def _cp(returncode, stdout="", stderr=""): return subprocess.CompletedProcess( args=["uip", "maestro", "flow", "debug"], returncode=returncode, stdout=stdout, stderr=stderr @@ -1029,6 +1046,80 @@ def test_run_debug_does_not_retry_present_but_empty_variables(monkeypatch): assert flow_check.collect_outputs(payload) == [] +def _uipx_project(tmp_path, solution_id="79cda3cb-5a10-4f37-e091-08df08c2afbe"): + """`uip solution init` layout: `/.uipx` beside `//`.""" + sol = tmp_path / "Sol" + proj = sol / "Proj" + proj.mkdir(parents=True) + (sol / "Sol.uipx").write_text( + '{\n "SolutionId": "%s",\n "Name": "Sol",\n "Projects": []\n}\n' % solution_id + ) + return sol, proj + + +def test_run_debug_rotates_the_solution_id_on_overwrite_1001_then_imports(monkeypatch, tmp_path, capsys): + """INTERIM (UiPath/cli#3938): the Studio Web Overwrite 400/1001 regression is + met with ONE retry as a new import โ€” the bundled SolutionId is rotated first.""" + sol, proj = _uipx_project(tmp_path) + calls = _stub_debug(monkeypatch, [_cp(1, _OVERWRITE_400_1001), _cp(0, _COMPLETED)]) + monkeypatch.setattr(flow_check, "_find_project", lambda pattern: str(proj)) + payload = run_debug(retries=1) + assert calls["n"] == 2 + assert flow_check.collect_outputs(payload) == ["Sev1"] + text = (sol / "Sol.uipx").read_text() + assert "79cda3cb-5a10-4f37-e091-08df08c2afbe" not in text + assert flow_check._SOLUTION_ID_RE.search(text) is not None + err = capsys.readouterr().err + assert "INTERIM (UiPath/cli#3938" in err + assert "rotated the bundled SolutionId 79cda3cb-5a10-4f37-e091-08df08c2afbe" in err + # The orphaned server id is named, because cleanup reads only the current one. + assert "previous: 79cda3cb-5a10-4f37-e091-08df08c2afbe" in err + + +def test_run_debug_rotates_only_once(monkeypatch, tmp_path): + _, proj = _uipx_project(tmp_path) + calls = _stub_debug(monkeypatch, [_cp(1, _OVERWRITE_400_1001), _cp(1, _OVERWRITE_400_1001)]) + monkeypatch.setattr(flow_check, "_find_project", lambda pattern: str(proj)) + with pytest.raises(SystemExit) as exc: + run_debug(retries=1) + assert calls["n"] == 2 + assert "Overwrite failed (400)" in str(exc.value) + + +def test_run_debug_does_not_rotate_on_a_content_overwrite_error(monkeypatch, tmp_path): + """Code 20001 (2026-08-26) is the archive being wrong; rotating would hide it.""" + sol, proj = _uipx_project(tmp_path) + calls = _stub_debug(monkeypatch, [_cp(1, _OVERWRITE_400_20001)]) + monkeypatch.setattr(flow_check, "_find_project", lambda pattern: str(proj)) + with pytest.raises(SystemExit): + run_debug(retries=1) + assert calls["n"] == 1 + assert "79cda3cb-5a10-4f37-e091-08df08c2afbe" in (sol / "Sol.uipx").read_text() + + +def test_run_debug_overwrite_1001_without_a_uipx_fails_plainly(monkeypatch, tmp_path): + proj = tmp_path / "Proj" + proj.mkdir() + calls = _stub_debug(monkeypatch, [_cp(1, _OVERWRITE_400_1001)]) + monkeypatch.setattr(flow_check, "_find_project", lambda pattern: str(proj)) + with pytest.raises(SystemExit): + run_debug(retries=1) + assert calls["n"] == 1 + + +@pytest.mark.parametrize( + "cp,expected", + [ + (_cp(1, _OVERWRITE_400_1001), True), + (_cp(1, _OVERWRITE_400_20001), False), + (_cp(0, _OVERWRITE_400_1001), False), + (_cp(1, _TRANSIENT_504), False), + ], +) +def test_is_overwrite_stuck(cp, expected): + assert flow_check._is_overwrite_stuck(cp) is expected + + def test_run_debug_unreadable_retry_does_not_burn_the_transient_budget(monkeypatch): """The unreadable-outputs retry is its own budget (2), independent of the 3 attempts reserved for 5xx/RetryLater.""" From d41fafb4cc5d484405eb6807c5d64963ac71f211 Mon Sep 17 00:00:00 2001 From: Tao Ma <51425734+tmatup@users.noreply.github.com> Date: Wed, 2 Sep 2026 01:36:18 -0700 Subject: [PATCH 42/52] fix(flow-check): close the .uipx read handle in the interim SolutionId rotation (#2990 review nit) (#2991) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ๐Ÿค– Generated with Claude Code Co-Authored-By: [Claude](mailto:noreply@anthropic.com) Claude-Session: https://claude.ai/code/session_013cn8thdbhgRMqTehFhnAWW Co-authored-by: Claude Fable 5.1 --- tests/tasks/uipath-maestro-flow/_shared/flow_check.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/tasks/uipath-maestro-flow/_shared/flow_check.py b/tests/tasks/uipath-maestro-flow/_shared/flow_check.py index f0e97d51ff..d0e5df18ee 100644 --- a/tests/tasks/uipath-maestro-flow/_shared/flow_check.py +++ b/tests/tasks/uipath-maestro-flow/_shared/flow_check.py @@ -202,7 +202,8 @@ def _rotate_solution_id(project_dir: str) -> tuple[str, str] | None: uipx = sorted(glob.glob(os.path.join(glob.escape(here), "*.uipx"))) if len(uipx) == 1: try: - text = open(uipx[0], encoding="utf-8").read() + with open(uipx[0], encoding="utf-8") as handle: + text = handle.read() except OSError: return None match = _SOLUTION_ID_RE.search(text) From 6fcc3ecb1e595a3c610b57c6e8190761ae6decce Mon Sep 17 00:00:00 2001 From: Tao Ma <51425734+tmatup@users.noreply.github.com> Date: Wed, 2 Sep 2026 02:38:36 -0700 Subject: [PATCH 43/52] fix(eval-prompt): name the graded flow artifact in local_crud (SmokeEval.flow) (#2992) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The same-ground sweep (#2752) rewrote this prompt loop-neutrally and, with the layout coaching, dropped every mention of the graded artifact: main named `SmokeEval.flow` three times, the campaign prompt zero. The graded criterion is `flow_contains.py --flow-name SmokeEval`. On 2026-09-01 the SDK-arm agent named its solution, project and flow `Smoke` (the only explicit-name miss in 918 graded --flow-name criteria; 1 of 6 SDK-arm runs, 0 of 6 v1 runs, same prompt and model; reasoning hidden). One loop-neutral clause restores the anchor; no layout, CLI or SDK coaching. ๐Ÿค– Generated with Claude Code Co-Authored-By: [Claude](mailto:noreply@anthropic.com) Claude-Session: https://claude.ai/code/session_01DjcoqnpHbNPvfCBsp3gB9V Co-authored-by: Claude Fable 5.1 --- tests/tasks/uipath-maestro-flow/evaluate/local_crud.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tasks/uipath-maestro-flow/evaluate/local_crud.yaml b/tests/tasks/uipath-maestro-flow/evaluate/local_crud.yaml index 10f1f6c09d..9b180a872b 100644 --- a/tests/tasks/uipath-maestro-flow/evaluate/local_crud.yaml +++ b/tests/tasks/uipath-maestro-flow/evaluate/local_crud.yaml @@ -21,7 +21,7 @@ initial_prompt: | Web round-trip. Create the solution and the Flow project, keeping the Flow inside a solution - of the same name. + of the same name; the flow file is `SmokeEval.flow`. Declare a string input variable named `name` before adding data points. The eval data point below uses `{"name":"Alice"}`, so its input must be declared From f2fe2c295bd4e0cbf00efe76e9f7bf800a1ef016 Mon Sep 17 00:00:00 2001 From: Tao Ma <51425734+tmatup@users.noreply.github.com> Date: Wed, 2 Sep 2026 10:33:52 -0700 Subject: [PATCH 44/52] =?UTF-8?q?fix(eval-prompt):=20routing=20tasks=20say?= =?UTF-8?q?=20where=20the=20deliverable=20goes=20=E2=80=94=20the=20current?= =?UTF-8?q?=20working=20directory,=20not=20a=20parent=20(#3012)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The v1 prompt had "Work inside the stated working directory โ€” do NOT `cd` to a parent." The campaign sweep dropped it with the rest of the "Important:" block, and the 09-01 restoration of the scope clause brought back the artifact ("the `.flow` file") but not its location. On 2026-09-02 the live-skill (v1, luna) agent on `ixp-routing-negative/teams-decision` ran `uip solution init` one directory above its row sandbox, wrote a correct `TeamsDecisionSolution/TeamsDecision/TeamsDecision.flow` there, and the checker (cwd = the row sandbox) found no `.flow`: 0.00. Same class as `ixp-routing/forms-classify` on 08-24 (2 of 495 archived fanned rows). Loop-neutral wording, both routing YAMLs; no checker or run-limit change. Claude-Session: https://claude.ai/code/session_01JizSrmrTUbJdHz8xKt65ks Co-authored-by: Claude Fable 5.1 --- tests/tasks/uipath-maestro-flow/ixp/routing.yaml | 3 ++- tests/tasks/uipath-maestro-flow/ixp/routing_negative.yaml | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/tasks/uipath-maestro-flow/ixp/routing.yaml b/tests/tasks/uipath-maestro-flow/ixp/routing.yaml index 033a54d279..a5c92eb7a1 100644 --- a/tests/tasks/uipath-maestro-flow/ixp/routing.yaml +++ b/tests/tasks/uipath-maestro-flow/ixp/routing.yaml @@ -61,7 +61,8 @@ initial_prompt: | confirmation, or missing details; pick a sensible default and state it in your final report. Do not iterate on validation errors to make the flow runnable โ€” a broken skeleton is acceptable, as long as the `.flow` file has - been produced. + been produced. Work inside the current working directory: create the + solution and project here, never in a parent directory. success_criteria: # What this checks: did the agent search the registry for IxP nodes? diff --git a/tests/tasks/uipath-maestro-flow/ixp/routing_negative.yaml b/tests/tasks/uipath-maestro-flow/ixp/routing_negative.yaml index 7286b450a5..f06f275792 100644 --- a/tests/tasks/uipath-maestro-flow/ixp/routing_negative.yaml +++ b/tests/tasks/uipath-maestro-flow/ixp/routing_negative.yaml @@ -66,7 +66,8 @@ initial_prompt: | confirmation, or missing details; pick a sensible default and state it in your final report. Do not iterate on validation errors to make the flow runnable โ€” a broken skeleton is acceptable, as long as the `.flow` file has - been produced. + been produced. Work inside the current working directory: create the + solution and project here, never in a parent directory. success_criteria: # Inverted twin of the positive routing.yaml artifact check. From 410415216c760aa89828f09d1a37a85a7f2231fe Mon Sep 17 00:00:00 2001 From: Tao Ma <51425734+tmatup@users.noreply.github.com> Date: Wed, 2 Sep 2026 12:54:17 -0700 Subject: [PATCH 45/52] test(maestro-flow): keep the corpus guard exact after the 2026-09-02 rebase onto main (#3016) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The rebase of campaign/same-ground-expansion onto main (0afe5c1ad) brought in two uipath-maestro-flow tasks that the alignment sweep never saw, and the permanent corpus guard (test_same_ground_corpus.py) went red on four contracts: - evaluate/child_simulation/child_simulation_crud.yaml (#2965, today) gates on `uip solution init` / `uip maestro flow init` telemetry โ€” v1-only authoring grammar. - ixp/e2e_03_project_creation_handoff (#2809, inherited on the previous rebase) gates on two skill_triggered criteria, tells the agent "do not run `uip maestro flow debug`", and its grader imported `flow_check` bare from a `../../_shared` sys.path entry. Carry the two tasks in the temporary allowlists, named with their upstream PR, so the guard stays exact (no silent widening) until each gets its own alignment pass. Fix the one contract that has no allowlist by design: handoff.py now uses the package-qualified `from _shared.flow_check import โ€ฆ` form every converted grader uses, which also resolves in coder-eval's isolated task mount where `_shared` is co-located rather than a sibling of `$TASK_DIR/..`. Verification: `pytest tests/tasks/uipath-maestro-flow/` 1140 passed (was 4 failed); e2e_03's own test_handoff.py 51 passed; handoff.py imports standalone. ๐Ÿค– Generated with Claude Code Claude-Session: https://claude.ai/code/session_01VwTyMj73xjaUbCMsADvTSr Co-authored-by: Claude Fable 5.1 --- .../_shared/test_same_ground_corpus.py | 24 ++++++++++++++----- .../handoff.py | 4 ++-- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/tests/tasks/uipath-maestro-flow/_shared/test_same_ground_corpus.py b/tests/tasks/uipath-maestro-flow/_shared/test_same_ground_corpus.py index 8a5b5902ee..8fbf4a4a67 100644 --- a/tests/tasks/uipath-maestro-flow/_shared/test_same_ground_corpus.py +++ b/tests/tasks/uipath-maestro-flow/_shared/test_same_ground_corpus.py @@ -1,8 +1,9 @@ """Permanent same-ground contract for uipath-maestro-flow eval tasks. -The temporary allowlists name pre-campaign debt. Each family batch removes its -own entries as the task contract becomes neutral; exact equality prevents stale -allowlist entries from hiding completed work. +The temporary allowlists name pre-campaign debt plus tasks that landed on +`main` after the alignment sweep and have not been aligned yet. Each family +batch removes its own entries as the task contract becomes neutral; exact +equality prevents stale allowlist entries from hiding completed work. """ from __future__ import annotations @@ -12,11 +13,22 @@ FLOW_TASKS = Path(__file__).resolve().parent.parent -V1_AUTHORING_ALLOWLIST = set() +# Post-sweep upstream arrivals, carried as-is until their own alignment pass: +# - evaluate/child_simulation (#2965, 2026-09-02) gates on `uip solution init` +# and `uip maestro flow init` telemetry. +# - ixp/e2e_03_project_creation_handoff (#2809, 2026-09-02) gates on two +# skill_triggered criteria and tells the agent not to run `flow debug`. +V1_AUTHORING_ALLOWLIST = { + "evaluate/child_simulation/child_simulation_crud.yaml", +} -SKILL_TELEMETRY_ALLOWLIST = set() +SKILL_TELEMETRY_ALLOWLIST = { + "ixp/e2e_03_project_creation_handoff/e2e_03_project_creation_handoff.yaml", +} -FORBIDDEN_PROMPT_ALLOWLIST = set() +FORBIDDEN_PROMPT_ALLOWLIST = { + "ixp/e2e_03_project_creation_handoff/e2e_03_project_creation_handoff.yaml", +} # These born-neutral escalation tasks are outside the recipe-edit sweep. Their # prompts already require the same-name solution and tell the agent to leave diff --git a/tests/tasks/uipath-maestro-flow/ixp/e2e_03_project_creation_handoff/handoff.py b/tests/tasks/uipath-maestro-flow/ixp/e2e_03_project_creation_handoff/handoff.py index 32f5180fa6..66552b68b1 100644 --- a/tests/tasks/uipath-maestro-flow/ixp/e2e_03_project_creation_handoff/handoff.py +++ b/tests/tasks/uipath-maestro-flow/ixp/e2e_03_project_creation_handoff/handoff.py @@ -83,9 +83,9 @@ from typing import Any TASK_DIR = os.path.dirname(os.path.abspath(__file__)) -sys.path.insert(0, os.path.join(TASK_DIR, "..", "..", "_shared")) +sys.path.insert(0, os.path.join(TASK_DIR, "..", "..")) -from flow_check import find_project_dir # noqa: E402 +from _shared.flow_check import find_project_dir # noqa: E402 # โ”€โ”€ shared helpers โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ From 1cd937210d495df725ed8c4a066ba0827e99eaf9 Mon Sep 17 00:00:00 2001 From: Tao Ma <51425734+tmatup@users.noreply.github.com> Date: Wed, 2 Sep 2026 16:46:16 -0700 Subject: [PATCH 46/52] fix(uipath-maestro-flow): route if/otherwise to a Decision, name the Decision's outputs, add the move-a-node recipe, refuse Merge after a Decision (#3024) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four one-row/one-sentence edits in the docs the failing v1 (luna) agents actually opened, from the 2026-09-02 v1-only batch RCA (skill-flow-decision, skill-flow-move-node): - greenfield.md "Select the node type" (opened by every decision run; the two failing runs never reached plugins/decision/*): a two-outcome "if โ€ฆ otherwise โ€ฆ" requirement is a core.logic.decision node, not a Script ternary. Terra's baseline already builds the Decision, so the sentence cannot steer it to a worse shape. - plugins/decision/impl.md: an "Outputs" section โ€” the Decision has no `.output`; readers use `$vars..matchedCaseId` / `.matchedCase` (registry outputDefinition). Four v1 runs and one nightly wrote `$vars..output.matchedCaseId` (โ†’ [300501]). Plus a Debug row for [400302] when the Decision's `expression` reads a node that runs after it (two v1 runs, after moving the Decision). - brownfield.md "Common edits": a "Move / reorder a node" row โ€” re-point every `$vars.` read in the moved node to a node that still runs before it. - plugins/merge/planning.md heuristics: joining ONE Decision's branches with a Merge hangs the run; wire both ports into the next node (validate refuses the shape since UiPath/cli#3930). .maintenance/check-all.sh: 78 files, 26 plugins, 13,442 lines, 0 violations. No prompt, checker, criterion, weight or run_limits change. Claude-Session: https://claude.ai/code/session_01ENT7zRXbZyvku1y9BNC9TH Co-authored-by: Claude Fable 5.1 --- skills/uipath-maestro-flow/references/author/brownfield.md | 1 + skills/uipath-maestro-flow/references/author/greenfield.md | 2 ++ .../references/author/plugins/decision/impl.md | 5 +++++ .../references/author/plugins/merge/planning.md | 1 + 4 files changed, 9 insertions(+) diff --git a/skills/uipath-maestro-flow/references/author/brownfield.md b/skills/uipath-maestro-flow/references/author/brownfield.md index 707203f2a1..4ee1ceabf0 100644 --- a/skills/uipath-maestro-flow/references/author/brownfield.md +++ b/skills/uipath-maestro-flow/references/author/brownfield.md @@ -33,6 +33,7 @@ For edits touching multiple top-level arrays, follow [parallel same-file Edit ru |---|---| | **Change a script body or node inputs** | Use `Edit` on `inputs`; do not delete/re-add because node IDs and `$vars` expressions must remain stable. Script nodes must return an object (`return { key: value }`). See [Edit/Write: Update node inputs](editing-operations-json.md#update-node-inputs). | | **Add a node between two existing nodes** | Remove the connecting edge; add the node; wire upstream โ†’ new โ†’ downstream. See [Edit/Write: Insert a node](editing-operations-json.md#insert-a-node-between-two-existing-nodes). | +| **Move / reorder a node** | Re-wire the edges, then re-point every `$vars.` read inside the moved node (a Decision's `expression`, a script body, input fields) to a node that still runs **before** it; a reference to a node that now runs later evaluates against nothing at runtime (`[400302]`/`[400300]`) even though `flow validate` passes. Nodes the branches rejoin at read the Decision as `$vars..matchedCaseId` (see [decision/impl.md โ€” Outputs](plugins/decision/impl.md#outputs)). See [Edit/Write: Delete an edge](editing-operations-json.md#delete-an-edge) and [Edit/Write: Update node inputs](editing-operations-json.md#update-node-inputs). | | **Add a branch (decision node)** | Remove an edge; add the decision; wire true/false branches. See [Edit/Write: Insert a decision branch](editing-operations-json.md#insert-a-decision-branch). | | **Remove a node** | Remove the node; sweep edges/definitions/variables; reconnect upstream to downstream. See [Edit/Write: Remove a node](editing-operations-json.md#remove-a-node-and-reconnect). | | **Remove an edge** | Find and remove its edge ID. See [Edit/Write: Delete an edge](editing-operations-json.md#delete-an-edge). | diff --git a/skills/uipath-maestro-flow/references/author/greenfield.md b/skills/uipath-maestro-flow/references/author/greenfield.md index 9f2d049dc1..4e53c2049e 100644 --- a/skills/uipath-maestro-flow/references/author/greenfield.md +++ b/skills/uipath-maestro-flow/references/author/greenfield.md @@ -238,6 +238,8 @@ Then pick the first match down this ladder: Manual HTTP is the **bottom of the ladder** โ€” only the search returning no connector authorizes it. Picking it without searching is the brand-name shortcut forbidden by [SKILL.md rule #3](../../SKILL.md#critical-rules-universal). +**Branching is a node, not a Script.** A requirement phrased as *if โ€ฆ otherwise โ€ฆ* (two outcomes) is a `core.logic.decision` node with both `true`/`false` ports wired ([decision/planning.md](plugins/decision/planning.md)); three or more outcomes โ†’ `core.logic.switch`. Do not fold the branch into a Script ternary โ€” the branch must exist as a node in the graph. + ### Document-extraction step โ€” route it to IxP (runs even when full planning is skipped) Pulling **named fields out of documents** (PDFs, scans, receipts, invoices, contracts, forms) is a document-extraction step โ€” its node is an **IxP node** (`uipath.ixp.*`). It's easy to miss because the fetch/post steps around it are ordinary nodes, but the extraction in between is its own node. **Always land a node for it โ€” never leave it out:** diff --git a/skills/uipath-maestro-flow/references/author/plugins/decision/impl.md b/skills/uipath-maestro-flow/references/author/plugins/decision/impl.md index 77be35d290..3afeef1f27 100644 --- a/skills/uipath-maestro-flow/references/author/plugins/decision/impl.md +++ b/skills/uipath-maestro-flow/references/author/plugins/decision/impl.md @@ -53,10 +53,15 @@ $vars.lookupUser.output.user !== null Output ports: `true` and `false`. Both branches must be wired. See [editing-operations.md](../../editing-operations.md) for edge add procedures. +## Outputs + +A Decision has **no `.output`**. Downstream nodes read which branch ran as `$vars..matchedCaseId` (`"true"` or `"false"`) or `$vars..matchedCase` (the branch label) โ€” e.g. `$vars.checkStatus.matchedCaseId === "true"`. `$vars..output.matchedCaseId` is undefined and faults the reader at runtime (`[300501]`). + ## Debug | Error | Cause | Fix | | --- | --- | --- | | Expression does not evaluate to boolean | Expression returns non-boolean value | Ensure expression uses comparison operators (`===`, `>`, etc.) | | `$vars.nodeId` is undefined | Upstream node not connected or wrong ID | Check edges and node IDs | +| `[400302] Error evaluating outgoing flow expression from gateway` | `expression` reads a node that runs *after* the Decision (typical after moving the Decision earlier in the flow) | Point `expression` at a node that still runs before the Decision (e.g. the HTTP node's `$vars..outputโ€ฆ`), not at the node the branches lead to | | Only one branch wired | Missing true or false edge | Add the missing edge โ€” both branches are required | diff --git a/skills/uipath-maestro-flow/references/author/plugins/merge/planning.md b/skills/uipath-maestro-flow/references/author/plugins/merge/planning.md index 374957dfb9..aaf580036e 100644 --- a/skills/uipath-maestro-flow/references/author/plugins/merge/planning.md +++ b/skills/uipath-maestro-flow/references/author/plugins/merge/planning.md @@ -10,6 +10,7 @@ Use a Merge node to synchronize two or more parallel branches before continuing. - Use it when parallel branches must rejoin or after one node forks into multiple downstream branches. - Do not use it for sequential pipelines; wire nodes directly. +- Do not use it to join the `true`/`false` branches of one Decision (or the cases of one Switch); wire both ports straight into the next node, which accepts several incoming edges. A Merge waits for ALL inputs and a Decision emits exactly one, so the run would hang forever; `flow validate` refuses this shape. ## Ports From b792074fc9469b428101f9a767809c648a87759f Mon Sep 17 00:00:00 2001 From: Tao Ma <51425734+tmatup@users.noreply.github.com> Date: Wed, 2 Sep 2026 17:07:31 -0700 Subject: [PATCH 47/52] fix(flow-check): INTERIM Overwrite rotation also recognises the CLI's staged error envelope (UiPath/cli#3951) (#3026) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Studio Web Overwrite 400/1001 regression (UiPath/cli#3938) is met by the campaign checker with one SolutionId rotation + import-as-new retry. The detector matched only the legacy text `Overwrite failed (400): {"code":"1001",โ€ฆ}`. Since UiPath/cli#3951 (main 2026-09-02, `016231474`) the CLI reports the refusal against its stage: Message: "Failed during overwrite-solution: HTTP 400 on POST โ€ฆ/Overwrite โ€” An argument had an invalid value." Context: {"HttpStatus": 400, "Stage": "overwrite-solution", "ErrorCode": "1001"} With an image whose CLI carries #3951, every second debug of a project failed with a plain `flow debug exit 1` and no rotation โ€” seen 2026-09-03 on every multi-debug task of the batch-A rerun (decision, wiki_pageviews), BOTH arms. The nightly will hit the same as soon as its image picks up main โ‰ฅ #3951. `_is_overwrite_stuck` now also matches the staged envelope (stage `overwrite-solution` + code 1001, or HTTP 400 with the platform's "invalid value" wording when the code is absent); another stage or another code never rotates. Tests: 4 new parametrized cases + a run_debug-level rotation on the staged envelope; _shared suite 980 passed; check-task-driver OK. No prompt, criterion, weight or run_limits change. Delete with the rest of the INTERIM block when Overwrite works again. Claude-Session: https://claude.ai/code/session_01ENT7zRXbZyvku1y9BNC9TH Co-authored-by: Claude Fable 5.1 --- .../uipath-maestro-flow/_shared/flow_check.py | 31 +++++++++++++- .../_shared/test_flow_check.py | 41 +++++++++++++++++++ 2 files changed, 70 insertions(+), 2 deletions(-) diff --git a/tests/tasks/uipath-maestro-flow/_shared/flow_check.py b/tests/tasks/uipath-maestro-flow/_shared/flow_check.py index d0e5df18ee..a541cf25c3 100644 --- a/tests/tasks/uipath-maestro-flow/_shared/flow_check.py +++ b/tests/tasks/uipath-maestro-flow/_shared/flow_check.py @@ -174,17 +174,44 @@ def _output_blob(result: subprocess.CompletedProcess) -> str: _OVERWRITE_STUCK_ISSUE = "UiPath/cli#3938" _OVERWRITE_STUCK_MARKER = "overwrite failed (400)" _OVERWRITE_STUCK_CODE = re.compile(r'\\?"code\\?":\s*\\?"1001\\?"') +# Since UiPath/cli#3951 (main 2026-09-02, `016231474`) the CLI reports a refused +# upload against its stage instead of echoing the raw body: +# Message: "Failed during overwrite-solution: HTTP 400 on POST โ€ฆ/Overwrite โ€” An argument had an invalid value." +# Context: {"HttpStatus": 400, "Stage": "overwrite-solution", "ErrorCode": "1001"} +# The rotation must recognise both envelopes: an image whose CLI carries #3951 +# but whose checker knows only the legacy text fails every second debug of a +# project with a plain `flow debug exit 1` (seen 2026-09-03 on every +# multi-debug task of the batch-A rerun, both arms). +_OVERWRITE_STUCK_STAGE = "overwrite-solution" _OVERWRITE_STUCK_ATTEMPTS = 2 # the failed overwrite + one import-as-new retry _SOLUTION_ID_RE = re.compile(r'("SolutionId"\s*:\s*")([0-9a-fA-F-]{36})(")') def _is_overwrite_stuck(result: subprocess.CompletedProcess) -> bool: """True iff ``flow debug`` died on the Studio Web Overwrite 400/1001 regression - (see :data:`_OVERWRITE_STUCK_ISSUE`) โ€” never on any other overwrite failure.""" + (see :data:`_OVERWRITE_STUCK_ISSUE`) โ€” never on any other overwrite failure. + + Matches the legacy envelope (``Overwrite failed (400): {"code":"1001",โ€ฆ}``) + and the staged one the CLI emits since UiPath/cli#3951 (``Context.Stage == + "overwrite-solution"`` with ``Context.ErrorCode == "1001"``, or HTTP 400 and + the platform's "invalid value" wording when the code is absent).""" if result.returncode == 0: return False blob = _output_blob(result) - return _OVERWRITE_STUCK_MARKER in blob and _OVERWRITE_STUCK_CODE.search(blob) is not None + if _OVERWRITE_STUCK_MARKER in blob and _OVERWRITE_STUCK_CODE.search(blob) is not None: + return True + data = _parse_json(result.stdout) + context = _get_ci(data or {}, "Context", default={}) + if not isinstance(context, dict): + return False + stage = _get_ci(context, "Stage") + if not isinstance(stage, str) or stage.lower() != _OVERWRITE_STUCK_STAGE: + return False + code = _get_ci(context, "ErrorCode") + if str(code) == "1001": + return True + http = _get_ci(context, "HttpStatus") + return http == 400 and "invalid value" in blob def _rotate_solution_id(project_dir: str) -> tuple[str, str] | None: diff --git a/tests/tasks/uipath-maestro-flow/_shared/test_flow_check.py b/tests/tasks/uipath-maestro-flow/_shared/test_flow_check.py index 8cb48e7409..074200ceed 100644 --- a/tests/tasks/uipath-maestro-flow/_shared/test_flow_check.py +++ b/tests/tasks/uipath-maestro-flow/_shared/test_flow_check.py @@ -799,6 +799,28 @@ def test_collect_outputs_pascalcase_matches_camelcase(): ' "ErrorCode": "unknown_error",\n "Retry": "RetryWillNotFix"\n}' ) # The 2026-08-26 shape: a real content error that must NOT be rotated around. +# The staged envelope the CLI emits since UiPath/cli#3951 (main 2026-09-02): the +# refusal is reported against its stage, the raw body is no longer echoed. +_OVERWRITE_STAGED_400_1001 = ( + '{\n "Result": "Failure",\n' + ' "Message": "Failed during overwrite-solution: HTTP 400 on POST ' + '/codereval/studio_/backend/api/Solution/5ebc2eff-4eab-45a5-df6a-08df093139b4/Overwrite ' + '\u2014 An argument had an invalid value.",\n' + ' "Instructions": "Studio Web refused to replace the contents of solution 5ebc2eff-4eab-45a5-df6a-08df093139b4. ' + 'Nothing was uploaded and nothing ran, so the flow is not the cause.",\n' + ' "Context": {"HttpStatus": 400, "Stage": "overwrite-solution", ' + '"Endpoint": "/codereval/studio_/backend/api/Solution/5ebc2eff-4eab-45a5-df6a-08df093139b4/Overwrite", ' + '"Method": "POST", "ErrorCode": "1001"},\n' + ' "ErrorCode": "invalid_argument",\n "Retry": "RetryWillNotFix"\n}' +) +# Same stage, a different platform code: a content refusal, never rotated. +_OVERWRITE_STAGED_400_20001 = _OVERWRITE_STAGED_400_1001.replace('"ErrorCode": "1001"', '"ErrorCode": "20001"').replace( + 'An argument had an invalid value.', + 'Archive is missing project directories referenced by solution metadata.', +) +# Another stage with the same code must not rotate either. +_UPLOAD_STAGED_400_1001 = _OVERWRITE_STAGED_400_1001.replace('overwrite-solution', 'upload-solution') + _OVERWRITE_400_20001 = _OVERWRITE_400_1001.replace('1001', '20001').replace( 'An argument had an invalid value.', 'Archive is missing project directories referenced by solution metadata: [X/temp/project.uiproj].', @@ -1076,6 +1098,20 @@ def test_run_debug_rotates_the_solution_id_on_overwrite_1001_then_imports(monkey assert "previous: 79cda3cb-5a10-4f37-e091-08df08c2afbe" in err +def test_run_debug_rotates_the_solution_id_on_the_staged_overwrite_envelope(monkeypatch, tmp_path, capsys): + """Same rotation, driven by the CLI's post-UiPath/cli#3951 envelope + (``Context.Stage == "overwrite-solution"``, ``Context.ErrorCode == "1001"``).""" + sol, proj = _uipx_project(tmp_path) + calls = _stub_debug(monkeypatch, [_cp(1, _OVERWRITE_STAGED_400_1001), _cp(0, _COMPLETED)]) + monkeypatch.setattr(flow_check, "_find_project", lambda pattern: str(proj)) + payload = run_debug(retries=1) + assert calls["n"] == 2 + assert flow_check.collect_outputs(payload) == ["Sev1"] + text = (sol / "Sol.uipx").read_text() + assert "79cda3cb-5a10-4f37-e091-08df08c2afbe" not in text + assert "INTERIM (UiPath/cli#3938" in capsys.readouterr().err + + def test_run_debug_rotates_only_once(monkeypatch, tmp_path): _, proj = _uipx_project(tmp_path) calls = _stub_debug(monkeypatch, [_cp(1, _OVERWRITE_400_1001), _cp(1, _OVERWRITE_400_1001)]) @@ -1114,6 +1150,11 @@ def test_run_debug_overwrite_1001_without_a_uipx_fails_plainly(monkeypatch, tmp_ (_cp(1, _OVERWRITE_400_20001), False), (_cp(0, _OVERWRITE_400_1001), False), (_cp(1, _TRANSIENT_504), False), + # UiPath/cli#3951 staged envelope + (_cp(1, _OVERWRITE_STAGED_400_1001), True), + (_cp(1, _OVERWRITE_STAGED_400_20001), False), + (_cp(1, _UPLOAD_STAGED_400_1001), False), + (_cp(0, _OVERWRITE_STAGED_400_1001), False), ], ) def test_is_overwrite_stuck(cp, expected): From 69cd5810ddbacdd2b9cb4b517819d097a0e4f910 Mon Sep 17 00:00:00 2001 From: Tao Ma <51425734+tmatup@users.noreply.github.com> Date: Wed, 2 Sep 2026 17:31:58 -0700 Subject: [PATCH 48/52] =?UTF-8?q?fix(uipath-maestro-flow):=20put=20the=20i?= =?UTF-8?q?f/otherwise=20=E2=86=92=20Decision=20routing=20where=20every=20?= =?UTF-8?q?greenfield=20read=20reaches=20it=20(#3027)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #3024 added the routing sentence under "Select the node type" โ€” line 241 of greenfield.md. The agents read the file with `sed -n '1,220p'` (34 of the 102 greenfield reads in the 09-02 run) or `'1,240p'` (17), and the first after-rerun of skill-flow-decision opened `1,220p`: the sentence was never in the model's context (verified 2026-09-03 from the transcript). One line in "Should you plan first?" (line 20, inside every observed read window) carries the same rule; the Step-3 paragraph stays for readers who get that far. .maintenance/check-all.sh green. No prompt, checker, criterion, weight or run_limits change. Claude-Session: https://claude.ai/code/session_01ENT7zRXbZyvku1y9BNC9TH Co-authored-by: Claude Fable 5.1 --- skills/uipath-maestro-flow/references/author/greenfield.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/skills/uipath-maestro-flow/references/author/greenfield.md b/skills/uipath-maestro-flow/references/author/greenfield.md index 4e53c2049e..ba7ea92f76 100644 --- a/skills/uipath-maestro-flow/references/author/greenfield.md +++ b/skills/uipath-maestro-flow/references/author/greenfield.md @@ -18,6 +18,8 @@ For complex flows, produce a plan before building. Reference [planning-arch.md]( - The flow is a straightforward linear pipeline (trigger โ†’ action โ†’ action โ†’ end) - The user has already described the exact topology they want +**Skipping the plan never skips the node choice.** Two outcomes (*if โ€ฆ otherwise โ€ฆ*) = a `core.logic.decision` node with both `true`/`false` ports wired โ€” never a Script ternary ([decision/planning.md](plugins/decision/planning.md)); three or more outcomes โ†’ `core.logic.switch`; a named external service โ†’ the Step 3 registry ladder below. + ### Examples **Plan:** "Build a flow that receives a Jira ticket, classifies it with an AI agent, routes urgent tickets to Slack and non-urgent to a queue, and logs everything to a Google Sheet." From a7cb7e3fcaaed3d5a78b475bdd37cfa13d69c9f7 Mon Sep 17 00:00:00 2001 From: Tao Ma <51425734+tmatup@users.noreply.github.com> Date: Thu, 3 Sep 2026 14:36:31 -0700 Subject: [PATCH 49/52] fix(maestro-flow): address #2756 review (pagination gate, rotation cleanup, node_modules prune) (#3045) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Revert "chore(codeowners): preview uipath-maestro-case owned by @UiPath/maestro-cli-team" This reverts commit a7868335f7d9fb01c8fe8661d3200941ca99515d. * fix(maestro-flow): address #2756 review โ€” pagination gate, rotation cleanup sidecar, node_modules prune Review items from UiPath/skills#2756 (rockymadden): - MAJOR 1 paginated_reference_lookup: the v1 paged-loop telemetry is advisory again at min_count 2; the gating criterion now requires "went past page 1" on a SUCCESSFUL command for either route (`โ€ฆ list โ€ฆ nextPage=` or `registry prepare โ€ฆ --resolve channel:`). Corpus test uses the real `npx flow-sdk registry prepare` form and adds negative cases. - MAJOR 3 Overwrite rotation: the rotated-away SolutionId is appended to a `.rotated-solution-ids` sidecar beside the `.uipx`; `cleanup_solutions.py` deletes those ids alongside the current one. Tests cover both halves. - MINOR 5 ceql_where / enhanced_enum prompts name neither `validate` nor `compile` (neither is graded there). - MINOR 7 drop the workspaces-only plan path from the 99 campaign headers; the three decisions stay inline. - MINOR 8 wiki_pageviews anchors operate.json to the discovered .bpmn's parent. - MINOR 9 cwd-rooted `**` scans in flow_check.py prune `node_modules` (the staged SDK symlink); test with a symlinked fixture tree. - MAJOR 2 SKILL.md prose casing `bpmn:ServiceTask` matches the template. - nits: stage_shared.sh guard compares realpaths; validate_flow.py dead branch removed (test updated); case_check.py long line wrapped; Dockerfile drops the unread FLOW_SDK_CONNECTORS_DIR plumbing (no consumer anywhere). - run-coder-eval.yml passes FLOW_SDK_VERSION from the repo variable (default `latest`). - Comment blocks over _OVERWRITE_STUCK / _VARIABLES_UNREADABLE trimmed; the evidence lives in UiPath/cli#3938 and #3929. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01UauCCMBpJ7ws1meQwBfYSS --------- Co-authored-by: Claude Fable 5.1 --- .github/workflows/run-coder-eval.yml | 1 + CODEOWNERS | 2 +- skills/uipath-maestro-bpmn/SKILL.md | 2 +- tests/docker/Dockerfile | 11 +-- tests/scripts/stage_shared.sh | 11 +-- .../wiki_pageviews/wiki_pageviews.yaml | 2 +- .../uipath-maestro-case/_shared/case_check.py | 5 +- .../_shared/cleanup_solutions.py | 47 +++++++--- .../uipath-maestro-flow/_shared/flow_check.py | 89 +++++++++---------- .../_shared/test_flow_check.py | 57 +++++++++++- .../_shared/test_same_ground_corpus.py | 31 ++++--- .../_shared/test_validate_flow.py | 13 ++- .../_shared/validate_flow.py | 5 +- .../bindings/idempotent_reconfigure.yaml | 1 - .../multi_connector_independence.yaml | 1 - .../no_duplicate_connection_bindings.yaml | 1 - .../reconfigure_different_connection.yaml | 1 - .../connector_features/ceql_where.yaml | 3 +- .../connector_features/complex_array.yaml | 1 - .../dtl_load_by_default_false.yaml | 1 - .../dtl_load_by_default_true.yaml | 1 - .../connector_features/enhanced_enum.yaml | 3 +- .../connector_features/enum.yaml | 1 - .../generic_dynamic_node.yaml | 1 - .../jdbc_databricks_query.yaml | 1 - .../connector_features/multiselect.yaml | 1 - .../non_catalog_http_fallback.yaml | 1 - .../paginated_reference_lookup.yaml | 24 +++-- .../connector_features/path_params.yaml | 1 - .../connector_features/query_params.yaml | 1 - .../connector_features/searchable_joins.yaml | 1 - .../slack_http_fallback.yaml | 1 - .../testmanager_attachments.yaml | 1 - .../testmanager_crud_grounded.yaml | 1 - .../testmanager_execution_results.yaml | 1 - .../testmanager_generic_records.yaml | 1 - .../testmanager_requirement_lifecycle.yaml | 1 - .../testmanager_testcase_lifecycle.yaml | 1 - .../testmanager_testset_lifecycle.yaml | 1 - .../trigger_with_filter.yaml | 1 - .../webhook_waitfor_parallel.yaml | 1 - .../batch_transform/batch_transform.yaml | 1 - .../summarize/summarize.yaml | 1 - .../e2e/devcon_expense_approval.yaml | 1 - .../jira_create_issue/jira_create_issue.yaml | 1 - .../e2e/jira_get_issue/jira_get_issue.yaml | 1 - .../e2e/jira_lifecycle/jira_lifecycle.yaml | 1 - .../jira_search_triage.yaml | 1 - .../edit/add_node/add_node.yaml | 1 - .../edit/add_output/add_output.yaml | 1 - .../group_to_subflow/group_to_subflow.yaml | 1 - .../edit/move_node/move_node.yaml | 1 - .../edit/remove_node/remove_node.yaml | 1 - .../edit/update_node/update_node.yaml | 1 - .../evaluate/evaluator_type_choice.yaml | 1 - .../inline_agent_eval/inline_agent_eval.yaml | 1 - .../evaluate/local_crud.yaml | 1 - .../evaluate/no_auto_upload.yaml | 1 - .../evaluate/simulation/simulation_crud.yaml | 1 - .../hitl/quality_01_schema_design.yaml | 1 - .../hitl/quality_02_result_downstream.yaml | 1 - .../hitl/quality_03_boolean_decision.yaml | 1 - .../hitl/quality_04_brownfield_insert.yaml | 1 - .../hitl/smoke_01_hitl_node_placed.yaml | 1 - .../hitl/smoke_02_completed_port_wired.yaml | 1 - .../hitl/smoke_03_multi_outcome_routing.yaml | 1 - .../bellevue_weather_simulated.yaml | 1 - .../cli_dice_roller_simulated.yaml | 1 - .../customer_escalation_simulated.yaml | 1 - .../customer_escalation_triage.yaml | 1 - .../expense_approval_simulated.yaml | 1 - .../hitl_schema_design_simulated.yaml | 1 - .../ixp_invoice_extraction_simulated.yaml | 1 - .../slack_channel_description_simulated.yaml | 1 - .../interactive/solution_select.yaml | 1 - .../ixp/e2e_02_project_selection.yaml | 1 - .../ixp/integration_handle_routing.yaml | 1 - .../uipath-maestro-flow/ixp/routing.yaml | 1 - .../ixp/routing_listing.yaml | 1 - .../ixp/routing_negative.yaml | 1 - .../ixp/scaffold_minimal.yaml | 1 - .../ixp/scaffold_multinode.yaml | 1 - .../bellevue_weather/bellevue_weather.yaml | 1 - .../multi_node/calculator/calculator.yaml | 1 - .../customer_escalation.yaml | 1 - .../multi_node/dice_roller/dice_roller.yaml | 1 - .../multi_node/feet_inches/feet_inches.yaml | 1 - .../loop_multiply/loop_multiply.yaml | 1 - .../multi_city_weather.yaml | 1 - .../multi_node/reading_list/reading_list.yaml | 1 - .../slack_channel_description.yaml | 1 - .../slack_weather_pipeline.yaml | 1 - .../wiki_pageviews/wiki_pageviews.yaml | 1 - .../api_workflow/api_workflow.yaml | 1 - .../single_node/decision/decision.yaml | 1 - .../single_node/delay/delay.yaml | 1 - .../file_attachment/file_attachment.yaml | 1 - .../lowcode_agent/lowcode_agent.yaml | 1 - .../openmeteo_weather/openmeteo_weather.yaml | 1 - .../outlook_trigger_inbox.yaml | 1 - .../outlook_waitfor_email.yaml | 1 - .../single_node/rpa/rpa.yaml | 1 - .../single_node/subflow/subflow.yaml | 1 - .../single_node/switch/switch.yaml | 1 - .../single_node/terminate/terminate.yaml | 1 - .../transform_filter/transform_filter.yaml | 1 - .../transform_group_by.yaml | 1 - .../transform_map/transform_map.yaml | 1 - .../smoke/init_validate.yaml | 1 - .../smoke/inline_agent_robust.yaml | 1 - .../smoke/merge_parallel_sync.yaml | 1 - .../smoke/registry_discovery.yaml | 1 - .../smoke/scheduled_trigger.yaml | 1 - 113 files changed, 199 insertions(+), 204 deletions(-) diff --git a/.github/workflows/run-coder-eval.yml b/.github/workflows/run-coder-eval.yml index dc60339b9b..67877d8595 100644 --- a/.github/workflows/run-coder-eval.yml +++ b/.github/workflows/run-coder-eval.yml @@ -333,6 +333,7 @@ jobs: --build-arg CODER_EVAL_IMAGE="ghcr.io/uipath/coder-eval-agent:${CE_IMAGE_TAG}" \ --secret id=npm_auth_token,env=NPM_AUTH_TOKEN \ --build-arg CLI_VERSION="${{ inputs.cli_version }}" \ + --build-arg FLOW_SDK_VERSION="${{ vars.FLOW_SDK_VERSION || 'latest' }}" \ -t skills-image:latest \ -f tests/docker/Dockerfile \ . diff --git a/CODEOWNERS b/CODEOWNERS index 73b7ce755b..ccb3a87ecb 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -19,7 +19,7 @@ # preview skills /preview/uipath-maestro-flow/ @UiPath/maestro-cli-team -/preview/uipath-maestro-case/ @UiPath/maestro-cli-team +/preview/uipath-maestro-case/ @dmetzgar @tmatup /preview/uipath-maestro-bpmn/ @UiPath/maestro-cli-team # Activity references (bundled inside uipath-rpa skill) diff --git a/skills/uipath-maestro-bpmn/SKILL.md b/skills/uipath-maestro-bpmn/SKILL.md index 3247fc88e5..6207b6d882 100644 --- a/skills/uipath-maestro-bpmn/SKILL.md +++ b/skills/uipath-maestro-bpmn/SKILL.md @@ -293,7 +293,7 @@ and honestly surfaced to the user as gaps when asked. `uipath:event` / `uipath:mapping` templates declare their type as a nested ``. Some runtime-authored templates use the payload's `type` attribute instead; notably, - `Orchestrator.StartAgentJob` is a direct child of `bpmn:serviceTask` with + `Orchestrator.StartAgentJob` is a direct child of `bpmn:ServiceTask` with ``. Both declarations are supported. Paste the selected registry template literally and do not normalize one form into the other. diff --git a/tests/docker/Dockerfile b/tests/docker/Dockerfile index 546f4976a1..ea7c343d7b 100644 --- a/tests/docker/Dockerfile +++ b/tests/docker/Dockerfile @@ -170,7 +170,6 @@ RUN cd "$PREVIEW_FLOW_SDK_ROOT" && \ uip maestro registry pull --force --output json && \ if [ -f "$UIP_MAESTRO_REGISTRY_HOME/current.json" ]; then \ registry_snapshot="$(jq -er '.snapshotPath' "$UIP_MAESTRO_REGISTRY_HOME/current.json")"; \ - registry_connectors="$registry_snapshot/connectors"; \ else \ library_json="$(readlink -f "$(uip maestro registry path --library-json)")" && \ registry_snapshot="$(dirname "$library_json")" && \ @@ -179,20 +178,16 @@ RUN cd "$PREVIEW_FLOW_SDK_ROOT" && \ mkdir -p "$UIP_MAESTRO_REGISTRY_HOME" && \ jq -n --arg libraryHash "$library_hash" --arg snapshotPath "$registry_snapshot" \ '{libraryHash: $libraryHash, snapshotPath: $snapshotPath}' \ - > "$UIP_MAESTRO_REGISTRY_HOME/current.json" && \ - registry_connectors="$PREVIEW_FLOW_SDK_ROOT/connectors" && \ - mkdir -p "$registry_connectors"; \ + > "$UIP_MAESTRO_REGISTRY_HOME/current.json"; \ fi && \ test -f "$registry_snapshot/library-json/index.json" && \ test -f "$registry_snapshot/library-md/index.json" && \ mkdir -p "$PREVIEW_FLOW_SDK_ASSETS_ROOT" && \ ln -s "$registry_snapshot/library-json" "$PREVIEW_FLOW_SDK_ASSETS_ROOT/library-json" && \ - ln -s "$registry_snapshot/library-md" "$PREVIEW_FLOW_SDK_ASSETS_ROOT/library-md" && \ - ln -s "$registry_connectors" "$PREVIEW_FLOW_SDK_ASSETS_ROOT/connectors" + ln -s "$registry_snapshot/library-md" "$PREVIEW_FLOW_SDK_ASSETS_ROOT/library-md" ENV FLOW_SDK_LIBRARY_JSON=/opt/preview-flow-sdk-assets/library-json \ - FLOW_SDK_LIBRARY_MD=/opt/preview-flow-sdk-assets/library-md \ - FLOW_SDK_CONNECTORS_DIR=/opt/preview-flow-sdk-assets/connectors + FLOW_SDK_LIBRARY_MD=/opt/preview-flow-sdk-assets/library-md # Prove that the installed SDK resolves without registry credentials and that # the product CLI can check real source against it. diff --git a/tests/scripts/stage_shared.sh b/tests/scripts/stage_shared.sh index cb98402ff1..916fc3e3c6 100755 --- a/tests/scripts/stage_shared.sh +++ b/tests/scripts/stage_shared.sh @@ -28,16 +28,17 @@ if [ -n "$manifest" ]; then fi search_root=${STAGE_SHARED_ROOT:-tests/tasks} -if [[ "$search_root" = "$PWD"/* ]]; then - search_root=${search_root#"$PWD"/} -fi -case "$search_root" in - tests/tasks | tests/tasks/*) ;; +# Compare resolved paths so `tests/tasks/../../etc` cannot pass the guard. +resolved_root=$(realpath -m -- "$search_root") +tasks_root=$(realpath -m -- "$PWD/tests/tasks") +case "$resolved_root" in + "$tasks_root" | "$tasks_root"/*) ;; *) echo "stage_shared: STAGE_SHARED_ROOT must be under tests/tasks: $search_root" >&2 exit 2 ;; esac +search_root=${resolved_root#"$PWD"/} count=0 while IFS= read -r f; do diff --git a/tests/tasks/uipath-maestro-bpmn/multi_node/wiki_pageviews/wiki_pageviews.yaml b/tests/tasks/uipath-maestro-bpmn/multi_node/wiki_pageviews/wiki_pageviews.yaml index d4caddb060..bf1a84b630 100644 --- a/tests/tasks/uipath-maestro-bpmn/multi_node/wiki_pageviews/wiki_pageviews.yaml +++ b/tests/tasks/uipath-maestro-bpmn/multi_node/wiki_pageviews/wiki_pageviews.yaml @@ -61,7 +61,7 @@ success_criteria: - type: run_command description: "Package operate metadata was created" - command: "python3 -c \"from pathlib import Path; import sys; sys.exit(0 if any(Path('.').rglob('operate.json')) else 'operate.json missing')\"" + command: "python3 -c \"from pathlib import Path; import sys; b=next(Path('.').rglob('WikiPageviews.bpmn'), None); sys.exit(0 if b and (b.parent / 'operate.json').is_file() else 'operate.json missing beside WikiPageviews.bpmn')\"" timeout: 30 expected_exit_code: 0 weight: 1.0 diff --git a/tests/tasks/uipath-maestro-case/_shared/case_check.py b/tests/tasks/uipath-maestro-case/_shared/case_check.py index 73d5ab1a94..51c4ac0a10 100644 --- a/tests/tasks/uipath-maestro-case/_shared/case_check.py +++ b/tests/tasks/uipath-maestro-case/_shared/case_check.py @@ -63,7 +63,10 @@ def find_caseplan(pattern: str = "**/caseplan.json") -> str: for _, plan in candidates } if len(signatures) == 1: - return min((path for path, _ in candidates), key=lambda path: (path.count(os.sep), len(path), path)) + return min( + (path for path, _ in candidates), + key=lambda path: (path.count(os.sep), len(path), path), + ) joined = "\n - ".join(matches) _fail(f"Multiple distinct caseplan.json files match {pattern!r}:\n - {joined}") diff --git a/tests/tasks/uipath-maestro-flow/_shared/cleanup_solutions.py b/tests/tasks/uipath-maestro-flow/_shared/cleanup_solutions.py index d7e9c29bfa..f89f82bed4 100644 --- a/tests/tasks/uipath-maestro-flow/_shared/cleanup_solutions.py +++ b/tests/tasks/uipath-maestro-flow/_shared/cleanup_solutions.py @@ -4,7 +4,9 @@ Wired in via ``post_run`` in flow e2e task YAMLs. Runs from the sandbox CWD after evaluation completes; finds every ``.uipx`` file under it, reads ``SolutionId``, and best-effort deletes each via ``uip solution delete``. -``.uipx`` files without a ``SolutionId`` are skipped. +``.uipx`` files without a ``SolutionId`` are skipped. Ids that the INTERIM +Overwrite rotation in ``flow_check.py`` moved away from are read from the +``.rotated-solution-ids`` sidecar beside each ``.uipx`` and deleted too. Cleanup policy is controlled by the ``FLOW_E2E_CLEANUP`` env var: @@ -28,6 +30,29 @@ logging.basicConfig(level=logging.INFO, format="cleanup_solutions: %(message)s") logger = logging.getLogger(__name__) +ROTATED_SOLUTION_IDS_SIDECAR = ".rotated-solution-ids" # written by flow_check.py + + +def _solution_ids(uipx_path: str) -> list[tuple[str, str]]: + """``(solution_id, source)`` pairs for one ``.uipx``: its current id plus any + ids the Overwrite rotation left on the tenant (sidecar, one per line).""" + ids: list[tuple[str, str]] = [] + with open(uipx_path) as f: + data = json.load(f) + sid = data.get("SolutionId") + if sid: + ids.append((sid, uipx_path)) + else: + logger.info("no SolutionId in %s", uipx_path) + sidecar = os.path.join(os.path.dirname(uipx_path), ROTATED_SOLUTION_IDS_SIDECAR) + if os.path.isfile(sidecar): + with open(sidecar) as f: + for line in f: + rotated = line.strip() + if rotated and all(rotated != known for known, _ in ids): + ids.append((rotated, sidecar)) + return ids + def _resolve_policy() -> str: policy = os.environ.get("FLOW_E2E_CLEANUP", "always").lower() @@ -53,21 +78,19 @@ def main() -> int: not_uploaded: list[str] = [] failed: list[str] = [] - for path in paths: + targets: list[tuple[str, str]] = [] + for uipx_path in paths: try: - with open(path) as f: - data = json.load(f) + ids = _solution_ids(uipx_path) except (OSError, json.JSONDecodeError) as e: - logger.warning("could not read %s: %s", path, e) - skipped.append(path) - continue - - sid = data.get("SolutionId") - if not sid: - logger.info("no SolutionId in %s, skipping", path) - skipped.append(path) + logger.warning("could not read %s: %s", uipx_path, e) + skipped.append(uipx_path) continue + if not ids: + skipped.append(uipx_path) + targets.extend(ids) + for sid, path in targets: if policy == "never": logger.info( "FLOW_E2E_CLEANUP=never; preserving %s (delete later with: uip solution delete %s)", diff --git a/tests/tasks/uipath-maestro-flow/_shared/flow_check.py b/tests/tasks/uipath-maestro-flow/_shared/flow_check.py index a541cf25c3..325094fa48 100644 --- a/tests/tasks/uipath-maestro-flow/_shared/flow_check.py +++ b/tests/tasks/uipath-maestro-flow/_shared/flow_check.py @@ -113,17 +113,10 @@ _DEFAULT_RETRIES = 2 _DEFAULT_BACKOFF_SECONDS = 5.0 -# A run that reports ``finalStatus: Completed`` but whose ``Data`` carries no -# ``variables`` key at all did NOT produce "no outputs": the CLI's -# post-completion GET of ``/debug-instances/{id}/variables`` failed and (before -# UiPath/cli fixed it) the failure was reduced to a stderr warning. A flow that -# genuinely declares no outputs still returns ``variables`` (with an empty -# ``globals`` map), so "key absent" is exact, not heuristic. Newer CLIs keep the -# key absent and add ``variablesError`` explaining why. Both shapes mean -# "outputs UNREADABLE" โ€” an infrastructure failure to retry, never a flow -# result to grade. Observed live on coder-eval ``skill-flow-move-node`` (v2, -# 2026-08-31 and 2026-09-01): every element Completed, ``Outputs: []``, task -# graded as a flow defect. +# ``finalStatus: Completed`` with NO ``variables`` key (or a ``variablesError``) +# means the CLI's post-completion outputs fetch failed โ€” infrastructure to retry, +# not an empty result to grade (a flow with no outputs still returns the key). +# Retries and the CLI-side fix: UiPath/cli#3929. _VARIABLES_UNREADABLE_ATTEMPTS = 2 _STDERR_CAPTURE_TAIL_CHARS = 4000 @@ -143,48 +136,45 @@ ) +def _rglob_pruned(pattern: str) -> list[str]: + """``glob.glob(pattern, recursive=True)`` from the CWD, but never descending + into ``node_modules`` (the preview workspace symlinks the baked SDK tree + there, and ``glob`` follows directory symlinks under ``**``).""" + if "**" not in pattern: + return sorted(glob.glob(pattern, recursive=True)) + prefix, _, suffix = pattern.partition("**") + root = prefix.rstrip("/") or "." + suffix = suffix.lstrip("/") + matches: list[str] = [] + for dirpath, dirnames, _ in os.walk(root, followlinks=True): + dirnames[:] = [d for d in dirnames if d != "node_modules"] + matches.extend(glob.glob(os.path.join(glob.escape(dirpath), suffix))) + if root == ".": + matches = [m[2:] if m.startswith("./") else m for m in matches] + return sorted(set(matches)) + + def _output_blob(result: subprocess.CompletedProcess) -> str: """Both streams, lowercased, for case-insensitive marker matching.""" return f"{result.stdout}\n{result.stderr}".lower() -# INTERIM โ€” Studio Web `Solution/{id}/Overwrite` regression (UiPath/cli#3938). -# -# Since 2026-09-02 (last known good: 2026-09-01 07:30Z) Studio Web answers -# `400 {"code":"1001","message":"An argument had an invalid value."}` when a -# solution that EXISTS is overwritten. `uip maestro flow debug` imports the -# project on its first run and writes the server's solution id back into the -# `.uipx`, so every LATER debug of the same project is an overwrite โ€” and the -# CLI treats any non-404 Overwrite as fatal. Result: the second seeded case of -# every multi-case checker died at solution upload, in all three arms -# (orchestrator_paths, escalation_slack_alert, billing_invoice_lookup, -# decision, wiki_pageviews, canary). Reproduced on the host with two CLI -# vintages and with `uip solution upload --force`; a fresh bundled SolutionId -# imports as new and Completes (Slack message posted), so ONLY overwrite is -# broken. The only earlier archived Overwrite 400 (2026-08-26) was code 20001, a -# content error that must still fail. -# -# So: on exactly that signature, rotate the bundled SolutionId to a fresh uuid4 -# and retry ONCE โ€” the retry is a new import. Self-limiting: once Overwrite -# recovers the signature never fires and nothing here runs. Cost while it does: -# one extra tenant solution per affected case, which `cleanup_solutions.py` -# cannot see (it reads the `.uipx`'s CURRENT id) โ€” the ids are printed so they -# are not lost. The platform fix and a CLI-side fallback are asked for in the -# issue above; delete this block when Overwrite works again. +# INTERIM โ€” Studio Web answers 400/1001 when an EXISTING solution is overwritten +# (UiPath/cli#3938 has the evidence). `flow debug` #2 of a project is an +# overwrite, so on exactly that signature the bundled SolutionId is rotated and +# the debug retried ONCE as a new import. Delete this block when Overwrite works. _OVERWRITE_STUCK_ISSUE = "UiPath/cli#3938" _OVERWRITE_STUCK_MARKER = "overwrite failed (400)" _OVERWRITE_STUCK_CODE = re.compile(r'\\?"code\\?":\s*\\?"1001\\?"') -# Since UiPath/cli#3951 (main 2026-09-02, `016231474`) the CLI reports a refused -# upload against its stage instead of echoing the raw body: -# Message: "Failed during overwrite-solution: HTTP 400 on POST โ€ฆ/Overwrite โ€” An argument had an invalid value." -# Context: {"HttpStatus": 400, "Stage": "overwrite-solution", "ErrorCode": "1001"} -# The rotation must recognise both envelopes: an image whose CLI carries #3951 -# but whose checker knows only the legacy text fails every second debug of a -# project with a plain `flow debug exit 1` (seen 2026-09-03 on every -# multi-debug task of the batch-A rerun, both arms). +# Since UiPath/cli#3951 the CLI reports the refusal as +# Context: {"HttpStatus": 400, "Stage": "overwrite-solution", "ErrorCode": "1001"} +# instead of echoing the raw body; both envelopes must rotate. _OVERWRITE_STUCK_STAGE = "overwrite-solution" _OVERWRITE_STUCK_ATTEMPTS = 2 # the failed overwrite + one import-as-new retry _SOLUTION_ID_RE = re.compile(r'("SolutionId"\s*:\s*")([0-9a-fA-F-]{36})(")') +# Rotated-away ids, one per line, beside the `.uipx`; `cleanup_solutions.py` +# deletes them alongside the current id so the rotation leaks nothing. +ROTATED_SOLUTION_IDS_SIDECAR = ".rotated-solution-ids" def _is_overwrite_stuck(result: subprocess.CompletedProcess) -> bool: @@ -221,7 +211,9 @@ def _rotate_solution_id(project_dir: str) -> tuple[str, str] | None: Walks up from ``project_dir`` to the nearest ancestor holding exactly one ``*.uipx`` (a JSON text file; ``uip solution init`` writes it beside the project directory). Returns ``(old, new)``, or ``None`` when no unambiguous - ``.uipx`` is found โ€” the caller then fails the plain way.""" + ``.uipx`` is found โ€” the caller then fails the plain way. The previous id is + appended to :data:`ROTATED_SOLUTION_IDS_SIDECAR` next to the ``.uipx`` so + ``cleanup_solutions.py`` can delete the solution it still names on the tenant.""" import uuid here = os.path.abspath(project_dir) @@ -239,6 +231,9 @@ def _rotate_solution_id(project_dir: str) -> tuple[str, str] | None: new = str(uuid.uuid4()) with open(uipx[0], "w", encoding="utf-8") as handle: handle.write(text[: match.start(2)] + new + text[match.end(2):]) + sidecar = os.path.join(here, ROTATED_SOLUTION_IDS_SIDECAR) + with open(sidecar, "a", encoding="utf-8") as handle: + handle.write(match.group(2) + "\n") return match.group(2), new parent = os.path.dirname(here) if parent == here: @@ -483,7 +478,7 @@ def run_debug( f"INTERIM ({_OVERWRITE_STUCK_ISSUE}: Studio Web Overwrite โ†’ 400/1001 for an " f"existing solution since 2026-09-02): rotated the bundled SolutionId " f"{rotated[0]} โ†’ {rotated[1]}; retrying as a new import. The solution " - f"{rotated[0]} stays on the tenant (cleanup reads only the current id).", + f"{rotated[0]} is recorded in {ROTATED_SOLUTION_IDS_SIDECAR} for cleanup.", file=sys.stderr, flush=True, ) @@ -1745,7 +1740,7 @@ def find_flow_files( Debug callers intentionally do not use this helper: ``uip maestro flow debug`` is project-scoped and must continue through :func:`find_project_dir`. """ - project_candidates = sorted(glob.glob(project_pattern, recursive=True)) + project_candidates = _rglob_pruned(project_pattern) flow_projects = [path for path in project_candidates if _is_flow_project(path)] root_matches = _dedupe_flow_candidates( sorted(glob.glob(os.path.basename(flow_glob))) @@ -1821,7 +1816,7 @@ def assert_no_flow_files() -> None: tasks. It deliberately scans every location because any emitted Flow is a failure; there is no preferred artifact to select in the absence case. """ - matches = sorted(glob.glob("**/*.flow", recursive=True)) + matches = _rglob_pruned("**/*.flow") if matches: joined = "\n - ".join(matches) _fail(f"Unexpected .flow file(s) found:\n - {joined}") @@ -1955,7 +1950,7 @@ def _find_project(pattern: str) -> str: abandoned scaffold โ€” see :func:`_dedupe_flow_projects` and :func:`_split_off_scaffold_husks`. Anything else stays a refusal. """ - candidates = sorted(glob.glob(pattern, recursive=True)) + candidates = _rglob_pruned(pattern) if not candidates: _fail(f"No project.uiproj found matching {pattern}") flow_projects = [p for p in candidates if _is_flow_project(p)] diff --git a/tests/tasks/uipath-maestro-flow/_shared/test_flow_check.py b/tests/tasks/uipath-maestro-flow/_shared/test_flow_check.py index 074200ceed..3da15026ba 100644 --- a/tests/tasks/uipath-maestro-flow/_shared/test_flow_check.py +++ b/tests/tasks/uipath-maestro-flow/_shared/test_flow_check.py @@ -596,6 +596,27 @@ def test_find_project_fails_when_multiple_flows(tmp_path, monkeypatch): _find_project("**/project.uiproj") +def test_find_project_ignores_the_staged_node_modules_symlink(tmp_path, monkeypatch): + """stage-preview-sdk-workspace.sh symlinks the baked SDK tree to + ./node_modules; `glob('**')` follows it, so a fixture .flow/.uiproj inside + the SDK must never become a candidate (or a false `assert_no_flow_files`).""" + monkeypatch.chdir(tmp_path) + solution = tmp_path / "Real" + solution.mkdir() + _make_proj(solution, "Real", "Flow") + sdk = tmp_path.parent / f"{tmp_path.name}-sdk" / "node_modules" / "@uipath" / "flow-sdk" / "fixtures" + sdk.mkdir(parents=True) + (sdk / "project.uiproj").write_text('{"ProjectType": "Flow"}') + (sdk / "Fixture.flow").write_text("{}") + os.symlink(sdk.parent.parent.parent, tmp_path / "node_modules") + assert _find_project("**/project.uiproj") == os.path.join("Real", "Real") + assert flow_check._rglob_pruned("**/*.flow") == [] + flow_check.assert_no_flow_files() # does not raise + (solution / "Real" / "Real.flow").write_text("{}") + with pytest.raises(SystemExit, match="Real.flow"): + flow_check.assert_no_flow_files() + + def test_find_project_fails_when_no_candidates(tmp_path, monkeypatch): """No project.uiproj at all โ€” original failure message preserved.""" monkeypatch.chdir(tmp_path) @@ -1094,8 +1115,42 @@ def test_run_debug_rotates_the_solution_id_on_overwrite_1001_then_imports(monkey err = capsys.readouterr().err assert "INTERIM (UiPath/cli#3938" in err assert "rotated the bundled SolutionId 79cda3cb-5a10-4f37-e091-08df08c2afbe" in err - # The orphaned server id is named, because cleanup reads only the current one. assert "previous: 79cda3cb-5a10-4f37-e091-08df08c2afbe" in err + # The rotated-away id is recorded beside the .uipx so cleanup_solutions.py + # deletes the tenant solution it still names (nothing leaks). + sidecar = sol / flow_check.ROTATED_SOLUTION_IDS_SIDECAR + assert sidecar.read_text().splitlines() == ["79cda3cb-5a10-4f37-e091-08df08c2afbe"] + + +def test_cleanup_solutions_deletes_rotated_ids_from_the_sidecar(monkeypatch, tmp_path): + """The sidecar written by the rotation is a cleanup input, not a log line.""" + import importlib.util + + sol, _ = _uipx_project(tmp_path, solution_id="11111111-1111-4111-8111-111111111111") + (sol / flow_check.ROTATED_SOLUTION_IDS_SIDECAR).write_text( + "22222222-2222-4222-8222-222222222222\n\n11111111-1111-4111-8111-111111111111\n" + ) + spec = importlib.util.spec_from_file_location( + "cleanup_solutions", os.path.join(os.path.dirname(flow_check.__file__), "cleanup_solutions.py") + ) + cleanup = importlib.util.module_from_spec(spec) + spec.loader.exec_module(cleanup) + deleted: list[str] = [] + + def fake_run(cmd, **kwargs): + deleted.append(cmd[3]) + return subprocess.CompletedProcess(cmd, 0, stdout="{}", stderr="") + + monkeypatch.setattr(cleanup.subprocess, "run", fake_run) + monkeypatch.chdir(tmp_path) + monkeypatch.delenv("FLOW_E2E_CLEANUP", raising=False) + assert cleanup.main() == 0 + # current id + rotated id, each exactly once (the sidecar's duplicate of the + # current id and its blank line are ignored) + assert sorted(deleted) == [ + "11111111-1111-4111-8111-111111111111", + "22222222-2222-4222-8222-222222222222", + ] def test_run_debug_rotates_the_solution_id_on_the_staged_overwrite_envelope(monkeypatch, tmp_path, capsys): diff --git a/tests/tasks/uipath-maestro-flow/_shared/test_same_ground_corpus.py b/tests/tasks/uipath-maestro-flow/_shared/test_same_ground_corpus.py index 8fbf4a4a67..934efdb0f0 100644 --- a/tests/tasks/uipath-maestro-flow/_shared/test_same_ground_corpus.py +++ b/tests/tasks/uipath-maestro-flow/_shared/test_same_ground_corpus.py @@ -226,19 +226,30 @@ def test_multiselect_prompt_names_slack_as_the_required_connector() -> None: def test_paginated_lookup_accepts_both_supported_resolution_routes() -> None: text = _task_text("connector_features/paginated_reference_lookup.yaml") + blocks = [block for kind, block in _criterion_blocks(text) if kind == "command_executed"] + manual_block, paged_block = blocks[:2] patterns = re.findall(r"(?m)^\s+command_pattern:\s*'([^']+)'", text) - resolve_pattern, pagination_pattern = map(re.compile, patterns[:2]) + manual_pattern, paged_pattern = map(re.compile, patterns[:2]) - manual_commands = ( - "uip is resources run list uipath-salesforce-slack conversations " - "--connection-id id --query nextPage=token" - ) + page_one = 'bash -lc "uip is resources run list \\"uipath-salesforce-slack\\" \\"conversations\\" --connection-id id"' + page_two = page_one[:-1] + ' --query nextPage=token"' sdk_command = ( - "uip maestro flow registry prepare uipath-salesforce-slack " + "npx flow-sdk registry prepare uipath-salesforce-slack " "send-message-to-channel --resolve channel:name=simple" ) - assert resolve_pattern.search(manual_commands) - assert pagination_pattern.search(manual_commands) - assert resolve_pattern.search(sdk_command) - assert pagination_pattern.search(sdk_command) + # The v1 route is telemetry the SDK arm never produces: advisory, but it + # still asks for the loop (page 1 alone is not pagination). + assert "min_count: 2" in manual_block and "pass_threshold: 0.0" in manual_block + assert manual_pattern.search(page_one) and manual_pattern.search(page_two) + assert not manual_pattern.search(sdk_command) + + # The gating criterion needs "went past page 1" on a SUCCESSFUL command: + # a page-1 call, or `--resolve channel:` typed on something other than + # `registry prepare`, earns nothing. + assert "require_success: true" in paged_block and "pass_threshold: 1.0" in paged_block + assert paged_pattern.search(page_two) + assert paged_pattern.search(sdk_command) + assert not paged_pattern.search(page_one) + assert not paged_pattern.search("uip maestro flow validate --resolve channel:name=simple") + assert not paged_pattern.search("echo nextPage=token") diff --git a/tests/tasks/uipath-maestro-flow/_shared/test_validate_flow.py b/tests/tasks/uipath-maestro-flow/_shared/test_validate_flow.py index 8b7fe34ca2..5486620362 100644 --- a/tests/tasks/uipath-maestro-flow/_shared/test_validate_flow.py +++ b/tests/tasks/uipath-maestro-flow/_shared/test_validate_flow.py @@ -164,10 +164,17 @@ def test_budget_exhaustion_reports_instead_of_running(monkeypatch, capsys): assert "-0s left" not in err # negative remainder is clamped for display -def test_no_flow_file_fails(monkeypatch, capsys): +def test_no_flow_file_fails(monkeypatch): + """`find_flow_files()` exits with its own message when nothing is found; + `main` has no empty-list branch of its own.""" _stub(monkeypatch, [], flows=()) - assert validate_flow.main() == 1 - assert "No .flow file found" in capsys.readouterr().err + + def no_flows(): + raise SystemExit("FAIL: No Flow project found matching '**/project.uiproj'") + + monkeypatch.setattr(validate_flow, "find_flow_files", no_flows) + with pytest.raises(SystemExit, match="No Flow project found"): + validate_flow.main() @pytest.mark.parametrize( diff --git a/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py b/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py index 04052663dd..7116080a5c 100644 --- a/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py +++ b/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py @@ -164,10 +164,7 @@ def _validate(flow: str, deadline: float, budget: int) -> int: def main() -> int: - flows = find_flow_files() - if not flows: - print("FAIL: No .flow file found", file=sys.stderr) - return 1 + flows = find_flow_files() # exits with its own message when nothing is found budget = _budget_seconds() deadline = time.monotonic() + max( diff --git a/tests/tasks/uipath-maestro-flow/bindings/idempotent_reconfigure.yaml b/tests/tasks/uipath-maestro-flow/bindings/idempotent_reconfigure.yaml index 210b467b1a..e1793bf5e5 100644 --- a/tests/tasks/uipath-maestro-flow/bindings/idempotent_reconfigure.yaml +++ b/tests/tasks/uipath-maestro-flow/bindings/idempotent_reconfigure.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-bindings-idempotent-reconfigure description: > Step 1 of the DAPField-mirrored upsert in `upsertConnectionResourceBinding` diff --git a/tests/tasks/uipath-maestro-flow/bindings/multi_connector_independence.yaml b/tests/tasks/uipath-maestro-flow/bindings/multi_connector_independence.yaml index 3d1d308a19..fff5a19647 100644 --- a/tests/tasks/uipath-maestro-flow/bindings/multi_connector_independence.yaml +++ b/tests/tasks/uipath-maestro-flow/bindings/multi_connector_independence.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-bindings-multi-connector-independence description: > Two distinct connector nodes (different connector keys) in the same flow, diff --git a/tests/tasks/uipath-maestro-flow/bindings/no_duplicate_connection_bindings.yaml b/tests/tasks/uipath-maestro-flow/bindings/no_duplicate_connection_bindings.yaml index 8f0a91d17c..51421273e5 100644 --- a/tests/tasks/uipath-maestro-flow/bindings/no_duplicate_connection_bindings.yaml +++ b/tests/tasks/uipath-maestro-flow/bindings/no_duplicate_connection_bindings.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-bindings-no-duplicates description: > Regression - `uip maestro flow node configure` previously appended diff --git a/tests/tasks/uipath-maestro-flow/bindings/reconfigure_different_connection.yaml b/tests/tasks/uipath-maestro-flow/bindings/reconfigure_different_connection.yaml index 23986756b8..3b0ba5ff5f 100644 --- a/tests/tasks/uipath-maestro-flow/bindings/reconfigure_different_connection.yaml +++ b/tests/tasks/uipath-maestro-flow/bindings/reconfigure_different_connection.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-bindings-reconfigure-different-connection description: > When the same connector node is reconfigured against a different connection, diff --git a/tests/tasks/uipath-maestro-flow/connector_features/ceql_where.yaml b/tests/tasks/uipath-maestro-flow/connector_features/ceql_where.yaml index 4d1d1909f7..62bf12ecda 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/ceql_where.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_features/ceql_where.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-ipe-ceql-where description: > Tests the CEQL where query IS feature โ€” agent plans a structured filter @@ -32,7 +31,7 @@ initial_prompt: | Branch on the result: if the call fails, stop the flow immediately; if it succeeds, log "CeqlWhere test passed". - Compile the final flow file. Do not fabricate a tenant connection merely to + Produce the final flow file. Do not fabricate a tenant connection merely to make product validation pass; the generated structure and the separately reviewable filter are the deliverables in this offline task. diff --git a/tests/tasks/uipath-maestro-flow/connector_features/complex_array.yaml b/tests/tasks/uipath-maestro-flow/connector_features/complex_array.yaml index ebfa75a8c2..16bfaf75e5 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/complex_array.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_features/complex_array.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-ipe-complex-array description: > Tests the complex array IS feature โ€” configures the Slack diff --git a/tests/tasks/uipath-maestro-flow/connector_features/dtl_load_by_default_false.yaml b/tests/tasks/uipath-maestro-flow/connector_features/dtl_load_by_default_false.yaml index 1d4f9584b9..c2e97a2040 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/dtl_load_by_default_false.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_features/dtl_load_by_default_false.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-ipe-dtl-load-by-default-false description: > Tests the DTL loadByDefault=false IS feature โ€” configures a connector node diff --git a/tests/tasks/uipath-maestro-flow/connector_features/dtl_load_by_default_true.yaml b/tests/tasks/uipath-maestro-flow/connector_features/dtl_load_by_default_true.yaml index e3733d57a3..b8ffa816b6 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/dtl_load_by_default_true.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_features/dtl_load_by_default_true.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-ipe-dtl-load-by-default-true description: > Tests the DTL loadByDefault=true IS feature โ€” configures a Jira Get Issue diff --git a/tests/tasks/uipath-maestro-flow/connector_features/enhanced_enum.yaml b/tests/tasks/uipath-maestro-flow/connector_features/enhanced_enum.yaml index 29d1680003..10634fca27 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/enhanced_enum.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_features/enhanced_enum.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-ipe-enhanced-enum description: > Tests the enhanced enum IS feature โ€” configures a connector node with an @@ -19,7 +18,7 @@ initial_prompt: | Build a flow that retrieves WooCommerce product reviews and lets the caller choose whether they come back oldest-first or newest-first, picking from friendly labels rather than raw codes. Discover the get-product-reviews operation and set - its sort-order field. Compile the final flow file. + its sort-order field. Produce the final flow file. If the WooCommerce operation exists in the registry but the tenant has no live connection, keep the real WooCommerce operation node with its inputs left unset and stop there. Do not fabricate a tenant connection merely to make product diff --git a/tests/tasks/uipath-maestro-flow/connector_features/enum.yaml b/tests/tasks/uipath-maestro-flow/connector_features/enum.yaml index cb9fb7239c..31839faaee 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/enum.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_features/enum.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-ipe-enum description: > Tests the enum IS feature โ€” configures a connector node with an enum diff --git a/tests/tasks/uipath-maestro-flow/connector_features/generic_dynamic_node/generic_dynamic_node.yaml b/tests/tasks/uipath-maestro-flow/connector_features/generic_dynamic_node/generic_dynamic_node.yaml index b17b4577a0..f6c7d2fe7b 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/generic_dynamic_node/generic_dynamic_node.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_features/generic_dynamic_node/generic_dynamic_node.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-generic-dynamic-node description: > Connector feature: validate a generic (dynamic) connector node end-to-end. A diff --git a/tests/tasks/uipath-maestro-flow/connector_features/jdbc_databricks_query/jdbc_databricks_query.yaml b/tests/tasks/uipath-maestro-flow/connector_features/jdbc_databricks_query/jdbc_databricks_query.yaml index 06611734e8..f77b019163 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/jdbc_databricks_query/jdbc_databricks_query.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_features/jdbc_databricks_query/jdbc_databricks_query.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. # Tenant-gated. Requires the Databricks-backed Database Hub (JDBC) # connection in the runner tenant, reaching the sandbox.globalmart schema whose # `employees` table is seeded with the fixture rows this task asserts on (Alice diff --git a/tests/tasks/uipath-maestro-flow/connector_features/multiselect.yaml b/tests/tasks/uipath-maestro-flow/connector_features/multiselect.yaml index b16551315c..09abdff77d 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/multiselect.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_features/multiselect.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-ipe-multiselect description: > Tests the multiselect IS feature โ€” configures a Slack group direct message diff --git a/tests/tasks/uipath-maestro-flow/connector_features/non-catalog-http-fallback/non_catalog_http_fallback.yaml b/tests/tasks/uipath-maestro-flow/connector_features/non-catalog-http-fallback/non_catalog_http_fallback.yaml index c6fc456c3a..a39d5333b1 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/non-catalog-http-fallback/non_catalog_http_fallback.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_features/non-catalog-http-fallback/non_catalog_http_fallback.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-non-catalog-http-fallback description: > Integration test: a NON-catalog service (Spotify) has no IS connector of its diff --git a/tests/tasks/uipath-maestro-flow/connector_features/paginated_reference_lookup.yaml b/tests/tasks/uipath-maestro-flow/connector_features/paginated_reference_lookup.yaml index 711df8b599..3791ffd3ff 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/paginated_reference_lookup.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_features/paginated_reference_lookup.yaml @@ -38,22 +38,28 @@ initial_prompt: | planning and implementation. Build the complete flow end-to-end in a single pass. success_criteria: -# TWO ROUTES ARE CORRECT, and these criteria accept either. Manual discovery -# may expose the channel collection as `conversations` or `curated_channels`; -# the final artifact criterion below is the authority on the resolved id. +# TWO ROUTES ARE CORRECT: the live-v1 loop pages `uip is resources run list` +# by hand (the collection may be exposed as `conversations` or +# `curated_channels`); the SDK route resolves the name inside +# `npx flow-sdk registry prepare โ€ฆ --resolve channel:`. The first criterion is +# v1-route telemetry and is advisory (the SDK arm never runs it); the second +# gates on "went past page 1" for either route and only counts a command that +# succeeded โ€” typing the flag on a failing command does not page. The final +# artifact criterion (resolved id C083AN4E61E) stays the authority. - type: command_executed - description: "Agent resolved the channel through the library โ€” either a paged Slack channel-resource loop or `registry prepare --resolve`" + description: "Advisory (live-v1 route): paged Slack channel-resource loop ran more than once" tool_name: "Bash" - command_pattern: '(uip\s+is\s+resources\s+run\s+list\s+\\?"?uipath-salesforce-slack\\?"?\s+\\?"?(curated_channels|conversations)|registry\s+prepare\s+\\?"?uipath-salesforce-slack)' - min_count: 1 + command_pattern: 'uip\s+is\s+resources\s+run\s+list\s+\\?"?uipath-salesforce-slack\\?"?\s+\\?"?(curated_channels|conversations)' + min_count: 2 weight: 3.0 - pass_threshold: 1.0 + pass_threshold: 0.0 - type: command_executed - description: "Agent went past page 1 โ€” `nextPage=` by hand, or `--resolve channel:` which pages inside `prepare`" + description: "Agent went past page 1 โ€” a channel-resource list call carrying `nextPage=`, or `registry prepare โ€ฆ --resolve channel:` which pages inside `prepare`" tool_name: "Bash" - command_pattern: '(nextPage=|--resolve\s+\\?"?channel:)' + command_pattern: '(uip\s+is\s+resources\s+run\s+list\s+\\?"?uipath-salesforce-slack\\?"?\s+\\?"?(curated_channels|conversations)[^\n]*nextPage=|registry\s+prepare\s+\\?"?uipath-salesforce-slack\\?"?[^\n]*--resolve\s+\\?"?channel:)' min_count: 1 + require_success: true weight: 3.0 pass_threshold: 1.0 diff --git a/tests/tasks/uipath-maestro-flow/connector_features/path_params.yaml b/tests/tasks/uipath-maestro-flow/connector_features/path_params.yaml index ebeea6b0ed..fdd69454f3 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/path_params.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_features/path_params.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-ipe-path-params description: > Tests the path parameters IS feature โ€” configures a connector node with a diff --git a/tests/tasks/uipath-maestro-flow/connector_features/query_params.yaml b/tests/tasks/uipath-maestro-flow/connector_features/query_params.yaml index 5ba1eaa0ef..ea765d06e2 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/query_params.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_features/query_params.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-ipe-query-params description: > Tests the query parameters IS feature โ€” configures a connector node with a diff --git a/tests/tasks/uipath-maestro-flow/connector_features/searchable_joins.yaml b/tests/tasks/uipath-maestro-flow/connector_features/searchable_joins.yaml index 2879bfa16f..8e16e9e422 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/searchable_joins.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_features/searchable_joins.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-ipe-searchable-joins description: > Tests the searchable joins IS feature โ€” configures a connector node with diff --git a/tests/tasks/uipath-maestro-flow/connector_features/slack-http-fallback/slack_http_fallback.yaml b/tests/tasks/uipath-maestro-flow/connector_features/slack-http-fallback/slack_http_fallback.yaml index 05c80214ca..11bc203d20 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/slack-http-fallback/slack_http_fallback.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_features/slack-http-fallback/slack_http_fallback.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-slack-http-fallback description: > E2E test: a catalog connector (Slack, uipath-salesforce-slack) has no native diff --git a/tests/tasks/uipath-maestro-flow/connector_features/testmanager_attachments/testmanager_attachments.yaml b/tests/tasks/uipath-maestro-flow/connector_features/testmanager_attachments/testmanager_attachments.yaml index 79766cbfb5..8b94c6b588 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/testmanager_attachments/testmanager_attachments.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_features/testmanager_attachments/testmanager_attachments.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. # skip:true, tenant-gated. CONFIRMED available in Maestro Flow (2026-06-16): # `uip maestro flow registry pull` then search returns the # uipath.connector.uipath-uipath-testmanager. nodes for this object group. A stale manifest diff --git a/tests/tasks/uipath-maestro-flow/connector_features/testmanager_crud_grounded/testmanager_crud_grounded.yaml b/tests/tasks/uipath-maestro-flow/connector_features/testmanager_crud_grounded/testmanager_crud_grounded.yaml index b4f897bfff..dc5b603ed3 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/testmanager_crud_grounded/testmanager_crud_grounded.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_features/testmanager_crud_grounded/testmanager_crud_grounded.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. # PROTOTYPE โ€” data-grounded on the Maestro Flow surface. skip:true. # Agent builds a flow (create-test-case node -> get-test-case node -> output the # retrieved name) and EXECUTES it (`uip maestro flow debug`) to produce the diff --git a/tests/tasks/uipath-maestro-flow/connector_features/testmanager_execution_results/testmanager_execution_results.yaml b/tests/tasks/uipath-maestro-flow/connector_features/testmanager_execution_results/testmanager_execution_results.yaml index 11183d547d..1c9ad6d871 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/testmanager_execution_results/testmanager_execution_results.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_features/testmanager_execution_results/testmanager_execution_results.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. # skip:true, tenant-gated. CONFIRMED available in Maestro Flow (2026-06-16): # `uip maestro flow registry pull` then search returns the # uipath.connector.uipath-uipath-testmanager. nodes for this object group. A stale manifest diff --git a/tests/tasks/uipath-maestro-flow/connector_features/testmanager_generic_records/testmanager_generic_records.yaml b/tests/tasks/uipath-maestro-flow/connector_features/testmanager_generic_records/testmanager_generic_records.yaml index 9ec8597c4f..7a2fd6c244 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/testmanager_generic_records/testmanager_generic_records.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_features/testmanager_generic_records/testmanager_generic_records.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. # skip:true, tenant-gated. CONFIRMED available in Maestro Flow (2026-06-16): # `uip maestro flow registry pull` then search returns the # uipath.connector.uipath-uipath-testmanager. nodes for this object group. A stale manifest diff --git a/tests/tasks/uipath-maestro-flow/connector_features/testmanager_requirement_lifecycle/testmanager_requirement_lifecycle.yaml b/tests/tasks/uipath-maestro-flow/connector_features/testmanager_requirement_lifecycle/testmanager_requirement_lifecycle.yaml index 97ca38cac5..b4c13b49a0 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/testmanager_requirement_lifecycle/testmanager_requirement_lifecycle.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_features/testmanager_requirement_lifecycle/testmanager_requirement_lifecycle.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. # skip:true, tenant-gated. CONFIRMED available in Maestro Flow (2026-06-16): # `uip maestro flow registry pull` then search returns the # uipath.connector.uipath-uipath-testmanager. nodes for this object group. A stale manifest diff --git a/tests/tasks/uipath-maestro-flow/connector_features/testmanager_testcase_lifecycle/testmanager_testcase_lifecycle.yaml b/tests/tasks/uipath-maestro-flow/connector_features/testmanager_testcase_lifecycle/testmanager_testcase_lifecycle.yaml index 7e252fdb75..a2f815a95f 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/testmanager_testcase_lifecycle/testmanager_testcase_lifecycle.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_features/testmanager_testcase_lifecycle/testmanager_testcase_lifecycle.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. # skip:true, tenant-gated. CONFIRMED available in Maestro Flow (2026-06-16): # `uip maestro flow registry pull` then search returns the # uipath.connector.uipath-uipath-testmanager. nodes for this object group. A stale manifest diff --git a/tests/tasks/uipath-maestro-flow/connector_features/testmanager_testset_lifecycle/testmanager_testset_lifecycle.yaml b/tests/tasks/uipath-maestro-flow/connector_features/testmanager_testset_lifecycle/testmanager_testset_lifecycle.yaml index 869a4d162d..0699893d53 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/testmanager_testset_lifecycle/testmanager_testset_lifecycle.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_features/testmanager_testset_lifecycle/testmanager_testset_lifecycle.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. # skip:true, tenant-gated. CONFIRMED available in Maestro Flow (2026-06-16): # `uip maestro flow registry pull` then search returns the # uipath.connector.uipath-uipath-testmanager. nodes for this object group. A stale manifest diff --git a/tests/tasks/uipath-maestro-flow/connector_trigger/trigger_with_filter.yaml b/tests/tasks/uipath-maestro-flow/connector_trigger/trigger_with_filter.yaml index 550779f2b7..8729610a27 100644 --- a/tests/tasks/uipath-maestro-flow/connector_trigger/trigger_with_filter.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_trigger/trigger_with_filter.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-trigger-with-filter description: > Verifies that the uipath-maestro-flow skill teaches agents to emit a structured `filter` tree. diff --git a/tests/tasks/uipath-maestro-flow/connector_trigger/webhook_waitfor_parallel.yaml b/tests/tasks/uipath-maestro-flow/connector_trigger/webhook_waitfor_parallel.yaml index 521c211ec8..a5f2b25faf 100644 --- a/tests/tasks/uipath-maestro-flow/connector_trigger/webhook_waitfor_parallel.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_trigger/webhook_waitfor_parallel.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-webhook-waitfor-parallel description: > E2E self-testing flow: a manual start trigger fans out into two parallel diff --git a/tests/tasks/uipath-maestro-flow/context-grounding/batch_transform/batch_transform.yaml b/tests/tasks/uipath-maestro-flow/context-grounding/batch_transform/batch_transform.yaml index 7d51e9ca8f..e231bed1a1 100644 --- a/tests/tasks/uipath-maestro-flow/context-grounding/batch_transform/batch_transform.yaml +++ b/tests/tasks/uipath-maestro-flow/context-grounding/batch_transform/batch_transform.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-batch-transform description: > Create a UiPath Flow that runs a Batch Transform pattern node over a CSV diff --git a/tests/tasks/uipath-maestro-flow/context-grounding/summarize/summarize.yaml b/tests/tasks/uipath-maestro-flow/context-grounding/summarize/summarize.yaml index f0a04b3cd9..b722c6759a 100644 --- a/tests/tasks/uipath-maestro-flow/context-grounding/summarize/summarize.yaml +++ b/tests/tasks/uipath-maestro-flow/context-grounding/summarize/summarize.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-summarize description: > Create a UiPath Flow that runs a Summarize pattern node (`uipath.pattern.deep-rag`) diff --git a/tests/tasks/uipath-maestro-flow/e2e/devcon_expense_approval.yaml b/tests/tasks/uipath-maestro-flow/e2e/devcon_expense_approval.yaml index a43490dfb4..6c6f2536c6 100644 --- a/tests/tasks/uipath-maestro-flow/e2e/devcon_expense_approval.yaml +++ b/tests/tasks/uipath-maestro-flow/e2e/devcon_expense_approval.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-e2e-devcon-expense-approval description: > DevCon end-to-end scenario: developer gives a vague expense approval requirement. diff --git a/tests/tasks/uipath-maestro-flow/e2e/jira_create_issue/jira_create_issue.yaml b/tests/tasks/uipath-maestro-flow/e2e/jira_create_issue/jira_create_issue.yaml index e5d4a587fc..2d22fcff23 100644 --- a/tests/tasks/uipath-maestro-flow/e2e/jira_create_issue/jira_create_issue.yaml +++ b/tests/tasks/uipath-maestro-flow/e2e/jira_create_issue/jira_create_issue.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-ipe-jira-create-issue description: > E2E live Jira coverage โ€” builds a Flow with a manual trigger and an diff --git a/tests/tasks/uipath-maestro-flow/e2e/jira_get_issue/jira_get_issue.yaml b/tests/tasks/uipath-maestro-flow/e2e/jira_get_issue/jira_get_issue.yaml index f7158a3c8c..348716627b 100644 --- a/tests/tasks/uipath-maestro-flow/e2e/jira_get_issue/jira_get_issue.yaml +++ b/tests/tasks/uipath-maestro-flow/e2e/jira_get_issue/jira_get_issue.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-ipe-jira-get-issue description: > E2E live Jira coverage โ€” builds a Flow with a manual trigger and an Atlassian diff --git a/tests/tasks/uipath-maestro-flow/e2e/jira_lifecycle/jira_lifecycle.yaml b/tests/tasks/uipath-maestro-flow/e2e/jira_lifecycle/jira_lifecycle.yaml index 4d1422007a..c57ee8310b 100644 --- a/tests/tasks/uipath-maestro-flow/e2e/jira_lifecycle/jira_lifecycle.yaml +++ b/tests/tasks/uipath-maestro-flow/e2e/jira_lifecycle/jira_lifecycle.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-ipe-jira-lifecycle description: > E2E live Jira coverage of a composite loop-and-switch flow. The agent builds diff --git a/tests/tasks/uipath-maestro-flow/e2e/jira_search_triage/jira_search_triage.yaml b/tests/tasks/uipath-maestro-flow/e2e/jira_search_triage/jira_search_triage.yaml index c536c51993..ebea0c9b75 100644 --- a/tests/tasks/uipath-maestro-flow/e2e/jira_search_triage/jira_search_triage.yaml +++ b/tests/tasks/uipath-maestro-flow/e2e/jira_search_triage/jira_search_triage.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-ipe-jira-search-triage description: > E2E live Jira coverage of a JQL-search-driven triage flow: a manual-trigger diff --git a/tests/tasks/uipath-maestro-flow/edit/add_node/add_node.yaml b/tests/tasks/uipath-maestro-flow/edit/add_node/add_node.yaml index f671195a46..e3d02d2cad 100644 --- a/tests/tasks/uipath-maestro-flow/edit/add_node/add_node.yaml +++ b/tests/tasks/uipath-maestro-flow/edit/add_node/add_node.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-add-node description: > Add a script node to an existing BellevueWeather flow that converts the diff --git a/tests/tasks/uipath-maestro-flow/edit/add_output/add_output.yaml b/tests/tasks/uipath-maestro-flow/edit/add_output/add_output.yaml index 0aff8d79b8..c43772ee84 100644 --- a/tests/tasks/uipath-maestro-flow/edit/add_output/add_output.yaml +++ b/tests/tasks/uipath-maestro-flow/edit/add_output/add_output.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-add-output description: > Add a "location" field to the end node outputs in the BellevueWeather flow. diff --git a/tests/tasks/uipath-maestro-flow/edit/group_to_subflow/group_to_subflow.yaml b/tests/tasks/uipath-maestro-flow/edit/group_to_subflow/group_to_subflow.yaml index d52710b1c4..228d3ab3fc 100644 --- a/tests/tasks/uipath-maestro-flow/edit/group_to_subflow/group_to_subflow.yaml +++ b/tests/tasks/uipath-maestro-flow/edit/group_to_subflow/group_to_subflow.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-group-to-subflow description: > Extract the getWeather HTTP node and formatSummary Script node into a subflow diff --git a/tests/tasks/uipath-maestro-flow/edit/move_node/move_node.yaml b/tests/tasks/uipath-maestro-flow/edit/move_node/move_node.yaml index b6db7d4c02..064c1fd1fc 100644 --- a/tests/tasks/uipath-maestro-flow/edit/move_node/move_node.yaml +++ b/tests/tasks/uipath-maestro-flow/edit/move_node/move_node.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-move-node description: > Move the decision node before formatSummary so both branches merge back into diff --git a/tests/tasks/uipath-maestro-flow/edit/remove_node/remove_node.yaml b/tests/tasks/uipath-maestro-flow/edit/remove_node/remove_node.yaml index db094365eb..937e678f91 100644 --- a/tests/tasks/uipath-maestro-flow/edit/remove_node/remove_node.yaml +++ b/tests/tasks/uipath-maestro-flow/edit/remove_node/remove_node.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-remove-node description: > Remove the formatSummary script node from the BellevueWeather flow and rewire diff --git a/tests/tasks/uipath-maestro-flow/edit/update_node/update_node.yaml b/tests/tasks/uipath-maestro-flow/edit/update_node/update_node.yaml index 78b869bced..c070c96651 100644 --- a/tests/tasks/uipath-maestro-flow/edit/update_node/update_node.yaml +++ b/tests/tasks/uipath-maestro-flow/edit/update_node/update_node.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-update-node description: > Edit an existing Bellevue-weather flow: rewrite the decision-branch script diff --git a/tests/tasks/uipath-maestro-flow/evaluate/evaluator_type_choice.yaml b/tests/tasks/uipath-maestro-flow/evaluate/evaluator_type_choice.yaml index 1d76b6d79d..e033682617 100644 --- a/tests/tasks/uipath-maestro-flow/evaluate/evaluator_type_choice.yaml +++ b/tests/tasks/uipath-maestro-flow/evaluate/evaluator_type_choice.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-eval-evaluator-type-choice description: > Smoke test: agent is given three evaluation goals and must pick the correct diff --git a/tests/tasks/uipath-maestro-flow/evaluate/inline_agent_eval/inline_agent_eval.yaml b/tests/tasks/uipath-maestro-flow/evaluate/inline_agent_eval/inline_agent_eval.yaml index d0e1217885..b4fd796b0c 100644 --- a/tests/tasks/uipath-maestro-flow/evaluate/inline_agent_eval/inline_agent_eval.yaml +++ b/tests/tasks/uipath-maestro-flow/evaluate/inline_agent_eval/inline_agent_eval.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-eval-inline-agent description: > Skill-guided evaluation: agent uses the uipath-maestro-flow skill to build a diff --git a/tests/tasks/uipath-maestro-flow/evaluate/local_crud.yaml b/tests/tasks/uipath-maestro-flow/evaluate/local_crud.yaml index 9b180a872b..033171651d 100644 --- a/tests/tasks/uipath-maestro-flow/evaluate/local_crud.yaml +++ b/tests/tasks/uipath-maestro-flow/evaluate/local_crud.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-eval-local-crud description: > Skill-guided evaluation: agent uses the uipath-maestro-flow skill's evaluate diff --git a/tests/tasks/uipath-maestro-flow/evaluate/no_auto_upload.yaml b/tests/tasks/uipath-maestro-flow/evaluate/no_auto_upload.yaml index 0eebb47918..59210de644 100644 --- a/tests/tasks/uipath-maestro-flow/evaluate/no_auto_upload.yaml +++ b/tests/tasks/uipath-maestro-flow/evaluate/no_auto_upload.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-eval-no-auto-upload description: > Smoke test (anti-pattern guard): agent is asked to "make the eval run work" diff --git a/tests/tasks/uipath-maestro-flow/evaluate/simulation/simulation_crud.yaml b/tests/tasks/uipath-maestro-flow/evaluate/simulation/simulation_crud.yaml index 950236f92b..5849a5b7ae 100644 --- a/tests/tasks/uipath-maestro-flow/evaluate/simulation/simulation_crud.yaml +++ b/tests/tasks/uipath-maestro-flow/evaluate/simulation/simulation_crud.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-eval-simulation-crud description: > Skill-guided simulation CRUD: agent uses the uipath-maestro-flow skill's diff --git a/tests/tasks/uipath-maestro-flow/hitl/quality_01_schema_design.yaml b/tests/tasks/uipath-maestro-flow/hitl/quality_01_schema_design.yaml index 35055f356b..5a0d50ea64 100644 --- a/tests/tasks/uipath-maestro-flow/hitl/quality_01_schema_design.yaml +++ b/tests/tasks/uipath-maestro-flow/hitl/quality_01_schema_design.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-hitl-quality-schema-design description: > Quality test: agent correctly maps a business description to a quickform diff --git a/tests/tasks/uipath-maestro-flow/hitl/quality_02_result_downstream.yaml b/tests/tasks/uipath-maestro-flow/hitl/quality_02_result_downstream.yaml index ca1c9bf468..7dee217acc 100644 --- a/tests/tasks/uipath-maestro-flow/hitl/quality_02_result_downstream.yaml +++ b/tests/tasks/uipath-maestro-flow/hitl/quality_02_result_downstream.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-hitl-quality-result-downstream description: > Quality test: agent correctly references the HITL node's output via diff --git a/tests/tasks/uipath-maestro-flow/hitl/quality_03_boolean_decision.yaml b/tests/tasks/uipath-maestro-flow/hitl/quality_03_boolean_decision.yaml index a9bc305813..5e85600dc3 100644 --- a/tests/tasks/uipath-maestro-flow/hitl/quality_03_boolean_decision.yaml +++ b/tests/tasks/uipath-maestro-flow/hitl/quality_03_boolean_decision.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-hitl-quality-boolean-decision description: > Quality test: agent correctly wires a boolean HITL output field into a diff --git a/tests/tasks/uipath-maestro-flow/hitl/quality_04_brownfield_insert.yaml b/tests/tasks/uipath-maestro-flow/hitl/quality_04_brownfield_insert.yaml index 53d1f521de..39a752d0c9 100644 --- a/tests/tasks/uipath-maestro-flow/hitl/quality_04_brownfield_insert.yaml +++ b/tests/tasks/uipath-maestro-flow/hitl/quality_04_brownfield_insert.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-hitl-quality-brownfield-insert description: > Quality test: agent inserts a HITL node into an existing flow without diff --git a/tests/tasks/uipath-maestro-flow/hitl/smoke_01_hitl_node_placed.yaml b/tests/tasks/uipath-maestro-flow/hitl/smoke_01_hitl_node_placed.yaml index 7d3dd116a8..2d0fe30c5b 100644 --- a/tests/tasks/uipath-maestro-flow/hitl/smoke_01_hitl_node_placed.yaml +++ b/tests/tasks/uipath-maestro-flow/hitl/smoke_01_hitl_node_placed.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-hitl-smoke-node-placed description: > Smoke test: agent builds a simple invoice approval flow containing an inline diff --git a/tests/tasks/uipath-maestro-flow/hitl/smoke_02_completed_port_wired.yaml b/tests/tasks/uipath-maestro-flow/hitl/smoke_02_completed_port_wired.yaml index 2e89626275..d577abf859 100644 --- a/tests/tasks/uipath-maestro-flow/hitl/smoke_02_completed_port_wired.yaml +++ b/tests/tasks/uipath-maestro-flow/hitl/smoke_02_completed_port_wired.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-hitl-smoke-completed-port description: > Smoke test: agent wires the HITL node's `outcome-completed` output port diff --git a/tests/tasks/uipath-maestro-flow/hitl/smoke_03_multi_outcome_routing.yaml b/tests/tasks/uipath-maestro-flow/hitl/smoke_03_multi_outcome_routing.yaml index c12768837d..f137b09ca4 100644 --- a/tests/tasks/uipath-maestro-flow/hitl/smoke_03_multi_outcome_routing.yaml +++ b/tests/tasks/uipath-maestro-flow/hitl/smoke_03_multi_outcome_routing.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-hitl-smoke-multi-outcome-routing description: > Smoke test: agent builds a flow where a Decision node reads the HITL diff --git a/tests/tasks/uipath-maestro-flow/interactive/bellevue_weather_simulated/bellevue_weather_simulated.yaml b/tests/tasks/uipath-maestro-flow/interactive/bellevue_weather_simulated/bellevue_weather_simulated.yaml index 1498ba5949..7170613000 100644 --- a/tests/tasks/uipath-maestro-flow/interactive/bellevue_weather_simulated/bellevue_weather_simulated.yaml +++ b/tests/tasks/uipath-maestro-flow/interactive/bellevue_weather_simulated/bellevue_weather_simulated.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-bellevue-weather-simulated description: > Bellevue weather flow (HTTP โ†’ script โ†’ decision), but driven by a simulated diff --git a/tests/tasks/uipath-maestro-flow/interactive/cli_dice_roller_simulated/cli_dice_roller_simulated.yaml b/tests/tasks/uipath-maestro-flow/interactive/cli_dice_roller_simulated/cli_dice_roller_simulated.yaml index dc759b052b..ed0cfc8f33 100644 --- a/tests/tasks/uipath-maestro-flow/interactive/cli_dice_roller_simulated/cli_dice_roller_simulated.yaml +++ b/tests/tasks/uipath-maestro-flow/interactive/cli_dice_roller_simulated/cli_dice_roller_simulated.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-cli-dice-roller-simulated description: > Dice-roller flow in interactive mode, driven by a simulated non-technical user diff --git a/tests/tasks/uipath-maestro-flow/interactive/customer_escalation_simulated/customer_escalation_simulated.yaml b/tests/tasks/uipath-maestro-flow/interactive/customer_escalation_simulated/customer_escalation_simulated.yaml index ad8208940e..0dd237efb8 100644 --- a/tests/tasks/uipath-maestro-flow/interactive/customer_escalation_simulated/customer_escalation_simulated.yaml +++ b/tests/tasks/uipath-maestro-flow/interactive/customer_escalation_simulated/customer_escalation_simulated.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-customer-escalation-simulated description: > Multi-branch Outlookโ†’classifyโ†’decision escalation flow, but driven by a diff --git a/tests/tasks/uipath-maestro-flow/interactive/customer_escalation_triage/customer_escalation_triage.yaml b/tests/tasks/uipath-maestro-flow/interactive/customer_escalation_triage/customer_escalation_triage.yaml index 8a2da15031..7f9012eb8a 100644 --- a/tests/tasks/uipath-maestro-flow/interactive/customer_escalation_triage/customer_escalation_triage.yaml +++ b/tests/tasks/uipath-maestro-flow/interactive/customer_escalation_triage/customer_escalation_triage.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-interactive-customer-escalation-triage description: > Interactive end-to-end Flow evaluation. A simulated support-operations diff --git a/tests/tasks/uipath-maestro-flow/interactive/expense_approval_simulated/expense_approval_simulated.yaml b/tests/tasks/uipath-maestro-flow/interactive/expense_approval_simulated/expense_approval_simulated.yaml index 8c9541cbcc..1ca2662735 100644 --- a/tests/tasks/uipath-maestro-flow/interactive/expense_approval_simulated/expense_approval_simulated.yaml +++ b/tests/tasks/uipath-maestro-flow/interactive/expense_approval_simulated/expense_approval_simulated.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-expense-approval-simulated description: > Expense-approval flow with an inline HITL review step (trigger โ†’ script โ†’ diff --git a/tests/tasks/uipath-maestro-flow/interactive/hitl_schema_design_simulated/hitl_schema_design_simulated.yaml b/tests/tasks/uipath-maestro-flow/interactive/hitl_schema_design_simulated/hitl_schema_design_simulated.yaml index 28617e906c..07daf7555a 100644 --- a/tests/tasks/uipath-maestro-flow/interactive/hitl_schema_design_simulated/hitl_schema_design_simulated.yaml +++ b/tests/tasks/uipath-maestro-flow/interactive/hitl_schema_design_simulated/hitl_schema_design_simulated.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-hitl-schema-design-simulated description: > Purchase-order review flow with an inline HITL quickform (trigger โ†’ script โ†’ diff --git a/tests/tasks/uipath-maestro-flow/interactive/ixp_invoice_extraction_simulated/ixp_invoice_extraction_simulated.yaml b/tests/tasks/uipath-maestro-flow/interactive/ixp_invoice_extraction_simulated/ixp_invoice_extraction_simulated.yaml index 8b129c81ae..3be74f21c2 100644 --- a/tests/tasks/uipath-maestro-flow/interactive/ixp_invoice_extraction_simulated/ixp_invoice_extraction_simulated.yaml +++ b/tests/tasks/uipath-maestro-flow/interactive/ixp_invoice_extraction_simulated/ixp_invoice_extraction_simulated.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-ixp-invoice-extraction-simulated description: > Invoice-processing flow (SharePoint trigger โ†’ IxP extraction โ†’ HTTP POST to diff --git a/tests/tasks/uipath-maestro-flow/interactive/slack_channel_description_simulated/slack_channel_description_simulated.yaml b/tests/tasks/uipath-maestro-flow/interactive/slack_channel_description_simulated/slack_channel_description_simulated.yaml index 422667cf98..21efc37fe2 100644 --- a/tests/tasks/uipath-maestro-flow/interactive/slack_channel_description_simulated/slack_channel_description_simulated.yaml +++ b/tests/tasks/uipath-maestro-flow/interactive/slack_channel_description_simulated/slack_channel_description_simulated.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-slack-channel-description-simulated description: > Single-connector Slack flow (read a channel's description and output it), diff --git a/tests/tasks/uipath-maestro-flow/interactive/solution_select.yaml b/tests/tasks/uipath-maestro-flow/interactive/solution_select.yaml index 8a50c5af48..4d6ff789cb 100644 --- a/tests/tasks/uipath-maestro-flow/interactive/solution_select.yaml +++ b/tests/tasks/uipath-maestro-flow/interactive/solution_select.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-solution-select-ask description: > Interactive-mode variant of init-validate. The working directory already diff --git a/tests/tasks/uipath-maestro-flow/ixp/e2e_02_project_selection.yaml b/tests/tasks/uipath-maestro-flow/ixp/e2e_02_project_selection.yaml index f266379c20..fa70d362b5 100644 --- a/tests/tasks/uipath-maestro-flow/ixp/e2e_02_project_selection.yaml +++ b/tests/tasks/uipath-maestro-flow/ixp/e2e_02_project_selection.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-ixp-e2e-project-selection description: > E2E (project selection): on a tenant where the demo IxP project set is diff --git a/tests/tasks/uipath-maestro-flow/ixp/integration_handle_routing.yaml b/tests/tasks/uipath-maestro-flow/ixp/integration_handle_routing.yaml index d107dbdd3a..d8f1008ed5 100644 --- a/tests/tasks/uipath-maestro-flow/ixp/integration_handle_routing.yaml +++ b/tests/tasks/uipath-maestro-flow/ixp/integration_handle_routing.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-ixp-integration-handle-routing description: > Integration: IxP extraction wired into a Decision node that routes on a diff --git a/tests/tasks/uipath-maestro-flow/ixp/routing.yaml b/tests/tasks/uipath-maestro-flow/ixp/routing.yaml index a5c92eb7a1..4779bf7c0f 100644 --- a/tests/tasks/uipath-maestro-flow/ixp/routing.yaml +++ b/tests/tasks/uipath-maestro-flow/ixp/routing.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-ixp-routing description: > IxP routing (positive) โ€” document-extraction build prompts fanned out via diff --git a/tests/tasks/uipath-maestro-flow/ixp/routing_listing.yaml b/tests/tasks/uipath-maestro-flow/ixp/routing_listing.yaml index 12fd2c29c6..9a04555225 100644 --- a/tests/tasks/uipath-maestro-flow/ixp/routing_listing.yaml +++ b/tests/tasks/uipath-maestro-flow/ixp/routing_listing.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-ixp-routing-listing description: > IxP listing routing โ€” user asks what IxP models / runtime projects are diff --git a/tests/tasks/uipath-maestro-flow/ixp/routing_negative.yaml b/tests/tasks/uipath-maestro-flow/ixp/routing_negative.yaml index f06f275792..5581d242bf 100644 --- a/tests/tasks/uipath-maestro-flow/ixp/routing_negative.yaml +++ b/tests/tasks/uipath-maestro-flow/ixp/routing_negative.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-ixp-routing-negative description: > Negative IxP routing โ€” verifies the agent does NOT add a uipath.ixp.* diff --git a/tests/tasks/uipath-maestro-flow/ixp/scaffold_minimal.yaml b/tests/tasks/uipath-maestro-flow/ixp/scaffold_minimal.yaml index b08463036e..9c7274da59 100644 --- a/tests/tasks/uipath-maestro-flow/ixp/scaffold_minimal.yaml +++ b/tests/tasks/uipath-maestro-flow/ixp/scaffold_minimal.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-ixp-scaffold-minimal description: > Integration: minimal scaffold โ€” manual trigger โ†’ IxP extract โ†’ script (logs "ok") diff --git a/tests/tasks/uipath-maestro-flow/ixp/scaffold_multinode.yaml b/tests/tasks/uipath-maestro-flow/ixp/scaffold_multinode.yaml index 4b1337770a..e8883126a3 100644 --- a/tests/tasks/uipath-maestro-flow/ixp/scaffold_multinode.yaml +++ b/tests/tasks/uipath-maestro-flow/ixp/scaffold_multinode.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-ixp-scaffold-multinode description: > Integration: multi-script fan-out from an IxP extraction โ€” manual trigger โ†’ diff --git a/tests/tasks/uipath-maestro-flow/multi_node/bellevue_weather/bellevue_weather.yaml b/tests/tasks/uipath-maestro-flow/multi_node/bellevue_weather/bellevue_weather.yaml index 0680ae7e63..9f4828a202 100644 --- a/tests/tasks/uipath-maestro-flow/multi_node/bellevue_weather/bellevue_weather.yaml +++ b/tests/tasks/uipath-maestro-flow/multi_node/bellevue_weather/bellevue_weather.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-bellevue-weather description: > Create a UiPath Flow that fetches today's weather in Bellevue from open-meteo, diff --git a/tests/tasks/uipath-maestro-flow/multi_node/calculator/calculator.yaml b/tests/tasks/uipath-maestro-flow/multi_node/calculator/calculator.yaml index 239b1b495a..3d0c3f5b86 100644 --- a/tests/tasks/uipath-maestro-flow/multi_node/calculator/calculator.yaml +++ b/tests/tasks/uipath-maestro-flow/multi_node/calculator/calculator.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-calculator description: > Create a UiPath Flow that takes two number inputs and calculates their product diff --git a/tests/tasks/uipath-maestro-flow/multi_node/customer_escalation/customer_escalation.yaml b/tests/tasks/uipath-maestro-flow/multi_node/customer_escalation/customer_escalation.yaml index e777656912..545d4319d6 100644 --- a/tests/tasks/uipath-maestro-flow/multi_node/customer_escalation/customer_escalation.yaml +++ b/tests/tasks/uipath-maestro-flow/multi_node/customer_escalation/customer_escalation.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-customer-escalation description: > Complex multi-branch escalation flow: Outlook email-received trigger โ†’ diff --git a/tests/tasks/uipath-maestro-flow/multi_node/dice_roller/dice_roller.yaml b/tests/tasks/uipath-maestro-flow/multi_node/dice_roller/dice_roller.yaml index 128e786a79..e82351795e 100644 --- a/tests/tasks/uipath-maestro-flow/multi_node/dice_roller/dice_roller.yaml +++ b/tests/tasks/uipath-maestro-flow/multi_node/dice_roller/dice_roller.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-dice-roller description: > Create a UiPath Flow from scratch that simulates rolling a fair six-sided die. diff --git a/tests/tasks/uipath-maestro-flow/multi_node/feet_inches/feet_inches.yaml b/tests/tasks/uipath-maestro-flow/multi_node/feet_inches/feet_inches.yaml index 4653595568..fa009818fd 100644 --- a/tests/tasks/uipath-maestro-flow/multi_node/feet_inches/feet_inches.yaml +++ b/tests/tasks/uipath-maestro-flow/multi_node/feet_inches/feet_inches.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-feet-inches description: > Create a UiPath Flow that converts a value between feet and inches based on diff --git a/tests/tasks/uipath-maestro-flow/multi_node/loop_multiply/loop_multiply.yaml b/tests/tasks/uipath-maestro-flow/multi_node/loop_multiply/loop_multiply.yaml index 23d97a8376..e3dbda4f21 100644 --- a/tests/tasks/uipath-maestro-flow/multi_node/loop_multiply/loop_multiply.yaml +++ b/tests/tasks/uipath-maestro-flow/multi_node/loop_multiply/loop_multiply.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-loop-multiply description: > Create a UiPath Flow that uses a Loop node to multiply the numbers diff --git a/tests/tasks/uipath-maestro-flow/multi_node/multi_city_weather/multi_city_weather.yaml b/tests/tasks/uipath-maestro-flow/multi_node/multi_city_weather/multi_city_weather.yaml index f989c15b0f..996d17370b 100644 --- a/tests/tasks/uipath-maestro-flow/multi_node/multi_city_weather/multi_city_weather.yaml +++ b/tests/tasks/uipath-maestro-flow/multi_node/multi_city_weather/multi_city_weather.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-multi-city-weather description: > Loop over 3 cities, fetch weather from open-meteo for each, classify warm/cold diff --git a/tests/tasks/uipath-maestro-flow/multi_node/reading_list/reading_list.yaml b/tests/tasks/uipath-maestro-flow/multi_node/reading_list/reading_list.yaml index 9c3e8fe1c1..9c0d24214c 100644 --- a/tests/tasks/uipath-maestro-flow/multi_node/reading_list/reading_list.yaml +++ b/tests/tasks/uipath-maestro-flow/multi_node/reading_list/reading_list.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-reading-list description: > Create a UiPath Flow that curates a reading list from a catalog of math/ML/stats diff --git a/tests/tasks/uipath-maestro-flow/multi_node/slack_channel_description/slack_channel_description.yaml b/tests/tasks/uipath-maestro-flow/multi_node/slack_channel_description/slack_channel_description.yaml index 1e704928c7..2f67e8bf1f 100644 --- a/tests/tasks/uipath-maestro-flow/multi_node/slack_channel_description/slack_channel_description.yaml +++ b/tests/tasks/uipath-maestro-flow/multi_node/slack_channel_description/slack_channel_description.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-slack-channel-description description: > Create a UiPath Flow that uses the Slack IS connector to retrieve the diff --git a/tests/tasks/uipath-maestro-flow/multi_node/slack_weather_pipeline/slack_weather_pipeline.yaml b/tests/tasks/uipath-maestro-flow/multi_node/slack_weather_pipeline/slack_weather_pipeline.yaml index 833fa90098..35009e15b3 100644 --- a/tests/tasks/uipath-maestro-flow/multi_node/slack_weather_pipeline/slack_weather_pipeline.yaml +++ b/tests/tasks/uipath-maestro-flow/multi_node/slack_weather_pipeline/slack_weather_pipeline.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-slack-weather-pipeline description: > Multi-step pipeline: read Slack channel description, extract city with a script, diff --git a/tests/tasks/uipath-maestro-flow/multi_node/wiki_pageviews/wiki_pageviews.yaml b/tests/tasks/uipath-maestro-flow/multi_node/wiki_pageviews/wiki_pageviews.yaml index 64f339af92..ec91269704 100644 --- a/tests/tasks/uipath-maestro-flow/multi_node/wiki_pageviews/wiki_pageviews.yaml +++ b/tests/tasks/uipath-maestro-flow/multi_node/wiki_pageviews/wiki_pageviews.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-wiki-pageviews description: > Create a UiPath Flow that fetches a range of Wikipedia pageviews, keeps diff --git a/tests/tasks/uipath-maestro-flow/single_node/api_workflow/api_workflow.yaml b/tests/tasks/uipath-maestro-flow/single_node/api_workflow/api_workflow.yaml index 7d478d5f51..5804d3e46f 100644 --- a/tests/tasks/uipath-maestro-flow/single_node/api_workflow/api_workflow.yaml +++ b/tests/tasks/uipath-maestro-flow/single_node/api_workflow/api_workflow.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-api-workflow description: > Create a UiPath Flow that invokes the name-to-age API workflow with the diff --git a/tests/tasks/uipath-maestro-flow/single_node/decision/decision.yaml b/tests/tasks/uipath-maestro-flow/single_node/decision/decision.yaml index 52741cbf68..0ec5b5c136 100644 --- a/tests/tasks/uipath-maestro-flow/single_node/decision/decision.yaml +++ b/tests/tasks/uipath-maestro-flow/single_node/decision/decision.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-decision description: > Create a UiPath Flow that takes a temperature in Fahrenheit and uses a Decision diff --git a/tests/tasks/uipath-maestro-flow/single_node/delay/delay.yaml b/tests/tasks/uipath-maestro-flow/single_node/delay/delay.yaml index da0784eaa3..042a889475 100644 --- a/tests/tasks/uipath-maestro-flow/single_node/delay/delay.yaml +++ b/tests/tasks/uipath-maestro-flow/single_node/delay/delay.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-delay description: > Create a UiPath Flow with a single OOTB Delay node (`core.logic.delay`) diff --git a/tests/tasks/uipath-maestro-flow/single_node/file_attachment/file_attachment.yaml b/tests/tasks/uipath-maestro-flow/single_node/file_attachment/file_attachment.yaml index 6a49769da8..6caa5d0ecc 100644 --- a/tests/tasks/uipath-maestro-flow/single_node/file_attachment/file_attachment.yaml +++ b/tests/tasks/uipath-maestro-flow/single_node/file_attachment/file_attachment.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-file-attachment-debug description: > Build a Flow whose trigger exposes a file-typed input variable, read the diff --git a/tests/tasks/uipath-maestro-flow/single_node/lowcode_agent/lowcode_agent.yaml b/tests/tasks/uipath-maestro-flow/single_node/lowcode_agent/lowcode_agent.yaml index 6fe44d88e7..4f9f15d7bb 100644 --- a/tests/tasks/uipath-maestro-flow/single_node/lowcode_agent/lowcode_agent.yaml +++ b/tests/tasks/uipath-maestro-flow/single_node/lowcode_agent/lowcode_agent.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-lowcode-agent description: > Create a UiPath Flow that wires in the existing CountLetters low-code agent diff --git a/tests/tasks/uipath-maestro-flow/single_node/openmeteo_weather/openmeteo_weather.yaml b/tests/tasks/uipath-maestro-flow/single_node/openmeteo_weather/openmeteo_weather.yaml index 1bdc369e9c..85a60454a7 100644 --- a/tests/tasks/uipath-maestro-flow/single_node/openmeteo_weather/openmeteo_weather.yaml +++ b/tests/tasks/uipath-maestro-flow/single_node/openmeteo_weather/openmeteo_weather.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-openmeteo-weather description: > End-to-end: build a Flow whose process fetches the CURRENT weather in Bellevue diff --git a/tests/tasks/uipath-maestro-flow/single_node/outlook_trigger_inbox/outlook_trigger_inbox.yaml b/tests/tasks/uipath-maestro-flow/single_node/outlook_trigger_inbox/outlook_trigger_inbox.yaml index f17f40b6f7..1234e24527 100644 --- a/tests/tasks/uipath-maestro-flow/single_node/outlook_trigger_inbox/outlook_trigger_inbox.yaml +++ b/tests/tasks/uipath-maestro-flow/single_node/outlook_trigger_inbox/outlook_trigger_inbox.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-outlook-trigger-inbox description: > Regression test for PR #348: verifies the agent freshly resolves the diff --git a/tests/tasks/uipath-maestro-flow/single_node/outlook_waitfor_email/outlook_waitfor_email.yaml b/tests/tasks/uipath-maestro-flow/single_node/outlook_waitfor_email/outlook_waitfor_email.yaml index 8dc9c97eeb..958b616dd4 100644 --- a/tests/tasks/uipath-maestro-flow/single_node/outlook_waitfor_email/outlook_waitfor_email.yaml +++ b/tests/tasks/uipath-maestro-flow/single_node/outlook_waitfor_email/outlook_waitfor_email.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-outlook-waitfor-email description: > Build-and-validate: a Flow with a manual start trigger, a mid-flow diff --git a/tests/tasks/uipath-maestro-flow/single_node/rpa/rpa.yaml b/tests/tasks/uipath-maestro-flow/single_node/rpa/rpa.yaml index 77bfdda93b..67a853d15b 100644 --- a/tests/tasks/uipath-maestro-flow/single_node/rpa/rpa.yaml +++ b/tests/tasks/uipath-maestro-flow/single_node/rpa/rpa.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-rpa description: > Create a UiPath Flow that uses the HelpDeskLookup RPA workflow to retrieve diff --git a/tests/tasks/uipath-maestro-flow/single_node/subflow/subflow.yaml b/tests/tasks/uipath-maestro-flow/single_node/subflow/subflow.yaml index 1713f9b752..fab2781f5c 100644 --- a/tests/tasks/uipath-maestro-flow/single_node/subflow/subflow.yaml +++ b/tests/tasks/uipath-maestro-flow/single_node/subflow/subflow.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-subflow description: > Create a UiPath Flow that uses a Subflow node to encapsulate string-reversal diff --git a/tests/tasks/uipath-maestro-flow/single_node/switch/switch.yaml b/tests/tasks/uipath-maestro-flow/single_node/switch/switch.yaml index 30828035a9..77afb89cbe 100644 --- a/tests/tasks/uipath-maestro-flow/single_node/switch/switch.yaml +++ b/tests/tasks/uipath-maestro-flow/single_node/switch/switch.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-switch description: > Create a UiPath Flow that takes a quarter number (1-4) and uses a Switch node diff --git a/tests/tasks/uipath-maestro-flow/single_node/terminate/terminate.yaml b/tests/tasks/uipath-maestro-flow/single_node/terminate/terminate.yaml index 366e44789e..15f8787c27 100644 --- a/tests/tasks/uipath-maestro-flow/single_node/terminate/terminate.yaml +++ b/tests/tasks/uipath-maestro-flow/single_node/terminate/terminate.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-terminate description: > Create a UiPath Flow with two parallel branches from the trigger. One branch diff --git a/tests/tasks/uipath-maestro-flow/single_node/transform_filter/transform_filter.yaml b/tests/tasks/uipath-maestro-flow/single_node/transform_filter/transform_filter.yaml index 1647384f26..59b23d26c3 100644 --- a/tests/tasks/uipath-maestro-flow/single_node/transform_filter/transform_filter.yaml +++ b/tests/tasks/uipath-maestro-flow/single_node/transform_filter/transform_filter.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-transform-filter description: > Create a UiPath Flow that uses a dedicated `core.action.transform.filter` diff --git a/tests/tasks/uipath-maestro-flow/single_node/transform_group_by/transform_group_by.yaml b/tests/tasks/uipath-maestro-flow/single_node/transform_group_by/transform_group_by.yaml index 20161257c6..a93ce655e4 100644 --- a/tests/tasks/uipath-maestro-flow/single_node/transform_group_by/transform_group_by.yaml +++ b/tests/tasks/uipath-maestro-flow/single_node/transform_group_by/transform_group_by.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-transform-group-by description: > Create a UiPath Flow with a single Group By transform node diff --git a/tests/tasks/uipath-maestro-flow/single_node/transform_map/transform_map.yaml b/tests/tasks/uipath-maestro-flow/single_node/transform_map/transform_map.yaml index 6dfc758cef..84e8ce768f 100644 --- a/tests/tasks/uipath-maestro-flow/single_node/transform_map/transform_map.yaml +++ b/tests/tasks/uipath-maestro-flow/single_node/transform_map/transform_map.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-transform-map description: > Smoke test: agent builds a pure-OOTB UiPath Flow with a single diff --git a/tests/tasks/uipath-maestro-flow/smoke/init_validate.yaml b/tests/tasks/uipath-maestro-flow/smoke/init_validate.yaml index 8c26381441..c40da3f960 100644 --- a/tests/tasks/uipath-maestro-flow/smoke/init_validate.yaml +++ b/tests/tasks/uipath-maestro-flow/smoke/init_validate.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-init-validate description: > Skill-guided evaluation: agent creates a new UiPath Flow project and diff --git a/tests/tasks/uipath-maestro-flow/smoke/inline_agent_robust.yaml b/tests/tasks/uipath-maestro-flow/smoke/inline_agent_robust.yaml index bd65c34ca7..4277929649 100644 --- a/tests/tasks/uipath-maestro-flow/smoke/inline_agent_robust.yaml +++ b/tests/tasks/uipath-maestro-flow/smoke/inline_agent_robust.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-inline-agent-robust description: > Skill-guided evaluation: agent uses the uipath-maestro-flow skill to build a diff --git a/tests/tasks/uipath-maestro-flow/smoke/merge_parallel_sync.yaml b/tests/tasks/uipath-maestro-flow/smoke/merge_parallel_sync.yaml index e3687a305c..f144f78dd8 100644 --- a/tests/tasks/uipath-maestro-flow/smoke/merge_parallel_sync.yaml +++ b/tests/tasks/uipath-maestro-flow/smoke/merge_parallel_sync.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-merge-parallel-sync description: > Build a UiPath Flow with two parallel branches that fork from the trigger and diff --git a/tests/tasks/uipath-maestro-flow/smoke/registry_discovery.yaml b/tests/tasks/uipath-maestro-flow/smoke/registry_discovery.yaml index 3c7426d74a..5ba1a9a65e 100644 --- a/tests/tasks/uipath-maestro-flow/smoke/registry_discovery.yaml +++ b/tests/tasks/uipath-maestro-flow/smoke/registry_discovery.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-registry-discovery description: > Skill-guided evaluation: agent uses the uipath-maestro-flow skill to explore diff --git a/tests/tasks/uipath-maestro-flow/smoke/scheduled_trigger.yaml b/tests/tasks/uipath-maestro-flow/smoke/scheduled_trigger.yaml index b27517d4c8..a6f71e6479 100644 --- a/tests/tasks/uipath-maestro-flow/smoke/scheduled_trigger.yaml +++ b/tests/tasks/uipath-maestro-flow/smoke/scheduled_trigger.yaml @@ -1,7 +1,6 @@ # Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral # v1-base prompt, outcome-graded criteria at preserved weights, and # arm-agnostic artifact discovery. -# See workspaces/docs/flow-sdk/2026-08-21-maestro-flow-alignment-expansion-plan.md. task_id: skill-flow-scheduled-trigger description: > Scaffold a UiPath Flow whose start node is a Scheduled Trigger From 7fd1a1cd8527ceba977b1befc682ab799a2f0abe Mon Sep 17 00:00:00 2001 From: Tao Ma <51425734+tmatup@users.noreply.github.com> Date: Thu, 3 Sep 2026 14:40:57 -0700 Subject: [PATCH 50/52] test(maestro-flow): pagination gate also accepts the uip maestro registry prepare form (#3047) The corpus test named the SDK resolver only as npx flow-sdk registry prepare; the CLI exposes the same resolver as uip maestro registry prepare (UiPath/cli#3969, top-level maestro registry, not maestro flow registry). Assert the yaml pattern matches that form too. Claude-Session: https://claude.ai/code/session_01UauCCMBpJ7ws1meQwBfYSS Co-authored-by: Claude Fable 5.1 --- .../uipath-maestro-flow/_shared/test_same_ground_corpus.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/tasks/uipath-maestro-flow/_shared/test_same_ground_corpus.py b/tests/tasks/uipath-maestro-flow/_shared/test_same_ground_corpus.py index 934efdb0f0..95d76f0481 100644 --- a/tests/tasks/uipath-maestro-flow/_shared/test_same_ground_corpus.py +++ b/tests/tasks/uipath-maestro-flow/_shared/test_same_ground_corpus.py @@ -237,6 +237,9 @@ def test_paginated_lookup_accepts_both_supported_resolution_routes() -> None: "npx flow-sdk registry prepare uipath-salesforce-slack " "send-message-to-channel --resolve channel:name=simple" ) + # The CLI wraps the same resolver as `uip maestro registry prepare` + # (UiPath/cli#3969) โ€” top-level `maestro registry`, not `maestro flow registry`. + uip_command = "uip maestro " + sdk_command[len("npx flow-sdk "):] # The v1 route is telemetry the SDK arm never produces: advisory, but it # still asks for the loop (page 1 alone is not pagination). @@ -250,6 +253,7 @@ def test_paginated_lookup_accepts_both_supported_resolution_routes() -> None: assert "require_success: true" in paged_block and "pass_threshold: 1.0" in paged_block assert paged_pattern.search(page_two) assert paged_pattern.search(sdk_command) + assert paged_pattern.search(uip_command) and not manual_pattern.search(uip_command) assert not paged_pattern.search(page_one) assert not paged_pattern.search("uip maestro flow validate --resolve channel:name=simple") assert not paged_pattern.search("echo nextPage=token") From 94ceb9a4a71fcbfa66bfa213d3fb7790ebb9434a Mon Sep 17 00:00:00 2001 From: Tao Ma <51425734+tmatup@users.noreply.github.com> Date: Thu, 3 Sep 2026 18:56:57 -0700 Subject: [PATCH 51/52] =?UTF-8?q?fix(preview):=20make=20preview/=20a=20rea?= =?UTF-8?q?l=20plugin=20root=20(skills/=20+=20.claude-plugin)=20=E2=80=94?= =?UTF-8?q?=20v2=20arm=20loaded=20zero=20skills=20(#3049)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(preview): make preview/ a real plugin root โ€” skills/ + .claude-plugin/plugin.json Claude Code loads a local plugin only from /skills//SKILL.md. preview/ had the three skill folders as direct children, so plugins.path=$SKILLS_REPO_PATH/preview loaded ZERO skills, silently, in every Flow v2 eval run since 08-20 (coder-eval only WARNs per task; 306 warnings in adhoc-2026-09-03_16-34-17). v2 agents fell back to node_modules/@uipath/flow-sdk/skill/SKILL.md by find / at median turn 10 (91/116), 25/116 never saw a skill doc, 9 Skill calls errored 'Unknown skill'. The Delegate SDK appends /skills to the plugin root and hit the same missing dir. - git mv preview/uipath-maestro-{flow,case,bpmn} -> preview/skills/ - add preview/.claude-plugin/plugin.json (name uipath-preview) so skills load as uipath-preview:uipath-maestro-* โ€” nothing matches on the prefix (skill_triggered strips it; preview docs never mention uipath:) - rewrite the 44 sync-script targets, its test, the sync workflow, CODEOWNERS, verb-gate glob (preview/skills/*/), tests/README.md, yaml comment - plugins.path in flow-v2-preview.yaml / same-ground-ab.yaml is unchanged Verified: node --test sync test 2/2; sync script against local flow-builder-sdk writes into preview/skills/* and passes all snapshot gates; live `claude --plugin-dir preview` lists uipath-preview:uipath-maestro-{flow,case,bpmn} (the old layout listed nothing). Follow-up: coder_eval_uipath comparison.py:101 provenance read must add the skills/ segment. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01JBM7mnPT6LzvHLCBtFXcek * fix(verb-gate): selective gate resolves preview/skills/, not preview/skills Review finding on #3049: with the deeper layout the selective-gate sed cut the changed path at preview/skills. Both branches of the gate now name the skill dir. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01JBM7mnPT6LzvHLCBtFXcek --------- Co-authored-by: Claude Fable 5.1 --- .../workflows/sync-maestro-sdk-preview.yml | 20 ++--- .github/workflows/verb-gate.yml | 6 +- CODEOWNERS | 6 +- preview/.claude-plugin/plugin.json | 9 ++ .../{ => skills}/uipath-maestro-bpmn/SKILL.md | 0 .../examples/InvoiceApproval.bpmn.ts | 0 .../examples/NotifyChannel.bpmn.ts | 0 .../uipath-maestro-bpmn/references/api.md | 0 .../references/bpmn-runtime.md | 0 .../{ => skills}/uipath-maestro-case/SKILL.md | 0 .../examples/ClaimReviewSLA.case.ts | 0 .../examples/IntakeBinding.case.ts | 0 .../examples/NightlyRollup.case.ts | 0 .../examples/NotifyOnApproval.case.ts | 0 .../examples/bindings.json | 0 .../uipath-maestro-case/references/api.md | 0 .../references/case-runtime.md | 0 .../{ => skills}/uipath-maestro-flow/SKILL.md | 0 .../examples/AquariumSafetyStop.flow.ts | 0 .../examples/ArchiveCardExtract.flow.ts | 0 .../examples/BakeOffEntryForm.flow.ts | 0 .../examples/BeeHiveLedger.flow.ts | 0 .../examples/BeltProgression.flow.ts | 0 .../examples/BirdCountLookup.flow.ts | 0 .../examples/ClubDirectory.flow.ts | 0 .../examples/ConcertSoundcheck.flow.ts | 0 .../examples/DoorbellLog.flow.ts | 0 .../examples/FestivalMapScaffold.flow.ts | 0 .../examples/FieldTripQuickForm.flow.ts | 0 .../examples/FossilCatalogEnrich.flow.ts | 0 .../examples/GallerySubmission.flow.ts | 0 .../examples/GreenhouseWatering.flow.ts | 0 .../examples/HarbourRadioLine.flow.ts | 0 .../examples/HerbariumDispatch.flow.ts | 0 .../examples/KilnReview.flow.ts | 0 .../examples/LibraryDeskChat.flow.ts | 0 .../examples/LighthouseSignal.flow.ts | 0 .../examples/MeteorShowerPages.flow.ts | 0 .../examples/NeighborhoodWalkPlanner.flow.ts | 0 .../examples/ObservatorySeeing.flow.ts | 0 .../examples/OralHistoryDigest.flow.ts | 0 .../examples/PlanetariumConfirmation.flow.ts | 0 .../examples/PlantNameAdvisor.flow.ts | 0 .../examples/PostcardCaption.flow.ts | 0 .../examples/PotteryStudioCallback.flow.ts | 0 .../examples/RecipeScaler.flow.ts | 0 .../examples/SeedPacketReader.flow.ts | 0 .../examples/TideTableConverter.flow.ts | 0 .../examples/TrailLogSummary.flow.ts | 0 .../examples/WorkshopInventory.flow.ts | 0 .../examples/bindings.json | 0 .../references/CLI-LOOP.md | 0 .../references/agent-resources.md | 0 .../uipath-maestro-flow/references/agent.md | 0 .../references/agentic-process.md | 0 .../references/api-workflow.md | 0 .../uipath-maestro-flow/references/api.md | 0 .../references/batch-transform.md | 0 .../references/bindings.md | 0 .../uipath-maestro-flow/references/branch.md | 0 .../references/brownfield.md | 0 .../references/connector-params.md | 0 .../references/conversational.md | 0 .../references/data-fabric.md | 0 .../uipath-maestro-flow/references/delay.md | 0 .../references/document-pipeline.md | 0 .../references/error-handling.md | 0 .../references/evaluate.md | 0 .../references/event-trigger.md | 0 .../references/form-trigger.md | 0 .../uipath-maestro-flow/references/hitl.md | 0 .../uipath-maestro-flow/references/http.md | 0 .../references/inline-agent.md | 0 .../uipath-maestro-flow/references/ixp.md | 0 .../uipath-maestro-flow/references/loops.md | 0 .../references/manual-trigger.md | 0 .../references/or-processes.md | 0 .../references/parallel-merge.md | 0 .../references/placeholder.md | 0 .../references/published-function.md | 0 .../uipath-maestro-flow/references/queue.md | 0 .../uipath-maestro-flow/references/return.md | 0 .../references/rpa-workflow.md | 0 .../references/scheduled-trigger.md | 0 .../uipath-maestro-flow/references/script.md | 0 .../uipath-maestro-flow/references/subflow.md | 0 .../references/summarize.md | 0 .../uipath-maestro-flow/references/switch.md | 0 .../references/terminate.md | 0 .../references/transform.md | 0 .../uipath-maestro-flow/references/voice.md | 0 scripts/sync-maestro-sdk-preview.mjs | 88 +++++++++---------- tests/README.md | 14 ++- tests/experiments/flow-v2-preview.yaml | 1 + .../scripts/sync-maestro-sdk-preview.test.mjs | 52 +++++------ 95 files changed, 106 insertions(+), 90 deletions(-) create mode 100644 preview/.claude-plugin/plugin.json rename preview/{ => skills}/uipath-maestro-bpmn/SKILL.md (100%) rename preview/{ => skills}/uipath-maestro-bpmn/examples/InvoiceApproval.bpmn.ts (100%) rename preview/{ => skills}/uipath-maestro-bpmn/examples/NotifyChannel.bpmn.ts (100%) rename preview/{ => skills}/uipath-maestro-bpmn/references/api.md (100%) rename preview/{ => skills}/uipath-maestro-bpmn/references/bpmn-runtime.md (100%) rename preview/{ => skills}/uipath-maestro-case/SKILL.md (100%) rename preview/{ => skills}/uipath-maestro-case/examples/ClaimReviewSLA.case.ts (100%) rename preview/{ => skills}/uipath-maestro-case/examples/IntakeBinding.case.ts (100%) rename preview/{ => skills}/uipath-maestro-case/examples/NightlyRollup.case.ts (100%) rename preview/{ => skills}/uipath-maestro-case/examples/NotifyOnApproval.case.ts (100%) rename preview/{ => skills}/uipath-maestro-case/examples/bindings.json (100%) rename preview/{ => skills}/uipath-maestro-case/references/api.md (100%) rename preview/{ => skills}/uipath-maestro-case/references/case-runtime.md (100%) rename preview/{ => skills}/uipath-maestro-flow/SKILL.md (100%) rename preview/{ => skills}/uipath-maestro-flow/examples/AquariumSafetyStop.flow.ts (100%) rename preview/{ => skills}/uipath-maestro-flow/examples/ArchiveCardExtract.flow.ts (100%) rename preview/{ => skills}/uipath-maestro-flow/examples/BakeOffEntryForm.flow.ts (100%) rename preview/{ => skills}/uipath-maestro-flow/examples/BeeHiveLedger.flow.ts (100%) rename preview/{ => skills}/uipath-maestro-flow/examples/BeltProgression.flow.ts (100%) rename preview/{ => skills}/uipath-maestro-flow/examples/BirdCountLookup.flow.ts (100%) rename preview/{ => skills}/uipath-maestro-flow/examples/ClubDirectory.flow.ts (100%) rename preview/{ => skills}/uipath-maestro-flow/examples/ConcertSoundcheck.flow.ts (100%) rename preview/{ => skills}/uipath-maestro-flow/examples/DoorbellLog.flow.ts (100%) rename preview/{ => skills}/uipath-maestro-flow/examples/FestivalMapScaffold.flow.ts (100%) rename preview/{ => skills}/uipath-maestro-flow/examples/FieldTripQuickForm.flow.ts (100%) rename preview/{ => skills}/uipath-maestro-flow/examples/FossilCatalogEnrich.flow.ts (100%) rename preview/{ => skills}/uipath-maestro-flow/examples/GallerySubmission.flow.ts (100%) rename preview/{ => skills}/uipath-maestro-flow/examples/GreenhouseWatering.flow.ts (100%) rename preview/{ => skills}/uipath-maestro-flow/examples/HarbourRadioLine.flow.ts (100%) rename preview/{ => skills}/uipath-maestro-flow/examples/HerbariumDispatch.flow.ts (100%) rename preview/{ => skills}/uipath-maestro-flow/examples/KilnReview.flow.ts (100%) rename preview/{ => skills}/uipath-maestro-flow/examples/LibraryDeskChat.flow.ts (100%) rename preview/{ => skills}/uipath-maestro-flow/examples/LighthouseSignal.flow.ts (100%) rename preview/{ => skills}/uipath-maestro-flow/examples/MeteorShowerPages.flow.ts (100%) rename preview/{ => skills}/uipath-maestro-flow/examples/NeighborhoodWalkPlanner.flow.ts (100%) rename preview/{ => skills}/uipath-maestro-flow/examples/ObservatorySeeing.flow.ts (100%) rename preview/{ => skills}/uipath-maestro-flow/examples/OralHistoryDigest.flow.ts (100%) rename preview/{ => skills}/uipath-maestro-flow/examples/PlanetariumConfirmation.flow.ts (100%) rename preview/{ => skills}/uipath-maestro-flow/examples/PlantNameAdvisor.flow.ts (100%) rename preview/{ => skills}/uipath-maestro-flow/examples/PostcardCaption.flow.ts (100%) rename preview/{ => skills}/uipath-maestro-flow/examples/PotteryStudioCallback.flow.ts (100%) rename preview/{ => skills}/uipath-maestro-flow/examples/RecipeScaler.flow.ts (100%) rename preview/{ => skills}/uipath-maestro-flow/examples/SeedPacketReader.flow.ts (100%) rename preview/{ => skills}/uipath-maestro-flow/examples/TideTableConverter.flow.ts (100%) rename preview/{ => skills}/uipath-maestro-flow/examples/TrailLogSummary.flow.ts (100%) rename preview/{ => skills}/uipath-maestro-flow/examples/WorkshopInventory.flow.ts (100%) rename preview/{ => skills}/uipath-maestro-flow/examples/bindings.json (100%) rename preview/{ => skills}/uipath-maestro-flow/references/CLI-LOOP.md (100%) rename preview/{ => skills}/uipath-maestro-flow/references/agent-resources.md (100%) rename preview/{ => skills}/uipath-maestro-flow/references/agent.md (100%) rename preview/{ => skills}/uipath-maestro-flow/references/agentic-process.md (100%) rename preview/{ => skills}/uipath-maestro-flow/references/api-workflow.md (100%) rename preview/{ => skills}/uipath-maestro-flow/references/api.md (100%) rename preview/{ => skills}/uipath-maestro-flow/references/batch-transform.md (100%) rename preview/{ => skills}/uipath-maestro-flow/references/bindings.md (100%) rename preview/{ => skills}/uipath-maestro-flow/references/branch.md (100%) rename preview/{ => skills}/uipath-maestro-flow/references/brownfield.md (100%) rename preview/{ => skills}/uipath-maestro-flow/references/connector-params.md (100%) rename preview/{ => skills}/uipath-maestro-flow/references/conversational.md (100%) rename preview/{ => skills}/uipath-maestro-flow/references/data-fabric.md (100%) rename preview/{ => skills}/uipath-maestro-flow/references/delay.md (100%) rename preview/{ => skills}/uipath-maestro-flow/references/document-pipeline.md (100%) rename preview/{ => skills}/uipath-maestro-flow/references/error-handling.md (100%) rename preview/{ => skills}/uipath-maestro-flow/references/evaluate.md (100%) rename preview/{ => skills}/uipath-maestro-flow/references/event-trigger.md (100%) rename preview/{ => skills}/uipath-maestro-flow/references/form-trigger.md (100%) rename preview/{ => skills}/uipath-maestro-flow/references/hitl.md (100%) rename preview/{ => skills}/uipath-maestro-flow/references/http.md (100%) rename preview/{ => skills}/uipath-maestro-flow/references/inline-agent.md (100%) rename preview/{ => skills}/uipath-maestro-flow/references/ixp.md (100%) rename preview/{ => skills}/uipath-maestro-flow/references/loops.md (100%) rename preview/{ => skills}/uipath-maestro-flow/references/manual-trigger.md (100%) rename preview/{ => skills}/uipath-maestro-flow/references/or-processes.md (100%) rename preview/{ => skills}/uipath-maestro-flow/references/parallel-merge.md (100%) rename preview/{ => skills}/uipath-maestro-flow/references/placeholder.md (100%) rename preview/{ => skills}/uipath-maestro-flow/references/published-function.md (100%) rename preview/{ => skills}/uipath-maestro-flow/references/queue.md (100%) rename preview/{ => skills}/uipath-maestro-flow/references/return.md (100%) rename preview/{ => skills}/uipath-maestro-flow/references/rpa-workflow.md (100%) rename preview/{ => skills}/uipath-maestro-flow/references/scheduled-trigger.md (100%) rename preview/{ => skills}/uipath-maestro-flow/references/script.md (100%) rename preview/{ => skills}/uipath-maestro-flow/references/subflow.md (100%) rename preview/{ => skills}/uipath-maestro-flow/references/summarize.md (100%) rename preview/{ => skills}/uipath-maestro-flow/references/switch.md (100%) rename preview/{ => skills}/uipath-maestro-flow/references/terminate.md (100%) rename preview/{ => skills}/uipath-maestro-flow/references/transform.md (100%) rename preview/{ => skills}/uipath-maestro-flow/references/voice.md (100%) diff --git a/.github/workflows/sync-maestro-sdk-preview.yml b/.github/workflows/sync-maestro-sdk-preview.yml index 8c2560b9d1..69d81a1935 100644 --- a/.github/workflows/sync-maestro-sdk-preview.yml +++ b/.github/workflows/sync-maestro-sdk-preview.yml @@ -80,7 +80,7 @@ jobs: for skill in flow case bpmn; do findings="$RUNNER_TEMP/maestro-$skill-verbs.ndjson" python3 scripts/check-skill-verbs.py \ - --json "preview/uipath-maestro-$skill" > "$findings" + --json "preview/skills/uipath-maestro-$skill" > "$findings" blocking=$(jq -c \ 'select(.severity=="Stale" and .matched_prefix == null)' \ "$findings" | wc -l | tr -d ' ') @@ -108,16 +108,16 @@ jobs: run: | set -euo pipefail if git diff --quiet -- \ - preview/uipath-maestro-flow \ - preview/uipath-maestro-case \ - preview/uipath-maestro-bpmn; then + preview/skills/uipath-maestro-flow \ + preview/skills/uipath-maestro-case \ + preview/skills/uipath-maestro-bpmn; then changed=false else changed=true git diff --stat -- \ - preview/uipath-maestro-flow \ - preview/uipath-maestro-case \ - preview/uipath-maestro-bpmn + preview/skills/uipath-maestro-flow \ + preview/skills/uipath-maestro-case \ + preview/skills/uipath-maestro-bpmn fi if [ "$changed" != "$SCRIPT_CHANGED" ]; then echo "Sync script reported changed=$SCRIPT_CHANGED but git reported changed=$changed." @@ -146,9 +146,9 @@ jobs: git config user.email "41898282+github-actions[bot]@users.noreply.github.com" git checkout -b "$BRANCH" git add \ - preview/uipath-maestro-flow \ - preview/uipath-maestro-case \ - preview/uipath-maestro-bpmn + preview/skills/uipath-maestro-flow \ + preview/skills/uipath-maestro-case \ + preview/skills/uipath-maestro-bpmn git commit \ -m "chore(preview): re-sync Maestro SDK skills to flow-builder-sdk@$NEW_PIN" \ -m "๐Ÿค– Generated with Claude Code diff --git a/.github/workflows/verb-gate.yml b/.github/workflows/verb-gate.yml index 1ea283dec1..dff1c5d8f4 100644 --- a/.github/workflows/verb-gate.yml +++ b/.github/workflows/verb-gate.yml @@ -49,11 +49,11 @@ jobs: # verbs exactly like `skills/` ones, and a retired verb rots the same # way in either place. if echo "$CHANGED" | grep -qE '^(assets/uip-catalog-snapshot\.json|scripts/check-skill-verbs\.py|\.github/workflows/verb-gate\.yml)$'; then - SKILLS=$(ls -d skills/*/ preview/*/ 2>/dev/null | sed 's|/$||' | sort -u) + SKILLS=$(ls -d skills/*/ preview/skills/*/ 2>/dev/null | sed 's|/$||' | sort -u) else SKILLS=$(echo "$CHANGED" \ - | grep -E '^(skills|preview)/[^/]+/.*\.md$' \ - | sed -E 's#^((skills|preview)/[^/]+)/.*#\1#' \ + | grep -E '^(skills|preview/skills)/[^/]+/.*\.md$' \ + | sed -E 's#^((skills|preview/skills)/[^/]+)/.*#\1#' \ | sort -u || true) fi diff --git a/CODEOWNERS b/CODEOWNERS index ccb3a87ecb..6e73b886ad 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -18,9 +18,9 @@ /skill-flavors/studioweb/ @astridstaicu # preview skills -/preview/uipath-maestro-flow/ @UiPath/maestro-cli-team -/preview/uipath-maestro-case/ @dmetzgar @tmatup -/preview/uipath-maestro-bpmn/ @UiPath/maestro-cli-team +/preview/skills/uipath-maestro-flow/ @UiPath/maestro-cli-team +/preview/skills/uipath-maestro-case/ @dmetzgar @tmatup +/preview/skills/uipath-maestro-bpmn/ @UiPath/maestro-cli-team # Activity references (bundled inside uipath-rpa skill) /skills/uipath-rpa/references/activity-docs/ @DragosUnguru diff --git a/preview/.claude-plugin/plugin.json b/preview/.claude-plugin/plugin.json new file mode 100644 index 0000000000..226b25e1a3 --- /dev/null +++ b/preview/.claude-plugin/plugin.json @@ -0,0 +1,9 @@ +{ + "name": "uipath-preview", + "version": "0.1.0", + "description": "UiPath Maestro preview skills โ€” the builder-SDK authoring path for uipath-maestro-{flow,case,bpmn}. Synced from UiPath/flow-builder-sdk by scripts/sync-maestro-sdk-preview.mjs; loads as a Claude Code plugin root (skills//SKILL.md).", + "author": { "name": "UiPath" }, + "license": "MIT", + "homepage": "https://github.com/UiPath/skills", + "repository": "https://github.com/UiPath/skills" +} diff --git a/preview/uipath-maestro-bpmn/SKILL.md b/preview/skills/uipath-maestro-bpmn/SKILL.md similarity index 100% rename from preview/uipath-maestro-bpmn/SKILL.md rename to preview/skills/uipath-maestro-bpmn/SKILL.md diff --git a/preview/uipath-maestro-bpmn/examples/InvoiceApproval.bpmn.ts b/preview/skills/uipath-maestro-bpmn/examples/InvoiceApproval.bpmn.ts similarity index 100% rename from preview/uipath-maestro-bpmn/examples/InvoiceApproval.bpmn.ts rename to preview/skills/uipath-maestro-bpmn/examples/InvoiceApproval.bpmn.ts diff --git a/preview/uipath-maestro-bpmn/examples/NotifyChannel.bpmn.ts b/preview/skills/uipath-maestro-bpmn/examples/NotifyChannel.bpmn.ts similarity index 100% rename from preview/uipath-maestro-bpmn/examples/NotifyChannel.bpmn.ts rename to preview/skills/uipath-maestro-bpmn/examples/NotifyChannel.bpmn.ts diff --git a/preview/uipath-maestro-bpmn/references/api.md b/preview/skills/uipath-maestro-bpmn/references/api.md similarity index 100% rename from preview/uipath-maestro-bpmn/references/api.md rename to preview/skills/uipath-maestro-bpmn/references/api.md diff --git a/preview/uipath-maestro-bpmn/references/bpmn-runtime.md b/preview/skills/uipath-maestro-bpmn/references/bpmn-runtime.md similarity index 100% rename from preview/uipath-maestro-bpmn/references/bpmn-runtime.md rename to preview/skills/uipath-maestro-bpmn/references/bpmn-runtime.md diff --git a/preview/uipath-maestro-case/SKILL.md b/preview/skills/uipath-maestro-case/SKILL.md similarity index 100% rename from preview/uipath-maestro-case/SKILL.md rename to preview/skills/uipath-maestro-case/SKILL.md diff --git a/preview/uipath-maestro-case/examples/ClaimReviewSLA.case.ts b/preview/skills/uipath-maestro-case/examples/ClaimReviewSLA.case.ts similarity index 100% rename from preview/uipath-maestro-case/examples/ClaimReviewSLA.case.ts rename to preview/skills/uipath-maestro-case/examples/ClaimReviewSLA.case.ts diff --git a/preview/uipath-maestro-case/examples/IntakeBinding.case.ts b/preview/skills/uipath-maestro-case/examples/IntakeBinding.case.ts similarity index 100% rename from preview/uipath-maestro-case/examples/IntakeBinding.case.ts rename to preview/skills/uipath-maestro-case/examples/IntakeBinding.case.ts diff --git a/preview/uipath-maestro-case/examples/NightlyRollup.case.ts b/preview/skills/uipath-maestro-case/examples/NightlyRollup.case.ts similarity index 100% rename from preview/uipath-maestro-case/examples/NightlyRollup.case.ts rename to preview/skills/uipath-maestro-case/examples/NightlyRollup.case.ts diff --git a/preview/uipath-maestro-case/examples/NotifyOnApproval.case.ts b/preview/skills/uipath-maestro-case/examples/NotifyOnApproval.case.ts similarity index 100% rename from preview/uipath-maestro-case/examples/NotifyOnApproval.case.ts rename to preview/skills/uipath-maestro-case/examples/NotifyOnApproval.case.ts diff --git a/preview/uipath-maestro-case/examples/bindings.json b/preview/skills/uipath-maestro-case/examples/bindings.json similarity index 100% rename from preview/uipath-maestro-case/examples/bindings.json rename to preview/skills/uipath-maestro-case/examples/bindings.json diff --git a/preview/uipath-maestro-case/references/api.md b/preview/skills/uipath-maestro-case/references/api.md similarity index 100% rename from preview/uipath-maestro-case/references/api.md rename to preview/skills/uipath-maestro-case/references/api.md diff --git a/preview/uipath-maestro-case/references/case-runtime.md b/preview/skills/uipath-maestro-case/references/case-runtime.md similarity index 100% rename from preview/uipath-maestro-case/references/case-runtime.md rename to preview/skills/uipath-maestro-case/references/case-runtime.md diff --git a/preview/uipath-maestro-flow/SKILL.md b/preview/skills/uipath-maestro-flow/SKILL.md similarity index 100% rename from preview/uipath-maestro-flow/SKILL.md rename to preview/skills/uipath-maestro-flow/SKILL.md diff --git a/preview/uipath-maestro-flow/examples/AquariumSafetyStop.flow.ts b/preview/skills/uipath-maestro-flow/examples/AquariumSafetyStop.flow.ts similarity index 100% rename from preview/uipath-maestro-flow/examples/AquariumSafetyStop.flow.ts rename to preview/skills/uipath-maestro-flow/examples/AquariumSafetyStop.flow.ts diff --git a/preview/uipath-maestro-flow/examples/ArchiveCardExtract.flow.ts b/preview/skills/uipath-maestro-flow/examples/ArchiveCardExtract.flow.ts similarity index 100% rename from preview/uipath-maestro-flow/examples/ArchiveCardExtract.flow.ts rename to preview/skills/uipath-maestro-flow/examples/ArchiveCardExtract.flow.ts diff --git a/preview/uipath-maestro-flow/examples/BakeOffEntryForm.flow.ts b/preview/skills/uipath-maestro-flow/examples/BakeOffEntryForm.flow.ts similarity index 100% rename from preview/uipath-maestro-flow/examples/BakeOffEntryForm.flow.ts rename to preview/skills/uipath-maestro-flow/examples/BakeOffEntryForm.flow.ts diff --git a/preview/uipath-maestro-flow/examples/BeeHiveLedger.flow.ts b/preview/skills/uipath-maestro-flow/examples/BeeHiveLedger.flow.ts similarity index 100% rename from preview/uipath-maestro-flow/examples/BeeHiveLedger.flow.ts rename to preview/skills/uipath-maestro-flow/examples/BeeHiveLedger.flow.ts diff --git a/preview/uipath-maestro-flow/examples/BeltProgression.flow.ts b/preview/skills/uipath-maestro-flow/examples/BeltProgression.flow.ts similarity index 100% rename from preview/uipath-maestro-flow/examples/BeltProgression.flow.ts rename to preview/skills/uipath-maestro-flow/examples/BeltProgression.flow.ts diff --git a/preview/uipath-maestro-flow/examples/BirdCountLookup.flow.ts b/preview/skills/uipath-maestro-flow/examples/BirdCountLookup.flow.ts similarity index 100% rename from preview/uipath-maestro-flow/examples/BirdCountLookup.flow.ts rename to preview/skills/uipath-maestro-flow/examples/BirdCountLookup.flow.ts diff --git a/preview/uipath-maestro-flow/examples/ClubDirectory.flow.ts b/preview/skills/uipath-maestro-flow/examples/ClubDirectory.flow.ts similarity index 100% rename from preview/uipath-maestro-flow/examples/ClubDirectory.flow.ts rename to preview/skills/uipath-maestro-flow/examples/ClubDirectory.flow.ts diff --git a/preview/uipath-maestro-flow/examples/ConcertSoundcheck.flow.ts b/preview/skills/uipath-maestro-flow/examples/ConcertSoundcheck.flow.ts similarity index 100% rename from preview/uipath-maestro-flow/examples/ConcertSoundcheck.flow.ts rename to preview/skills/uipath-maestro-flow/examples/ConcertSoundcheck.flow.ts diff --git a/preview/uipath-maestro-flow/examples/DoorbellLog.flow.ts b/preview/skills/uipath-maestro-flow/examples/DoorbellLog.flow.ts similarity index 100% rename from preview/uipath-maestro-flow/examples/DoorbellLog.flow.ts rename to preview/skills/uipath-maestro-flow/examples/DoorbellLog.flow.ts diff --git a/preview/uipath-maestro-flow/examples/FestivalMapScaffold.flow.ts b/preview/skills/uipath-maestro-flow/examples/FestivalMapScaffold.flow.ts similarity index 100% rename from preview/uipath-maestro-flow/examples/FestivalMapScaffold.flow.ts rename to preview/skills/uipath-maestro-flow/examples/FestivalMapScaffold.flow.ts diff --git a/preview/uipath-maestro-flow/examples/FieldTripQuickForm.flow.ts b/preview/skills/uipath-maestro-flow/examples/FieldTripQuickForm.flow.ts similarity index 100% rename from preview/uipath-maestro-flow/examples/FieldTripQuickForm.flow.ts rename to preview/skills/uipath-maestro-flow/examples/FieldTripQuickForm.flow.ts diff --git a/preview/uipath-maestro-flow/examples/FossilCatalogEnrich.flow.ts b/preview/skills/uipath-maestro-flow/examples/FossilCatalogEnrich.flow.ts similarity index 100% rename from preview/uipath-maestro-flow/examples/FossilCatalogEnrich.flow.ts rename to preview/skills/uipath-maestro-flow/examples/FossilCatalogEnrich.flow.ts diff --git a/preview/uipath-maestro-flow/examples/GallerySubmission.flow.ts b/preview/skills/uipath-maestro-flow/examples/GallerySubmission.flow.ts similarity index 100% rename from preview/uipath-maestro-flow/examples/GallerySubmission.flow.ts rename to preview/skills/uipath-maestro-flow/examples/GallerySubmission.flow.ts diff --git a/preview/uipath-maestro-flow/examples/GreenhouseWatering.flow.ts b/preview/skills/uipath-maestro-flow/examples/GreenhouseWatering.flow.ts similarity index 100% rename from preview/uipath-maestro-flow/examples/GreenhouseWatering.flow.ts rename to preview/skills/uipath-maestro-flow/examples/GreenhouseWatering.flow.ts diff --git a/preview/uipath-maestro-flow/examples/HarbourRadioLine.flow.ts b/preview/skills/uipath-maestro-flow/examples/HarbourRadioLine.flow.ts similarity index 100% rename from preview/uipath-maestro-flow/examples/HarbourRadioLine.flow.ts rename to preview/skills/uipath-maestro-flow/examples/HarbourRadioLine.flow.ts diff --git a/preview/uipath-maestro-flow/examples/HerbariumDispatch.flow.ts b/preview/skills/uipath-maestro-flow/examples/HerbariumDispatch.flow.ts similarity index 100% rename from preview/uipath-maestro-flow/examples/HerbariumDispatch.flow.ts rename to preview/skills/uipath-maestro-flow/examples/HerbariumDispatch.flow.ts diff --git a/preview/uipath-maestro-flow/examples/KilnReview.flow.ts b/preview/skills/uipath-maestro-flow/examples/KilnReview.flow.ts similarity index 100% rename from preview/uipath-maestro-flow/examples/KilnReview.flow.ts rename to preview/skills/uipath-maestro-flow/examples/KilnReview.flow.ts diff --git a/preview/uipath-maestro-flow/examples/LibraryDeskChat.flow.ts b/preview/skills/uipath-maestro-flow/examples/LibraryDeskChat.flow.ts similarity index 100% rename from preview/uipath-maestro-flow/examples/LibraryDeskChat.flow.ts rename to preview/skills/uipath-maestro-flow/examples/LibraryDeskChat.flow.ts diff --git a/preview/uipath-maestro-flow/examples/LighthouseSignal.flow.ts b/preview/skills/uipath-maestro-flow/examples/LighthouseSignal.flow.ts similarity index 100% rename from preview/uipath-maestro-flow/examples/LighthouseSignal.flow.ts rename to preview/skills/uipath-maestro-flow/examples/LighthouseSignal.flow.ts diff --git a/preview/uipath-maestro-flow/examples/MeteorShowerPages.flow.ts b/preview/skills/uipath-maestro-flow/examples/MeteorShowerPages.flow.ts similarity index 100% rename from preview/uipath-maestro-flow/examples/MeteorShowerPages.flow.ts rename to preview/skills/uipath-maestro-flow/examples/MeteorShowerPages.flow.ts diff --git a/preview/uipath-maestro-flow/examples/NeighborhoodWalkPlanner.flow.ts b/preview/skills/uipath-maestro-flow/examples/NeighborhoodWalkPlanner.flow.ts similarity index 100% rename from preview/uipath-maestro-flow/examples/NeighborhoodWalkPlanner.flow.ts rename to preview/skills/uipath-maestro-flow/examples/NeighborhoodWalkPlanner.flow.ts diff --git a/preview/uipath-maestro-flow/examples/ObservatorySeeing.flow.ts b/preview/skills/uipath-maestro-flow/examples/ObservatorySeeing.flow.ts similarity index 100% rename from preview/uipath-maestro-flow/examples/ObservatorySeeing.flow.ts rename to preview/skills/uipath-maestro-flow/examples/ObservatorySeeing.flow.ts diff --git a/preview/uipath-maestro-flow/examples/OralHistoryDigest.flow.ts b/preview/skills/uipath-maestro-flow/examples/OralHistoryDigest.flow.ts similarity index 100% rename from preview/uipath-maestro-flow/examples/OralHistoryDigest.flow.ts rename to preview/skills/uipath-maestro-flow/examples/OralHistoryDigest.flow.ts diff --git a/preview/uipath-maestro-flow/examples/PlanetariumConfirmation.flow.ts b/preview/skills/uipath-maestro-flow/examples/PlanetariumConfirmation.flow.ts similarity index 100% rename from preview/uipath-maestro-flow/examples/PlanetariumConfirmation.flow.ts rename to preview/skills/uipath-maestro-flow/examples/PlanetariumConfirmation.flow.ts diff --git a/preview/uipath-maestro-flow/examples/PlantNameAdvisor.flow.ts b/preview/skills/uipath-maestro-flow/examples/PlantNameAdvisor.flow.ts similarity index 100% rename from preview/uipath-maestro-flow/examples/PlantNameAdvisor.flow.ts rename to preview/skills/uipath-maestro-flow/examples/PlantNameAdvisor.flow.ts diff --git a/preview/uipath-maestro-flow/examples/PostcardCaption.flow.ts b/preview/skills/uipath-maestro-flow/examples/PostcardCaption.flow.ts similarity index 100% rename from preview/uipath-maestro-flow/examples/PostcardCaption.flow.ts rename to preview/skills/uipath-maestro-flow/examples/PostcardCaption.flow.ts diff --git a/preview/uipath-maestro-flow/examples/PotteryStudioCallback.flow.ts b/preview/skills/uipath-maestro-flow/examples/PotteryStudioCallback.flow.ts similarity index 100% rename from preview/uipath-maestro-flow/examples/PotteryStudioCallback.flow.ts rename to preview/skills/uipath-maestro-flow/examples/PotteryStudioCallback.flow.ts diff --git a/preview/uipath-maestro-flow/examples/RecipeScaler.flow.ts b/preview/skills/uipath-maestro-flow/examples/RecipeScaler.flow.ts similarity index 100% rename from preview/uipath-maestro-flow/examples/RecipeScaler.flow.ts rename to preview/skills/uipath-maestro-flow/examples/RecipeScaler.flow.ts diff --git a/preview/uipath-maestro-flow/examples/SeedPacketReader.flow.ts b/preview/skills/uipath-maestro-flow/examples/SeedPacketReader.flow.ts similarity index 100% rename from preview/uipath-maestro-flow/examples/SeedPacketReader.flow.ts rename to preview/skills/uipath-maestro-flow/examples/SeedPacketReader.flow.ts diff --git a/preview/uipath-maestro-flow/examples/TideTableConverter.flow.ts b/preview/skills/uipath-maestro-flow/examples/TideTableConverter.flow.ts similarity index 100% rename from preview/uipath-maestro-flow/examples/TideTableConverter.flow.ts rename to preview/skills/uipath-maestro-flow/examples/TideTableConverter.flow.ts diff --git a/preview/uipath-maestro-flow/examples/TrailLogSummary.flow.ts b/preview/skills/uipath-maestro-flow/examples/TrailLogSummary.flow.ts similarity index 100% rename from preview/uipath-maestro-flow/examples/TrailLogSummary.flow.ts rename to preview/skills/uipath-maestro-flow/examples/TrailLogSummary.flow.ts diff --git a/preview/uipath-maestro-flow/examples/WorkshopInventory.flow.ts b/preview/skills/uipath-maestro-flow/examples/WorkshopInventory.flow.ts similarity index 100% rename from preview/uipath-maestro-flow/examples/WorkshopInventory.flow.ts rename to preview/skills/uipath-maestro-flow/examples/WorkshopInventory.flow.ts diff --git a/preview/uipath-maestro-flow/examples/bindings.json b/preview/skills/uipath-maestro-flow/examples/bindings.json similarity index 100% rename from preview/uipath-maestro-flow/examples/bindings.json rename to preview/skills/uipath-maestro-flow/examples/bindings.json diff --git a/preview/uipath-maestro-flow/references/CLI-LOOP.md b/preview/skills/uipath-maestro-flow/references/CLI-LOOP.md similarity index 100% rename from preview/uipath-maestro-flow/references/CLI-LOOP.md rename to preview/skills/uipath-maestro-flow/references/CLI-LOOP.md diff --git a/preview/uipath-maestro-flow/references/agent-resources.md b/preview/skills/uipath-maestro-flow/references/agent-resources.md similarity index 100% rename from preview/uipath-maestro-flow/references/agent-resources.md rename to preview/skills/uipath-maestro-flow/references/agent-resources.md diff --git a/preview/uipath-maestro-flow/references/agent.md b/preview/skills/uipath-maestro-flow/references/agent.md similarity index 100% rename from preview/uipath-maestro-flow/references/agent.md rename to preview/skills/uipath-maestro-flow/references/agent.md diff --git a/preview/uipath-maestro-flow/references/agentic-process.md b/preview/skills/uipath-maestro-flow/references/agentic-process.md similarity index 100% rename from preview/uipath-maestro-flow/references/agentic-process.md rename to preview/skills/uipath-maestro-flow/references/agentic-process.md diff --git a/preview/uipath-maestro-flow/references/api-workflow.md b/preview/skills/uipath-maestro-flow/references/api-workflow.md similarity index 100% rename from preview/uipath-maestro-flow/references/api-workflow.md rename to preview/skills/uipath-maestro-flow/references/api-workflow.md diff --git a/preview/uipath-maestro-flow/references/api.md b/preview/skills/uipath-maestro-flow/references/api.md similarity index 100% rename from preview/uipath-maestro-flow/references/api.md rename to preview/skills/uipath-maestro-flow/references/api.md diff --git a/preview/uipath-maestro-flow/references/batch-transform.md b/preview/skills/uipath-maestro-flow/references/batch-transform.md similarity index 100% rename from preview/uipath-maestro-flow/references/batch-transform.md rename to preview/skills/uipath-maestro-flow/references/batch-transform.md diff --git a/preview/uipath-maestro-flow/references/bindings.md b/preview/skills/uipath-maestro-flow/references/bindings.md similarity index 100% rename from preview/uipath-maestro-flow/references/bindings.md rename to preview/skills/uipath-maestro-flow/references/bindings.md diff --git a/preview/uipath-maestro-flow/references/branch.md b/preview/skills/uipath-maestro-flow/references/branch.md similarity index 100% rename from preview/uipath-maestro-flow/references/branch.md rename to preview/skills/uipath-maestro-flow/references/branch.md diff --git a/preview/uipath-maestro-flow/references/brownfield.md b/preview/skills/uipath-maestro-flow/references/brownfield.md similarity index 100% rename from preview/uipath-maestro-flow/references/brownfield.md rename to preview/skills/uipath-maestro-flow/references/brownfield.md diff --git a/preview/uipath-maestro-flow/references/connector-params.md b/preview/skills/uipath-maestro-flow/references/connector-params.md similarity index 100% rename from preview/uipath-maestro-flow/references/connector-params.md rename to preview/skills/uipath-maestro-flow/references/connector-params.md diff --git a/preview/uipath-maestro-flow/references/conversational.md b/preview/skills/uipath-maestro-flow/references/conversational.md similarity index 100% rename from preview/uipath-maestro-flow/references/conversational.md rename to preview/skills/uipath-maestro-flow/references/conversational.md diff --git a/preview/uipath-maestro-flow/references/data-fabric.md b/preview/skills/uipath-maestro-flow/references/data-fabric.md similarity index 100% rename from preview/uipath-maestro-flow/references/data-fabric.md rename to preview/skills/uipath-maestro-flow/references/data-fabric.md diff --git a/preview/uipath-maestro-flow/references/delay.md b/preview/skills/uipath-maestro-flow/references/delay.md similarity index 100% rename from preview/uipath-maestro-flow/references/delay.md rename to preview/skills/uipath-maestro-flow/references/delay.md diff --git a/preview/uipath-maestro-flow/references/document-pipeline.md b/preview/skills/uipath-maestro-flow/references/document-pipeline.md similarity index 100% rename from preview/uipath-maestro-flow/references/document-pipeline.md rename to preview/skills/uipath-maestro-flow/references/document-pipeline.md diff --git a/preview/uipath-maestro-flow/references/error-handling.md b/preview/skills/uipath-maestro-flow/references/error-handling.md similarity index 100% rename from preview/uipath-maestro-flow/references/error-handling.md rename to preview/skills/uipath-maestro-flow/references/error-handling.md diff --git a/preview/uipath-maestro-flow/references/evaluate.md b/preview/skills/uipath-maestro-flow/references/evaluate.md similarity index 100% rename from preview/uipath-maestro-flow/references/evaluate.md rename to preview/skills/uipath-maestro-flow/references/evaluate.md diff --git a/preview/uipath-maestro-flow/references/event-trigger.md b/preview/skills/uipath-maestro-flow/references/event-trigger.md similarity index 100% rename from preview/uipath-maestro-flow/references/event-trigger.md rename to preview/skills/uipath-maestro-flow/references/event-trigger.md diff --git a/preview/uipath-maestro-flow/references/form-trigger.md b/preview/skills/uipath-maestro-flow/references/form-trigger.md similarity index 100% rename from preview/uipath-maestro-flow/references/form-trigger.md rename to preview/skills/uipath-maestro-flow/references/form-trigger.md diff --git a/preview/uipath-maestro-flow/references/hitl.md b/preview/skills/uipath-maestro-flow/references/hitl.md similarity index 100% rename from preview/uipath-maestro-flow/references/hitl.md rename to preview/skills/uipath-maestro-flow/references/hitl.md diff --git a/preview/uipath-maestro-flow/references/http.md b/preview/skills/uipath-maestro-flow/references/http.md similarity index 100% rename from preview/uipath-maestro-flow/references/http.md rename to preview/skills/uipath-maestro-flow/references/http.md diff --git a/preview/uipath-maestro-flow/references/inline-agent.md b/preview/skills/uipath-maestro-flow/references/inline-agent.md similarity index 100% rename from preview/uipath-maestro-flow/references/inline-agent.md rename to preview/skills/uipath-maestro-flow/references/inline-agent.md diff --git a/preview/uipath-maestro-flow/references/ixp.md b/preview/skills/uipath-maestro-flow/references/ixp.md similarity index 100% rename from preview/uipath-maestro-flow/references/ixp.md rename to preview/skills/uipath-maestro-flow/references/ixp.md diff --git a/preview/uipath-maestro-flow/references/loops.md b/preview/skills/uipath-maestro-flow/references/loops.md similarity index 100% rename from preview/uipath-maestro-flow/references/loops.md rename to preview/skills/uipath-maestro-flow/references/loops.md diff --git a/preview/uipath-maestro-flow/references/manual-trigger.md b/preview/skills/uipath-maestro-flow/references/manual-trigger.md similarity index 100% rename from preview/uipath-maestro-flow/references/manual-trigger.md rename to preview/skills/uipath-maestro-flow/references/manual-trigger.md diff --git a/preview/uipath-maestro-flow/references/or-processes.md b/preview/skills/uipath-maestro-flow/references/or-processes.md similarity index 100% rename from preview/uipath-maestro-flow/references/or-processes.md rename to preview/skills/uipath-maestro-flow/references/or-processes.md diff --git a/preview/uipath-maestro-flow/references/parallel-merge.md b/preview/skills/uipath-maestro-flow/references/parallel-merge.md similarity index 100% rename from preview/uipath-maestro-flow/references/parallel-merge.md rename to preview/skills/uipath-maestro-flow/references/parallel-merge.md diff --git a/preview/uipath-maestro-flow/references/placeholder.md b/preview/skills/uipath-maestro-flow/references/placeholder.md similarity index 100% rename from preview/uipath-maestro-flow/references/placeholder.md rename to preview/skills/uipath-maestro-flow/references/placeholder.md diff --git a/preview/uipath-maestro-flow/references/published-function.md b/preview/skills/uipath-maestro-flow/references/published-function.md similarity index 100% rename from preview/uipath-maestro-flow/references/published-function.md rename to preview/skills/uipath-maestro-flow/references/published-function.md diff --git a/preview/uipath-maestro-flow/references/queue.md b/preview/skills/uipath-maestro-flow/references/queue.md similarity index 100% rename from preview/uipath-maestro-flow/references/queue.md rename to preview/skills/uipath-maestro-flow/references/queue.md diff --git a/preview/uipath-maestro-flow/references/return.md b/preview/skills/uipath-maestro-flow/references/return.md similarity index 100% rename from preview/uipath-maestro-flow/references/return.md rename to preview/skills/uipath-maestro-flow/references/return.md diff --git a/preview/uipath-maestro-flow/references/rpa-workflow.md b/preview/skills/uipath-maestro-flow/references/rpa-workflow.md similarity index 100% rename from preview/uipath-maestro-flow/references/rpa-workflow.md rename to preview/skills/uipath-maestro-flow/references/rpa-workflow.md diff --git a/preview/uipath-maestro-flow/references/scheduled-trigger.md b/preview/skills/uipath-maestro-flow/references/scheduled-trigger.md similarity index 100% rename from preview/uipath-maestro-flow/references/scheduled-trigger.md rename to preview/skills/uipath-maestro-flow/references/scheduled-trigger.md diff --git a/preview/uipath-maestro-flow/references/script.md b/preview/skills/uipath-maestro-flow/references/script.md similarity index 100% rename from preview/uipath-maestro-flow/references/script.md rename to preview/skills/uipath-maestro-flow/references/script.md diff --git a/preview/uipath-maestro-flow/references/subflow.md b/preview/skills/uipath-maestro-flow/references/subflow.md similarity index 100% rename from preview/uipath-maestro-flow/references/subflow.md rename to preview/skills/uipath-maestro-flow/references/subflow.md diff --git a/preview/uipath-maestro-flow/references/summarize.md b/preview/skills/uipath-maestro-flow/references/summarize.md similarity index 100% rename from preview/uipath-maestro-flow/references/summarize.md rename to preview/skills/uipath-maestro-flow/references/summarize.md diff --git a/preview/uipath-maestro-flow/references/switch.md b/preview/skills/uipath-maestro-flow/references/switch.md similarity index 100% rename from preview/uipath-maestro-flow/references/switch.md rename to preview/skills/uipath-maestro-flow/references/switch.md diff --git a/preview/uipath-maestro-flow/references/terminate.md b/preview/skills/uipath-maestro-flow/references/terminate.md similarity index 100% rename from preview/uipath-maestro-flow/references/terminate.md rename to preview/skills/uipath-maestro-flow/references/terminate.md diff --git a/preview/uipath-maestro-flow/references/transform.md b/preview/skills/uipath-maestro-flow/references/transform.md similarity index 100% rename from preview/uipath-maestro-flow/references/transform.md rename to preview/skills/uipath-maestro-flow/references/transform.md diff --git a/preview/uipath-maestro-flow/references/voice.md b/preview/skills/uipath-maestro-flow/references/voice.md similarity index 100% rename from preview/uipath-maestro-flow/references/voice.md rename to preview/skills/uipath-maestro-flow/references/voice.md diff --git a/scripts/sync-maestro-sdk-preview.mjs b/scripts/sync-maestro-sdk-preview.mjs index 15905fe022..10e1fa0d82 100644 --- a/scripts/sync-maestro-sdk-preview.mjs +++ b/scripts/sync-maestro-sdk-preview.mjs @@ -13,31 +13,31 @@ const sourceRoot = 'typescript/sdk'; const skillMappings = [ { source: `${sourceRoot}/skill/SKILL.md`, - target: 'preview/uipath-maestro-flow/SKILL.md', + target: 'preview/skills/uipath-maestro-flow/SKILL.md', }, { source: `${sourceRoot}/skill/SKILL-case.md`, - target: 'preview/uipath-maestro-case/SKILL.md', + target: 'preview/skills/uipath-maestro-case/SKILL.md', }, { source: `${sourceRoot}/skill/SKILL-bpmn.md`, - target: 'preview/uipath-maestro-bpmn/SKILL.md', + target: 'preview/skills/uipath-maestro-bpmn/SKILL.md', }, ]; const provenanceFiles = [ - ['preview/uipath-maestro-flow/SKILL.md', 'typescript/sdk/skill/SKILL.md'], - ['preview/uipath-maestro-case/SKILL.md', 'typescript/sdk/skill/SKILL-case.md'], - ['preview/uipath-maestro-bpmn/SKILL.md', 'typescript/sdk/skill/SKILL-bpmn.md'], + ['preview/skills/uipath-maestro-flow/SKILL.md', 'typescript/sdk/skill/SKILL.md'], + ['preview/skills/uipath-maestro-case/SKILL.md', 'typescript/sdk/skill/SKILL-case.md'], + ['preview/skills/uipath-maestro-bpmn/SKILL.md', 'typescript/sdk/skill/SKILL-bpmn.md'], ]; const managedDirectories = [ - 'preview/uipath-maestro-flow/references', - 'preview/uipath-maestro-flow/examples', - 'preview/uipath-maestro-case/references', - 'preview/uipath-maestro-case/examples', - 'preview/uipath-maestro-bpmn/references', - 'preview/uipath-maestro-bpmn/examples', + 'preview/skills/uipath-maestro-flow/references', + 'preview/skills/uipath-maestro-flow/examples', + 'preview/skills/uipath-maestro-case/references', + 'preview/skills/uipath-maestro-case/examples', + 'preview/skills/uipath-maestro-bpmn/references', + 'preview/skills/uipath-maestro-bpmn/examples', ]; const oldSiblingParagraph = [ @@ -190,13 +190,13 @@ function collectMappings(upstreamRoot, refs) { 'bpmn-runtime.md', ].includes(name)) continue; const relative = path.posix.relative(referencesRoot, source); - add(source, `preview/uipath-maestro-flow/references/${relative}`); + add(source, `preview/skills/uipath-maestro-flow/references/${relative}`); } const flowExamplesRoot = `${sourceRoot}/example-eval`; for (const source of gitFiles(upstreamRoot, ref, flowExamplesRoot)) { const relative = path.posix.relative(flowExamplesRoot, source); - add(source, `preview/uipath-maestro-flow/examples/${relative}`); + add(source, `preview/skills/uipath-maestro-flow/examples/${relative}`); } const sharedExamplesRoot = `${sourceRoot}/example`; @@ -204,26 +204,26 @@ function collectMappings(upstreamRoot, refs) { const name = path.posix.basename(source); const relative = path.posix.relative(sharedExamplesRoot, source); if (name.endsWith('.case.ts')) { - add(source, `preview/uipath-maestro-case/examples/${relative}`); + add(source, `preview/skills/uipath-maestro-case/examples/${relative}`); } else if (relative === 'case-bindings.json') { - add(source, 'preview/uipath-maestro-case/examples/bindings.json'); + add(source, 'preview/skills/uipath-maestro-case/examples/bindings.json'); } else if (name.endsWith('.bpmn.ts')) { - add(source, `preview/uipath-maestro-bpmn/examples/${relative}`); + add(source, `preview/skills/uipath-maestro-bpmn/examples/${relative}`); } } } add( `${sourceRoot}/skill/references/case-api.md`, - 'preview/uipath-maestro-case/references/api.md', + 'preview/skills/uipath-maestro-case/references/api.md', ); add( `${sourceRoot}/skill/references/bpmn-api.md`, - 'preview/uipath-maestro-bpmn/references/api.md', + 'preview/skills/uipath-maestro-bpmn/references/api.md', ); for (const [source, target] of [ - [`${sourceRoot}/skill/references/case-runtime.md`, 'preview/uipath-maestro-case/references/case-runtime.md'], - [`${sourceRoot}/skill/references/bpmn-runtime.md`, 'preview/uipath-maestro-bpmn/references/bpmn-runtime.md'], + [`${sourceRoot}/skill/references/case-runtime.md`, 'preview/skills/uipath-maestro-case/references/case-runtime.md'], + [`${sourceRoot}/skill/references/bpmn-runtime.md`, 'preview/skills/uipath-maestro-bpmn/references/bpmn-runtime.md'], ]) { if (refs.some((ref) => gitObjectExists(upstreamRoot, ref, source))) add(source, target); } @@ -406,13 +406,13 @@ function adaptBpmnRuntime(text) { } const snapshotAdaptations = new Map([ - ['preview/uipath-maestro-flow/SKILL.md', adaptFlowSkill], - ['preview/uipath-maestro-case/SKILL.md', adaptCaseSkill], - ['preview/uipath-maestro-bpmn/SKILL.md', adaptBpmnSkill], - ['preview/uipath-maestro-flow/references/api.md', adaptFlowApi], - ['preview/uipath-maestro-case/examples/NotifyOnApproval.case.ts', adaptCaseExample], - ['preview/uipath-maestro-case/references/case-runtime.md', adaptCaseRuntime], - ['preview/uipath-maestro-bpmn/references/bpmn-runtime.md', adaptBpmnRuntime], + ['preview/skills/uipath-maestro-flow/SKILL.md', adaptFlowSkill], + ['preview/skills/uipath-maestro-case/SKILL.md', adaptCaseSkill], + ['preview/skills/uipath-maestro-bpmn/SKILL.md', adaptBpmnSkill], + ['preview/skills/uipath-maestro-flow/references/api.md', adaptFlowApi], + ['preview/skills/uipath-maestro-case/examples/NotifyOnApproval.case.ts', adaptCaseExample], + ['preview/skills/uipath-maestro-case/references/case-runtime.md', adaptCaseRuntime], + ['preview/skills/uipath-maestro-bpmn/references/bpmn-runtime.md', adaptBpmnRuntime], ]); function snapshotAdaptationFor(target) { @@ -470,10 +470,10 @@ function verifySkillBody(skillsRoot, upstreamRoot, newCommit, target, source, ad function verifyExactMappings(skillsRoot, upstreamRoot, newCommit, mappings) { const adaptedTargets = new Map([ - ['preview/uipath-maestro-flow/references/api.md', adaptFlowApi], - ['preview/uipath-maestro-case/examples/NotifyOnApproval.case.ts', adaptCaseExample], - ['preview/uipath-maestro-case/references/case-runtime.md', adaptCaseRuntime], - ['preview/uipath-maestro-bpmn/references/bpmn-runtime.md', adaptBpmnRuntime], + ['preview/skills/uipath-maestro-flow/references/api.md', adaptFlowApi], + ['preview/skills/uipath-maestro-case/examples/NotifyOnApproval.case.ts', adaptCaseExample], + ['preview/skills/uipath-maestro-case/references/case-runtime.md', adaptCaseRuntime], + ['preview/skills/uipath-maestro-bpmn/references/bpmn-runtime.md', adaptBpmnRuntime], ]); const skillTargets = new Set(skillMappings.map(({ target }) => target)); for (const mapping of mappings) { @@ -492,7 +492,7 @@ function verifyExactMappings(skillsRoot, upstreamRoot, newCommit, mappings) { function verifyFrontmatterAndPins(skillsRoot, newPin) { for (const skill of ['flow', 'case', 'bpmn']) { - const target = `preview/uipath-maestro-${skill}/SKILL.md`; + const target = `preview/skills/uipath-maestro-${skill}/SKILL.md`; const text = readText(path.join(skillsRoot, target)); const frontmatter = text.match(/^---\n([\s\S]*?)\n---\n/); if (!frontmatter) fail(`${target} is missing YAML frontmatter`); @@ -510,7 +510,7 @@ function verifyFrontmatterAndPins(skillsRoot, newPin) { function verifyDeadLinks(skillsRoot) { for (const skill of ['flow', 'case', 'bpmn']) { - const skillRoot = path.join(skillsRoot, `preview/uipath-maestro-${skill}`); + const skillRoot = path.join(skillsRoot, `preview/skills/uipath-maestro-${skill}`); const documents = [path.join(skillRoot, 'SKILL.md')]; documents.push( ...walkFiles(path.join(skillRoot, 'references')).filter((file) => file.endsWith('.md')), @@ -545,7 +545,7 @@ function routerRows(text) { } function verifyRouter(skillsRoot, upstreamRoot, newCommit) { - const target = readText(path.join(skillsRoot, 'preview/uipath-maestro-flow/SKILL.md')); + const target = readText(path.join(skillsRoot, 'preview/skills/uipath-maestro-flow/SKILL.md')); const source = gitShow(upstreamRoot, newCommit, `${sourceRoot}/skill/SKILL.md`).toString('utf8'); const rows = routerRows(target); const sourceRows = routerRows(source); @@ -555,19 +555,19 @@ function verifyRouter(skillsRoot, upstreamRoot, newCommit) { for (const row of rows) { const reference = row.match(/\]\(references\/([^)]+)\)/)?.[1]; const example = row.match(/`examples\/([^`]+)`/)?.[1]; - if (!reference || !fs.existsSync(path.join(skillsRoot, 'preview/uipath-maestro-flow/references', reference))) { + if (!reference || !fs.existsSync(path.join(skillsRoot, 'preview/skills/uipath-maestro-flow/references', reference))) { fail(`Flow router row has a missing reference: ${row}`); } - if (!example || !fs.existsSync(path.join(skillsRoot, 'preview/uipath-maestro-flow/examples', example))) { + if (!example || !fs.existsSync(path.join(skillsRoot, 'preview/skills/uipath-maestro-flow/examples', example))) { fail(`Flow router row has a missing example: ${row}`); } } } function verifyStalePaths(skillsRoot) { - const flow = readText(path.join(skillsRoot, 'preview/uipath-maestro-flow/SKILL.md')); - const caseSkill = readText(path.join(skillsRoot, 'preview/uipath-maestro-case/SKILL.md')); - const bpmn = readText(path.join(skillsRoot, 'preview/uipath-maestro-bpmn/SKILL.md')); + const flow = readText(path.join(skillsRoot, 'preview/skills/uipath-maestro-flow/SKILL.md')); + const caseSkill = readText(path.join(skillsRoot, 'preview/skills/uipath-maestro-case/SKILL.md')); + const bpmn = readText(path.join(skillsRoot, 'preview/skills/uipath-maestro-bpmn/SKILL.md')); if (/\bexample\//.test(flow) || /references\/(?:case|bpmn)-api\.md/.test(flow)) { fail('Flow SKILL.md contains an upstream-only path'); } @@ -577,7 +577,7 @@ function verifyStalePaths(skillsRoot) { if (/bpmn-api|\bexample\//.test(bpmn)) { fail('BPMN SKILL.md contains an upstream-only path'); } - for (const file of walkFiles(path.join(skillsRoot, 'preview/uipath-maestro-case/examples'))) { + for (const file of walkFiles(path.join(skillsRoot, 'preview/skills/uipath-maestro-case/examples'))) { if (/case-bindings|\bexample\//.test(readText(file))) { fail(`${normalizeRelative(path.relative(skillsRoot, file))} contains an upstream-only path`); } @@ -618,7 +618,7 @@ export function verifySnapshot({ skillsRoot, upstreamRoot, newCommit, newPin, ma skillsRoot, upstreamRoot, newCommit, - 'preview/uipath-maestro-flow/SKILL.md', + 'preview/skills/uipath-maestro-flow/SKILL.md', `${sourceRoot}/skill/SKILL.md`, adaptFlowSkill, ); @@ -626,7 +626,7 @@ export function verifySnapshot({ skillsRoot, upstreamRoot, newCommit, newPin, ma skillsRoot, upstreamRoot, newCommit, - 'preview/uipath-maestro-case/SKILL.md', + 'preview/skills/uipath-maestro-case/SKILL.md', `${sourceRoot}/skill/SKILL-case.md`, adaptCaseSkill, ); @@ -634,7 +634,7 @@ export function verifySnapshot({ skillsRoot, upstreamRoot, newCommit, newPin, ma skillsRoot, upstreamRoot, newCommit, - 'preview/uipath-maestro-bpmn/SKILL.md', + 'preview/skills/uipath-maestro-bpmn/SKILL.md', `${sourceRoot}/skill/SKILL-bpmn.md`, adaptBpmnSkill, ); diff --git a/tests/README.md b/tests/README.md index abddabdff2..b7324fa911 100644 --- a/tests/README.md +++ b/tests/README.md @@ -190,10 +190,16 @@ runner. The image build passes the package credential as exists only for the external nightly caller during migration. Regular nightly and smoke jobs continue to use `skills-image:latest`. -`flow-v2-preview.yaml` runs the three `preview/uipath-maestro-{flow,case,bpmn}` -builder-SDK skills as the ONLY skill catalog, shadowing the shipped v1 skills of -the same name, so a run measures the Flow v2 authoring path rather than a mix of -both generations. Narrowing `plugins.path` to `preview/` drops the automatic +`flow-v2-preview.yaml` runs the three `preview/skills/uipath-maestro-{flow,case,bpmn}` +builder-SDK skills as the ONLY skill catalog, so a run measures the Flow v2 +authoring path rather than a mix of both generations. `preview/` is a Claude Code +**plugin root** (`preview/.claude-plugin/plugin.json` + `preview/skills//SKILL.md`), +which is the one layout every harness loads: Claude Code requires it, the +Delegate SDK appends `/skills` to it, Codex and Antigravity accept it. Skills load +as `uipath-preview:uipath-maestro-flow` (the repo-root catalog is `uipath:`). Never +point `plugins.path` at a bare directory of skill folders: Claude Code loads +nothing from it and says so only as a per-task WARNING in task.log (every v2 run +08-20 โ†’ 09-03 ran that way). Narrowing `plugins.path` to `preview/` drops the automatic repo-root bind mount, so the root is remounted explicitly; the image also needs runtime npm auth for the `@uipath` scope. Login state mounts at `/.uipath`, identical to `nightly.yaml`. Confirm that mount resolves before a full run, or diff --git a/tests/experiments/flow-v2-preview.yaml b/tests/experiments/flow-v2-preview.yaml index 772a60ddc0..a10480421d 100644 --- a/tests/experiments/flow-v2-preview.yaml +++ b/tests/experiments/flow-v2-preview.yaml @@ -44,6 +44,7 @@ defaults: allowed_tools: ["Skill", "Bash", "Read", "Write", "Edit", "Glob", "Grep"] plugins: # PREVIEW ONLY. Also the source of the automatic repo-path bind mount. + # Must stay the plugin ROOT (holds .claude-plugin/ + skills/), never preview/skills. - type: "local" path: "$SKILLS_REPO_PATH/preview" ignore_patterns: [] diff --git a/tests/scripts/sync-maestro-sdk-preview.test.mjs b/tests/scripts/sync-maestro-sdk-preview.test.mjs index e4bd66eaea..b3577b4f9e 100644 --- a/tests/scripts/sync-maestro-sdk-preview.test.mjs +++ b/tests/scripts/sync-maestro-sdk-preview.test.mjs @@ -160,19 +160,19 @@ function createInitialSnapshot(skillsRoot, upstreamRoot, pin) { [ 'uipath-maestro-flow', 'typescript/sdk/skill/SKILL.md', - 'preview/uipath-maestro-flow/SKILL.md', + 'preview/skills/uipath-maestro-flow/SKILL.md', adaptFlowSkill, ], [ 'uipath-maestro-case', 'typescript/sdk/skill/SKILL-case.md', - 'preview/uipath-maestro-case/SKILL.md', + 'preview/skills/uipath-maestro-case/SKILL.md', adaptCaseSkill, ], [ 'uipath-maestro-bpmn', 'typescript/sdk/skill/SKILL-bpmn.md', - 'preview/uipath-maestro-bpmn/SKILL.md', + 'preview/skills/uipath-maestro-bpmn/SKILL.md', adaptBpmnSkill, ], ]; @@ -186,52 +186,52 @@ function createInitialSnapshot(skillsRoot, upstreamRoot, pin) { write( skillsRoot, - 'preview/uipath-maestro-flow/references/api.md', + 'preview/skills/uipath-maestro-flow/references/api.md', adaptFlowApi(read(upstreamRoot, 'typescript/sdk/skill/references/api.md')), ); write( skillsRoot, - 'preview/uipath-maestro-flow/references/guide.md', + 'preview/skills/uipath-maestro-flow/references/guide.md', read(upstreamRoot, 'typescript/sdk/skill/references/guide.md'), ); write( skillsRoot, - 'preview/uipath-maestro-case/references/api.md', + 'preview/skills/uipath-maestro-case/references/api.md', read(upstreamRoot, 'typescript/sdk/skill/references/case-api.md'), ); write( skillsRoot, - 'preview/uipath-maestro-bpmn/references/api.md', + 'preview/skills/uipath-maestro-bpmn/references/api.md', read(upstreamRoot, 'typescript/sdk/skill/references/bpmn-api.md'), ); write( skillsRoot, - 'preview/uipath-maestro-flow/examples/Foo.flow.ts', + 'preview/skills/uipath-maestro-flow/examples/Foo.flow.ts', read(upstreamRoot, 'typescript/sdk/example-eval/Foo.flow.ts'), ); write( skillsRoot, - 'preview/uipath-maestro-flow/examples/bindings.json', + 'preview/skills/uipath-maestro-flow/examples/bindings.json', read(upstreamRoot, 'typescript/sdk/example-eval/bindings.json'), ); write( skillsRoot, - 'preview/uipath-maestro-case/examples/Other.case.ts', + 'preview/skills/uipath-maestro-case/examples/Other.case.ts', read(upstreamRoot, 'typescript/sdk/example/Other.case.ts'), ); write( skillsRoot, - 'preview/uipath-maestro-case/examples/NotifyOnApproval.case.ts', + 'preview/skills/uipath-maestro-case/examples/NotifyOnApproval.case.ts', adaptCaseExample(read(upstreamRoot, 'typescript/sdk/example/NotifyOnApproval.case.ts')), ); write( skillsRoot, - 'preview/uipath-maestro-case/examples/bindings.json', + 'preview/skills/uipath-maestro-case/examples/bindings.json', read(upstreamRoot, 'typescript/sdk/example/case-bindings.json'), ); write( skillsRoot, - 'preview/uipath-maestro-bpmn/examples/NotifyChannel.bpmn.ts', + 'preview/skills/uipath-maestro-bpmn/examples/NotifyChannel.bpmn.ts', read(upstreamRoot, 'typescript/sdk/example/NotifyChannel.bpmn.ts'), ); } @@ -302,56 +302,56 @@ test('syncSnapshots three-way merges drift and reapplies only snapshot adaptatio { oldPin, newPin, changed: true }, ); assert.equal( - read(skillsRoot, 'preview/uipath-maestro-flow/references/api.md'), + read(skillsRoot, 'preview/skills/uipath-maestro-flow/references/api.md'), '# Flow API v2\n\nWorked example: `examples/Foo.flow.ts`\n', ); assert.equal( - fs.existsSync(path.join(skillsRoot, 'preview/uipath-maestro-flow/references/guide.md')), + fs.existsSync(path.join(skillsRoot, 'preview/skills/uipath-maestro-flow/references/guide.md')), false, ); assert.equal( - read(skillsRoot, 'preview/uipath-maestro-flow/references/new.md'), + read(skillsRoot, 'preview/skills/uipath-maestro-flow/references/new.md'), '# New reference\n', ); assert.equal( - read(skillsRoot, 'preview/uipath-maestro-flow/examples/Foo.flow.ts'), + read(skillsRoot, 'preview/skills/uipath-maestro-flow/examples/Foo.flow.ts'), 'export const value = 2;\n', ); assert.match( - read(skillsRoot, 'preview/uipath-maestro-flow/SKILL.md'), + read(skillsRoot, 'preview/skills/uipath-maestro-flow/SKILL.md'), /\| Script action \|.*`examples\/Foo\.flow\.ts`/, ); assert.match( - read(skillsRoot, 'preview/uipath-maestro-case/SKILL.md'), + read(skillsRoot, 'preview/skills/uipath-maestro-case/SKILL.md'), /New upstream Case guidance/, ); assert.match( - read(skillsRoot, 'preview/uipath-maestro-bpmn/SKILL.md'), + read(skillsRoot, 'preview/skills/uipath-maestro-bpmn/SKILL.md'), /# BPMN fixture v2[\s\S]*`examples\/NotifyChannel\.bpmn\.ts`/, ); assert.doesNotMatch( - read(skillsRoot, 'preview/uipath-maestro-bpmn/SKILL.md'), + read(skillsRoot, 'preview/skills/uipath-maestro-bpmn/SKILL.md'), /representative process/, ); assert.equal( - read(skillsRoot, 'preview/uipath-maestro-case/references/case-runtime.md'), + read(skillsRoot, 'preview/skills/uipath-maestro-case/references/case-runtime.md'), '# Case runtime\n\nSee [Case API](api.md).\n', ); assert.equal( - read(skillsRoot, 'preview/uipath-maestro-bpmn/references/bpmn-runtime.md'), + read(skillsRoot, 'preview/skills/uipath-maestro-bpmn/references/bpmn-runtime.md'), '# BPMN runtime\n\nSee [BPMN API](api.md).\n', ); assert.equal( - fs.existsSync(path.join(skillsRoot, 'preview/uipath-maestro-flow/references/case-runtime.md')), + fs.existsSync(path.join(skillsRoot, 'preview/skills/uipath-maestro-flow/references/case-runtime.md')), false, ); assert.equal( - fs.existsSync(path.join(skillsRoot, 'preview/uipath-maestro-flow/references/bpmn-runtime.md')), + fs.existsSync(path.join(skillsRoot, 'preview/skills/uipath-maestro-flow/references/bpmn-runtime.md')), false, ); for (const skill of ['flow', 'case', 'bpmn']) { assert.match( - read(skillsRoot, `preview/uipath-maestro-${skill}/SKILL.md`), + read(skillsRoot, `preview/skills/uipath-maestro-${skill}/SKILL.md`), new RegExp(`@ ${newPin}\\. Canonical source`), ); } From 843cb7428a08f5a80c4709236e360e1165d75ca6 Mon Sep 17 00:00:00 2001 From: tmatup <51425734+tmatup@users.noreply.github.com> Date: Fri, 4 Sep 2026 02:48:15 +0000 Subject: [PATCH 52/52] fix: address remaining #2756 review feedback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pin the Flow SDK version in both eval images, align the bundled BPMN contracts with CLI main, and make timeout-sensitive pagination telemetry advisory. ๐Ÿค– Generated with Claude Code Co-Authored-By: [Claude](mailto:noreply@anthropic.com) --- .github/workflows/smoke-skills.yml | 1 + .github/workflows/test-helpers.yml | 1 + .../validator/bpmn-spec.json | 109 ++++++++++++++---- .../_shared/test_bpmn_check.py | 43 +++++++ .../_shared/test_same_ground_corpus.py | 8 +- .../paginated_reference_lookup.yaml | 11 +- 6 files changed, 140 insertions(+), 33 deletions(-) diff --git a/.github/workflows/smoke-skills.yml b/.github/workflows/smoke-skills.yml index 213654d0a2..6af1075033 100644 --- a/.github/workflows/smoke-skills.yml +++ b/.github/workflows/smoke-skills.yml @@ -383,6 +383,7 @@ jobs: docker build \ --build-arg CODER_EVAL_IMAGE=ghcr.io/uipath/coder-eval-agent:${{ steps.ceref.outputs.version }} \ --secret id=npm_auth_token,env=NPM_AUTH_TOKEN \ + --build-arg FLOW_SDK_VERSION="${{ vars.FLOW_SDK_VERSION || 'latest' }}" \ -t skills-image:latest \ -f tests/docker/Dockerfile \ . diff --git a/.github/workflows/test-helpers.yml b/.github/workflows/test-helpers.yml index 32abfe85e6..05641572b3 100644 --- a/.github/workflows/test-helpers.yml +++ b/.github/workflows/test-helpers.yml @@ -6,6 +6,7 @@ on: - 'tests/tasks/uipath-maestro-flow/**' - 'tests/tasks/uipath-maestro-case/**' - 'tests/tasks/uipath-maestro-bpmn/**' + - 'skills/uipath-maestro-bpmn/validator/bpmn-spec.json' - 'tests/tasks/uipath-planner/**' - 'tests/tasks/uipath-agents/**' - 'tests/tasks/uipath-admin/**' diff --git a/skills/uipath-maestro-bpmn/validator/bpmn-spec.json b/skills/uipath-maestro-bpmn/validator/bpmn-spec.json index 0d7a5d5325..b4543e92b9 100644 --- a/skills/uipath-maestro-bpmn/validator/bpmn-spec.json +++ b/skills/uipath-maestro-bpmn/validator/bpmn-spec.json @@ -310,20 +310,24 @@ "extensionTag": "uipath:activity", "contextFields": [ { - "name": "activity", - "displayName": "Activity", + "name": "activityConfigurationVersion", + "displayName": "Activity configuration version", "type": "string", - "required": true, - "hidden": false, - "isPrimaryKey": true, + "required": false, + "hidden": true, + "isPrimaryKey": false, "binding": false, - "defaultValue": null + "defaultValue": "v1", + "enum": [ + "v1" + ], + "description": "Marks the Integration Service activity format used by this template" }, { "name": "connectorKey", "displayName": "Connector key", "type": "string", - "required": false, + "required": true, "hidden": true, "isPrimaryKey": false, "binding": false, @@ -337,7 +341,12 @@ "hidden": true, "isPrimaryKey": false, "binding": true, - "defaultValue": null + "defaultValue": null, + "bindingInfo": { + "resource": "Connection", + "resourceKeyPattern": "${resourceKey}", + "propertyAttribute": "ConnectionId" + } }, { "name": "folderKey", @@ -346,43 +355,85 @@ "required": false, "hidden": true, "isPrimaryKey": false, + "binding": true, + "defaultValue": null, + "bindingInfo": { + "resource": "Connection", + "resourceKeyPattern": "${resourceKey}", + "propertyAttribute": "folderKey" + } + }, + { + "name": "operation", + "displayName": "Operation", + "type": "string", + "required": true, + "hidden": true, + "isPrimaryKey": true, "binding": false, - "defaultValue": null + "defaultValue": null, + "description": "Activity catalog Name used as the runtime operation" + }, + { + "name": "objectName", + "displayName": "Object name", + "type": "string", + "required": true, + "hidden": true, + "isPrimaryKey": false, + "binding": false, + "defaultValue": null, + "description": "Activity catalog ObjectName" }, { "name": "method", "displayName": "Method", "type": "string", - "required": false, + "required": true, "hidden": true, "isPrimaryKey": false, "binding": false, - "defaultValue": null + "defaultValue": null, + "description": "Activity catalog MethodName" }, { "name": "path", "displayName": "Path", "type": "string", + "required": true, + "hidden": true, + "isPrimaryKey": false, + "binding": false, + "defaultValue": null, + "description": "Request path returned by resource metadata" + }, + { + "name": "metadata", + "displayName": "Metadata", + "type": "json", "required": false, "hidden": true, "isPrimaryKey": false, "binding": false, - "defaultValue": null + "defaultValue": {}, + "description": "Optional activity metadata" } ], + "inputFields": [], "inputPattern": "separateInputs", "inputName": "body", "inputTarget": "body", - "outputSource": ".", - "outputType": "custom", - "outputName": "result", + "outputSource": "=response", + "outputType": "jsonSchema", + "outputName": "response", "useRelativeSource": false, "bindingPattern": "connection", "requiresDiscovery": true, "isDynamic": true, - "xmlTemplate": "\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n {incomingEdge}\n {outgoingEdge}\n", - "discoveryNotes": "Uses getConnectorConnections(connectorKey) to list available connections. Activity/connector metadata comes from Integration Service catalog.", - "inputNotes": "isSplitInputs=false in design schema; inputs are sent as separate elements, not merged into a single JSON body", + "xmlTemplate": "\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n {incomingEdge}\n {outgoingEdge}\n", + "discoveryNotes": "Use `uip is activities list --output json` to select an activity. Set operation from Name, then describe ObjectName to get its current path, parameters, and fields.", + "inputNotes": "Add each request field from the selected operation as its own uipath:input. Keep the field's type and target: path, query, body, or file.", + "contextFieldNotes": "connection points to a root binding for a Connection resource. When folder scoping is required, folderKey points to a second binding for the same resourceKey. Use propertyAttribute ConnectionId for connection and folderKey for folderKey. Omit folderKey when the selected activity has no folder scoping, and omit both fields only when it does not use a connection.", "extensionTagNotes": "In isExecution array, NOT in isEvents array, so uses uipath:activity" }, "Intsvc.HttpExecution": { @@ -585,7 +636,7 @@ "bindingPattern": "connection", "requiresDiscovery": true, "isDynamic": false, - "xmlTemplate": "\n \n \n \n \n \n \n {incomingEdge}\n {outgoingEdge}\n", + "xmlTemplate": "\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n {incomingEdge}\n {outgoingEdge}\n", "discoveryNotes": "Uses getConnectorConnections(connectorKey) to list available connections", "inputNotes": "isSplitInputs=false; inputs are defined in input.fields (not jsonBody). Sent as separate elements.", "contextFieldNotes": "Context fields are empty in design schema; all field definitions are in inputFields below", @@ -600,7 +651,12 @@ "isPrimaryKey": false, "binding": true, "defaultValue": null, - "description": "Connection ID for the IS connector (bindable)" + "description": "Connection ID for the IS connector (bindable)", + "bindingInfo": { + "resource": "Connection", + "resourceKeyPattern": "${resourceKey}", + "propertyAttribute": "ConnectionId" + } }, { "name": "connectorKey", @@ -714,7 +770,7 @@ "bindingPattern": "connection", "requiresDiscovery": true, "isDynamic": false, - "xmlTemplate": "\n \n \n \n \n \n \n \n {incomingEdge}\n {outgoingEdge}\n", + "xmlTemplate": "\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n {incomingEdge}\n {outgoingEdge}\n", "discoveryNotes": "Uses getConnectorConnections(connectorKey) to list available connections", "inputNotes": "isSplitInputs=false; inputs defined in input.fields. Sent as separate elements.", "contextFieldNotes": "Context fields are empty in design schema; all field definitions are in inputFields below", @@ -729,7 +785,12 @@ "isPrimaryKey": false, "binding": true, "defaultValue": null, - "description": "Connection ID for the IS connector (bindable)" + "description": "Connection ID for the IS connector (bindable)", + "bindingInfo": { + "resource": "Connection", + "resourceKeyPattern": "${resourceKey}", + "propertyAttribute": "ConnectionId" + } }, { "name": "connectorKey", @@ -1827,8 +1888,8 @@ "isDynamic": false, "xmlTemplate": "\n \n \n \n \n \n {incomingEdge}\n {outgoingEdge}\n", "inputNotes": "Variable mappings use uipath:mapping with direct input/output elements, not a jsonBody pattern", - "bpmnElementNotes": "Also used on bpmn:StartEvent and bpmn:EndEvent for variable mappings. On bpmn:Task with no serviceType, defaults to BPMN.Variables.", - "extensionTagNotes": "Uses uipath:mapping (not uipath:activity). createExtensionTask checks for serviceType=BPMN.Variables on StartEvent/EndEvent/Task and routes to createModdleExtensionVariableMapping." + "bpmnElementNotes": "Also used on bpmn:StartEvent, bpmn:EndEvent, bpmn:ScriptTask, and bpmn:SubProcess for variable mappings. On bpmn:Task with no serviceType, defaults to BPMN.Variables.", + "extensionTagNotes": "Uses uipath:mapping (not uipath:activity). BPMN.Variables mappings are supported on StartEvent, EndEvent, Task, ScriptTask, and SubProcess owners." }, "BPMN.ScriptTask": { "extensionType": "BPMN.ScriptTask", diff --git a/tests/tasks/uipath-maestro-bpmn/_shared/test_bpmn_check.py b/tests/tasks/uipath-maestro-bpmn/_shared/test_bpmn_check.py index 1df5a1925a..233316a57b 100644 --- a/tests/tasks/uipath-maestro-bpmn/_shared/test_bpmn_check.py +++ b/tests/tasks/uipath-maestro-bpmn/_shared/test_bpmn_check.py @@ -73,3 +73,46 @@ def test_bundled_start_agent_contract_matches_runtime_registry() -> None: ] assert "" not in start_agent["xmlTemplate"] assert "releaseKey" not in start_agent["xmlTemplate"] + + +def test_bundled_intsvc_activity_contract_matches_cli_manifest() -> None: + repo_root = Path(__file__).parents[4] + spec_path = repo_root / "skills/uipath-maestro-bpmn/validator/bpmn-spec.json" + spec = json.loads(spec_path.read_text(encoding="utf-8")) + activity = spec["extensionTypes"]["Intsvc.ActivityExecution"] + + assert [field["name"] for field in activity["contextFields"]] == [ + "activityConfigurationVersion", + "connectorKey", + "connection", + "folderKey", + "operation", + "objectName", + "method", + "path", + "metadata", + ] + assert activity["outputName"] == "response" + assert activity["outputType"] == "jsonSchema" + assert activity["outputSource"] == "=response" + assert activity["xmlTemplate"].startswith(" None: + repo_root = Path(__file__).parents[4] + spec_path = repo_root / "skills/uipath-maestro-bpmn/validator/bpmn-spec.json" + spec = json.loads(spec_path.read_text(encoding="utf-8")) + + for extension_type in ("Intsvc.EventTrigger", "Intsvc.WaitForEvent"): + event = spec["extensionTypes"][extension_type] + connection = next( + field for field in event["inputFields"] if field["name"] == "connectionId" + ) + assert connection["bindingInfo"] == { + "resource": "Connection", + "resourceKeyPattern": "${resourceKey}", + "propertyAttribute": "ConnectionId", + } + assert "" in event["xmlTemplate"] + assert "=bindings.{connectionBindingId}" in event["xmlTemplate"] diff --git a/tests/tasks/uipath-maestro-flow/_shared/test_same_ground_corpus.py b/tests/tasks/uipath-maestro-flow/_shared/test_same_ground_corpus.py index 95d76f0481..f934edb576 100644 --- a/tests/tasks/uipath-maestro-flow/_shared/test_same_ground_corpus.py +++ b/tests/tasks/uipath-maestro-flow/_shared/test_same_ground_corpus.py @@ -247,10 +247,10 @@ def test_paginated_lookup_accepts_both_supported_resolution_routes() -> None: assert manual_pattern.search(page_one) and manual_pattern.search(page_two) assert not manual_pattern.search(sdk_command) - # The gating criterion needs "went past page 1" on a SUCCESSFUL command: - # a page-1 call, or `--resolve channel:` typed on something other than - # `registry prepare`, earns nothing. - assert "require_success: true" in paged_block and "pass_threshold: 1.0" in paged_block + # This is advisory because a command cut off by a turn timeout has unknown + # status. It still records only successful page-2 work; the artifact check + # gates the resolved channel id. + assert "require_success: true" in paged_block and "pass_threshold: 0.0" in paged_block assert paged_pattern.search(page_two) assert paged_pattern.search(sdk_command) assert paged_pattern.search(uip_command) and not manual_pattern.search(uip_command) diff --git a/tests/tasks/uipath-maestro-flow/connector_features/paginated_reference_lookup.yaml b/tests/tasks/uipath-maestro-flow/connector_features/paginated_reference_lookup.yaml index 3791ffd3ff..2e2cc79e16 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/paginated_reference_lookup.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_features/paginated_reference_lookup.yaml @@ -42,10 +42,11 @@ success_criteria: # by hand (the collection may be exposed as `conversations` or # `curated_channels`); the SDK route resolves the name inside # `npx flow-sdk registry prepare โ€ฆ --resolve channel:`. The first criterion is -# v1-route telemetry and is advisory (the SDK arm never runs it); the second -# gates on "went past page 1" for either route and only counts a command that -# succeeded โ€” typing the flag on a failing command does not page. The final -# artifact criterion (resolved id C083AN4E61E) stays the authority. +# v1-route telemetry is advisory (the SDK arm never runs it); the second records +# "went past page 1" for either route and only counts a command that succeeded โ€” +# typing the flag on a failing command does not page. It is also advisory because +# a command cut off by a turn timeout has unknown status. The final artifact +# criterion (resolved id C083AN4E61E) stays the authority. - type: command_executed description: "Advisory (live-v1 route): paged Slack channel-resource loop ran more than once" tool_name: "Bash" @@ -61,7 +62,7 @@ success_criteria: min_count: 1 require_success: true weight: 3.0 - pass_threshold: 1.0 + pass_threshold: 0.0 - type: command_executed description: "Agent ran `uip maestro flow validate` on the generated flow"