diff --git a/.github/workflows/run-coder-eval.yml b/.github/workflows/run-coder-eval.yml index df40b5bb73..67877d8595 100644 --- a/.github/workflows/run-coder-eval.yml +++ b/.github/workflows/run-coder-eval.yml @@ -320,8 +320,9 @@ jobs: echo "${{ secrets.GH_PACKAGES_TOKEN || secrets.GH_PAT }}" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin docker pull "ghcr.io/uipath/coder-eval-agent:${CE_IMAGE_TAG}" - # Build the skills extension image (`@uipath/cli` + `@uipath/admin-tool` - # on top of coder-eval-agent). Matches smoke-skills.yml + daily.sh. + # Build the skills extension image (`@uipath/cli`, its tool plugins, and + # `@uipath/flow-sdk` on top of coder-eval-agent). Matches + # smoke-skills.yml + daily.sh. - name: Build skills Docker image env: DOCKER_BUILDKIT: "1" @@ -332,6 +333,7 @@ jobs: --build-arg CODER_EVAL_IMAGE="ghcr.io/uipath/coder-eval-agent:${CE_IMAGE_TAG}" \ --secret id=npm_auth_token,env=NPM_AUTH_TOKEN \ --build-arg CLI_VERSION="${{ inputs.cli_version }}" \ + --build-arg FLOW_SDK_VERSION="${{ vars.FLOW_SDK_VERSION || 'latest' }}" \ -t skills-image:latest \ -f tests/docker/Dockerfile \ . diff --git a/.github/workflows/smoke-skills.yml b/.github/workflows/smoke-skills.yml index 47efd682d9..6af1075033 100644 --- a/.github/workflows/smoke-skills.yml +++ b/.github/workflows/smoke-skills.yml @@ -368,12 +368,13 @@ jobs: echo "${{ secrets.GH_PACKAGES_TOKEN || secrets.GH_PAT }}" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin docker pull ghcr.io/uipath/coder-eval-agent:${{ steps.ceref.outputs.version }} - # Build custom smoke image extending coder-eval-agent with @uipath/cli and - # every whitelisted tool plugin from GitHub Packages @dev (see - # tests/docker/Dockerfile). Every push to `cli/main` publishes a - # `-dev.` prerelease under the `dev` dist-tag (cli PR #2650 renamed the - # old `alpha` tag → `dev` on 2026-06-26), so smoke tracks main — not stale - # `latest` from public npm (which lags by ~2 weeks). + # Build custom smoke image extending coder-eval-agent with @uipath/cli, + # every whitelisted tool plugin from GitHub Packages @dev, and the latest + # published @uipath/flow-sdk (see tests/docker/Dockerfile). Every push to + # `cli/main` publishes a `-dev.` prerelease under the `dev` dist-tag + # (cli PR #2650 renamed the old `alpha` tag → `dev` on 2026-06-26), so + # smoke tracks main — not stale `latest` from public npm (which lags by ~2 + # weeks). - name: Build skills smoke Docker image env: DOCKER_BUILDKIT: "1" @@ -382,6 +383,7 @@ jobs: docker build \ --build-arg CODER_EVAL_IMAGE=ghcr.io/uipath/coder-eval-agent:${{ steps.ceref.outputs.version }} \ --secret id=npm_auth_token,env=NPM_AUTH_TOKEN \ + --build-arg FLOW_SDK_VERSION="${{ vars.FLOW_SDK_VERSION || 'latest' }}" \ -t skills-image:latest \ -f tests/docker/Dockerfile \ . diff --git a/.github/workflows/sync-maestro-sdk-preview.yml b/.github/workflows/sync-maestro-sdk-preview.yml index 8c2560b9d1..69d81a1935 100644 --- a/.github/workflows/sync-maestro-sdk-preview.yml +++ b/.github/workflows/sync-maestro-sdk-preview.yml @@ -80,7 +80,7 @@ jobs: for skill in flow case bpmn; do findings="$RUNNER_TEMP/maestro-$skill-verbs.ndjson" python3 scripts/check-skill-verbs.py \ - --json "preview/uipath-maestro-$skill" > "$findings" + --json "preview/skills/uipath-maestro-$skill" > "$findings" blocking=$(jq -c \ 'select(.severity=="Stale" and .matched_prefix == null)' \ "$findings" | wc -l | tr -d ' ') @@ -108,16 +108,16 @@ jobs: run: | set -euo pipefail if git diff --quiet -- \ - preview/uipath-maestro-flow \ - preview/uipath-maestro-case \ - preview/uipath-maestro-bpmn; then + preview/skills/uipath-maestro-flow \ + preview/skills/uipath-maestro-case \ + preview/skills/uipath-maestro-bpmn; then changed=false else changed=true git diff --stat -- \ - preview/uipath-maestro-flow \ - preview/uipath-maestro-case \ - preview/uipath-maestro-bpmn + preview/skills/uipath-maestro-flow \ + preview/skills/uipath-maestro-case \ + preview/skills/uipath-maestro-bpmn fi if [ "$changed" != "$SCRIPT_CHANGED" ]; then echo "Sync script reported changed=$SCRIPT_CHANGED but git reported changed=$changed." @@ -146,9 +146,9 @@ jobs: git config user.email "41898282+github-actions[bot]@users.noreply.github.com" git checkout -b "$BRANCH" git add \ - preview/uipath-maestro-flow \ - preview/uipath-maestro-case \ - preview/uipath-maestro-bpmn + preview/skills/uipath-maestro-flow \ + preview/skills/uipath-maestro-case \ + preview/skills/uipath-maestro-bpmn git commit \ -m "chore(preview): re-sync Maestro SDK skills to flow-builder-sdk@$NEW_PIN" \ -m "🤖 Generated with Claude Code diff --git a/.github/workflows/test-helpers.yml b/.github/workflows/test-helpers.yml index 3b77dcafa2..05641572b3 100644 --- a/.github/workflows/test-helpers.yml +++ b/.github/workflows/test-helpers.yml @@ -5,6 +5,8 @@ on: paths: - 'tests/tasks/uipath-maestro-flow/**' - 'tests/tasks/uipath-maestro-case/**' + - 'tests/tasks/uipath-maestro-bpmn/**' + - 'skills/uipath-maestro-bpmn/validator/bpmn-spec.json' - 'tests/tasks/uipath-planner/**' - 'tests/tasks/uipath-agents/**' - 'tests/tasks/uipath-admin/**' @@ -15,6 +17,22 @@ on: workflow_dispatch: jobs: + pytest-bpmn-check: + runs-on: uipath-ubuntu-latest + name: maestro-bpmn checker unit tests + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: '3.13' + + - name: Install pytest + run: pip install pytest + + - name: Run pytest + run: pytest tests/tasks/uipath-maestro-bpmn/ -v + pytest-flow-check: runs-on: uipath-ubuntu-latest name: maestro-flow checker unit tests diff --git a/.github/workflows/verb-gate.yml b/.github/workflows/verb-gate.yml index 1ea283dec1..dff1c5d8f4 100644 --- a/.github/workflows/verb-gate.yml +++ b/.github/workflows/verb-gate.yml @@ -49,11 +49,11 @@ jobs: # verbs exactly like `skills/` ones, and a retired verb rots the same # way in either place. if echo "$CHANGED" | grep -qE '^(assets/uip-catalog-snapshot\.json|scripts/check-skill-verbs\.py|\.github/workflows/verb-gate\.yml)$'; then - SKILLS=$(ls -d skills/*/ preview/*/ 2>/dev/null | sed 's|/$||' | sort -u) + SKILLS=$(ls -d skills/*/ preview/skills/*/ 2>/dev/null | sed 's|/$||' | sort -u) else SKILLS=$(echo "$CHANGED" \ - | grep -E '^(skills|preview)/[^/]+/.*\.md$' \ - | sed -E 's#^((skills|preview)/[^/]+)/.*#\1#' \ + | grep -E '^(skills|preview/skills)/[^/]+/.*\.md$' \ + | sed -E 's#^((skills|preview/skills)/[^/]+)/.*#\1#' \ | sort -u || true) fi diff --git a/CODEOWNERS b/CODEOWNERS index 73b7ce755b..6e73b886ad 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -18,9 +18,9 @@ /skill-flavors/studioweb/ @astridstaicu # preview skills -/preview/uipath-maestro-flow/ @UiPath/maestro-cli-team -/preview/uipath-maestro-case/ @UiPath/maestro-cli-team -/preview/uipath-maestro-bpmn/ @UiPath/maestro-cli-team +/preview/skills/uipath-maestro-flow/ @UiPath/maestro-cli-team +/preview/skills/uipath-maestro-case/ @dmetzgar @tmatup +/preview/skills/uipath-maestro-bpmn/ @UiPath/maestro-cli-team # Activity references (bundled inside uipath-rpa skill) /skills/uipath-rpa/references/activity-docs/ @DragosUnguru diff --git a/preview/.claude-plugin/plugin.json b/preview/.claude-plugin/plugin.json new file mode 100644 index 0000000000..226b25e1a3 --- /dev/null +++ b/preview/.claude-plugin/plugin.json @@ -0,0 +1,9 @@ +{ + "name": "uipath-preview", + "version": "0.1.0", + "description": "UiPath Maestro preview skills — the builder-SDK authoring path for uipath-maestro-{flow,case,bpmn}. Synced from UiPath/flow-builder-sdk by scripts/sync-maestro-sdk-preview.mjs; loads as a Claude Code plugin root (skills//SKILL.md).", + "author": { "name": "UiPath" }, + "license": "MIT", + "homepage": "https://github.com/UiPath/skills", + "repository": "https://github.com/UiPath/skills" +} diff --git a/preview/uipath-maestro-bpmn/SKILL.md b/preview/skills/uipath-maestro-bpmn/SKILL.md similarity index 100% rename from preview/uipath-maestro-bpmn/SKILL.md rename to preview/skills/uipath-maestro-bpmn/SKILL.md diff --git a/preview/uipath-maestro-bpmn/examples/InvoiceApproval.bpmn.ts b/preview/skills/uipath-maestro-bpmn/examples/InvoiceApproval.bpmn.ts similarity index 100% rename from preview/uipath-maestro-bpmn/examples/InvoiceApproval.bpmn.ts rename to preview/skills/uipath-maestro-bpmn/examples/InvoiceApproval.bpmn.ts diff --git a/preview/uipath-maestro-bpmn/examples/NotifyChannel.bpmn.ts b/preview/skills/uipath-maestro-bpmn/examples/NotifyChannel.bpmn.ts similarity index 100% rename from preview/uipath-maestro-bpmn/examples/NotifyChannel.bpmn.ts rename to preview/skills/uipath-maestro-bpmn/examples/NotifyChannel.bpmn.ts diff --git a/preview/uipath-maestro-bpmn/references/api.md b/preview/skills/uipath-maestro-bpmn/references/api.md similarity index 100% rename from preview/uipath-maestro-bpmn/references/api.md rename to preview/skills/uipath-maestro-bpmn/references/api.md diff --git a/preview/uipath-maestro-bpmn/references/bpmn-runtime.md b/preview/skills/uipath-maestro-bpmn/references/bpmn-runtime.md similarity index 100% rename from preview/uipath-maestro-bpmn/references/bpmn-runtime.md rename to preview/skills/uipath-maestro-bpmn/references/bpmn-runtime.md diff --git a/preview/uipath-maestro-case/SKILL.md b/preview/skills/uipath-maestro-case/SKILL.md similarity index 100% rename from preview/uipath-maestro-case/SKILL.md rename to preview/skills/uipath-maestro-case/SKILL.md diff --git a/preview/uipath-maestro-case/examples/ClaimReviewSLA.case.ts b/preview/skills/uipath-maestro-case/examples/ClaimReviewSLA.case.ts similarity index 100% rename from preview/uipath-maestro-case/examples/ClaimReviewSLA.case.ts rename to preview/skills/uipath-maestro-case/examples/ClaimReviewSLA.case.ts diff --git a/preview/uipath-maestro-case/examples/IntakeBinding.case.ts b/preview/skills/uipath-maestro-case/examples/IntakeBinding.case.ts similarity index 100% rename from preview/uipath-maestro-case/examples/IntakeBinding.case.ts rename to preview/skills/uipath-maestro-case/examples/IntakeBinding.case.ts diff --git a/preview/uipath-maestro-case/examples/NightlyRollup.case.ts b/preview/skills/uipath-maestro-case/examples/NightlyRollup.case.ts similarity index 100% rename from preview/uipath-maestro-case/examples/NightlyRollup.case.ts rename to preview/skills/uipath-maestro-case/examples/NightlyRollup.case.ts diff --git a/preview/uipath-maestro-case/examples/NotifyOnApproval.case.ts b/preview/skills/uipath-maestro-case/examples/NotifyOnApproval.case.ts similarity index 100% rename from preview/uipath-maestro-case/examples/NotifyOnApproval.case.ts rename to preview/skills/uipath-maestro-case/examples/NotifyOnApproval.case.ts diff --git a/preview/uipath-maestro-case/examples/bindings.json b/preview/skills/uipath-maestro-case/examples/bindings.json similarity index 100% rename from preview/uipath-maestro-case/examples/bindings.json rename to preview/skills/uipath-maestro-case/examples/bindings.json diff --git a/preview/uipath-maestro-case/references/api.md b/preview/skills/uipath-maestro-case/references/api.md similarity index 100% rename from preview/uipath-maestro-case/references/api.md rename to preview/skills/uipath-maestro-case/references/api.md diff --git a/preview/uipath-maestro-case/references/case-runtime.md b/preview/skills/uipath-maestro-case/references/case-runtime.md similarity index 100% rename from preview/uipath-maestro-case/references/case-runtime.md rename to preview/skills/uipath-maestro-case/references/case-runtime.md diff --git a/preview/uipath-maestro-flow/SKILL.md b/preview/skills/uipath-maestro-flow/SKILL.md similarity index 100% rename from preview/uipath-maestro-flow/SKILL.md rename to preview/skills/uipath-maestro-flow/SKILL.md diff --git a/preview/uipath-maestro-flow/examples/AquariumSafetyStop.flow.ts b/preview/skills/uipath-maestro-flow/examples/AquariumSafetyStop.flow.ts similarity index 100% rename from preview/uipath-maestro-flow/examples/AquariumSafetyStop.flow.ts rename to preview/skills/uipath-maestro-flow/examples/AquariumSafetyStop.flow.ts diff --git a/preview/uipath-maestro-flow/examples/ArchiveCardExtract.flow.ts b/preview/skills/uipath-maestro-flow/examples/ArchiveCardExtract.flow.ts similarity index 100% rename from preview/uipath-maestro-flow/examples/ArchiveCardExtract.flow.ts rename to preview/skills/uipath-maestro-flow/examples/ArchiveCardExtract.flow.ts diff --git a/preview/uipath-maestro-flow/examples/BakeOffEntryForm.flow.ts b/preview/skills/uipath-maestro-flow/examples/BakeOffEntryForm.flow.ts similarity index 100% rename from preview/uipath-maestro-flow/examples/BakeOffEntryForm.flow.ts rename to preview/skills/uipath-maestro-flow/examples/BakeOffEntryForm.flow.ts diff --git a/preview/uipath-maestro-flow/examples/BeeHiveLedger.flow.ts b/preview/skills/uipath-maestro-flow/examples/BeeHiveLedger.flow.ts similarity index 100% rename from preview/uipath-maestro-flow/examples/BeeHiveLedger.flow.ts rename to preview/skills/uipath-maestro-flow/examples/BeeHiveLedger.flow.ts diff --git a/preview/uipath-maestro-flow/examples/BeltProgression.flow.ts b/preview/skills/uipath-maestro-flow/examples/BeltProgression.flow.ts similarity index 100% rename from preview/uipath-maestro-flow/examples/BeltProgression.flow.ts rename to preview/skills/uipath-maestro-flow/examples/BeltProgression.flow.ts diff --git a/preview/uipath-maestro-flow/examples/BirdCountLookup.flow.ts b/preview/skills/uipath-maestro-flow/examples/BirdCountLookup.flow.ts similarity index 100% rename from preview/uipath-maestro-flow/examples/BirdCountLookup.flow.ts rename to preview/skills/uipath-maestro-flow/examples/BirdCountLookup.flow.ts diff --git a/preview/uipath-maestro-flow/examples/ClubDirectory.flow.ts b/preview/skills/uipath-maestro-flow/examples/ClubDirectory.flow.ts similarity index 100% rename from preview/uipath-maestro-flow/examples/ClubDirectory.flow.ts rename to preview/skills/uipath-maestro-flow/examples/ClubDirectory.flow.ts diff --git a/preview/uipath-maestro-flow/examples/ConcertSoundcheck.flow.ts b/preview/skills/uipath-maestro-flow/examples/ConcertSoundcheck.flow.ts similarity index 100% rename from preview/uipath-maestro-flow/examples/ConcertSoundcheck.flow.ts rename to preview/skills/uipath-maestro-flow/examples/ConcertSoundcheck.flow.ts diff --git a/preview/uipath-maestro-flow/examples/DoorbellLog.flow.ts b/preview/skills/uipath-maestro-flow/examples/DoorbellLog.flow.ts similarity index 100% rename from preview/uipath-maestro-flow/examples/DoorbellLog.flow.ts rename to preview/skills/uipath-maestro-flow/examples/DoorbellLog.flow.ts diff --git a/preview/uipath-maestro-flow/examples/FestivalMapScaffold.flow.ts b/preview/skills/uipath-maestro-flow/examples/FestivalMapScaffold.flow.ts similarity index 100% rename from preview/uipath-maestro-flow/examples/FestivalMapScaffold.flow.ts rename to preview/skills/uipath-maestro-flow/examples/FestivalMapScaffold.flow.ts diff --git a/preview/uipath-maestro-flow/examples/FieldTripQuickForm.flow.ts b/preview/skills/uipath-maestro-flow/examples/FieldTripQuickForm.flow.ts similarity index 100% rename from preview/uipath-maestro-flow/examples/FieldTripQuickForm.flow.ts rename to preview/skills/uipath-maestro-flow/examples/FieldTripQuickForm.flow.ts diff --git a/preview/uipath-maestro-flow/examples/FossilCatalogEnrich.flow.ts b/preview/skills/uipath-maestro-flow/examples/FossilCatalogEnrich.flow.ts similarity index 100% rename from preview/uipath-maestro-flow/examples/FossilCatalogEnrich.flow.ts rename to preview/skills/uipath-maestro-flow/examples/FossilCatalogEnrich.flow.ts diff --git a/preview/uipath-maestro-flow/examples/GallerySubmission.flow.ts b/preview/skills/uipath-maestro-flow/examples/GallerySubmission.flow.ts similarity index 100% rename from preview/uipath-maestro-flow/examples/GallerySubmission.flow.ts rename to preview/skills/uipath-maestro-flow/examples/GallerySubmission.flow.ts diff --git a/preview/uipath-maestro-flow/examples/GreenhouseWatering.flow.ts b/preview/skills/uipath-maestro-flow/examples/GreenhouseWatering.flow.ts similarity index 100% rename from preview/uipath-maestro-flow/examples/GreenhouseWatering.flow.ts rename to preview/skills/uipath-maestro-flow/examples/GreenhouseWatering.flow.ts diff --git a/preview/uipath-maestro-flow/examples/HarbourRadioLine.flow.ts b/preview/skills/uipath-maestro-flow/examples/HarbourRadioLine.flow.ts similarity index 100% rename from preview/uipath-maestro-flow/examples/HarbourRadioLine.flow.ts rename to preview/skills/uipath-maestro-flow/examples/HarbourRadioLine.flow.ts diff --git a/preview/uipath-maestro-flow/examples/HerbariumDispatch.flow.ts b/preview/skills/uipath-maestro-flow/examples/HerbariumDispatch.flow.ts similarity index 100% rename from preview/uipath-maestro-flow/examples/HerbariumDispatch.flow.ts rename to preview/skills/uipath-maestro-flow/examples/HerbariumDispatch.flow.ts diff --git a/preview/uipath-maestro-flow/examples/KilnReview.flow.ts b/preview/skills/uipath-maestro-flow/examples/KilnReview.flow.ts similarity index 100% rename from preview/uipath-maestro-flow/examples/KilnReview.flow.ts rename to preview/skills/uipath-maestro-flow/examples/KilnReview.flow.ts diff --git a/preview/uipath-maestro-flow/examples/LibraryDeskChat.flow.ts b/preview/skills/uipath-maestro-flow/examples/LibraryDeskChat.flow.ts similarity index 100% rename from preview/uipath-maestro-flow/examples/LibraryDeskChat.flow.ts rename to preview/skills/uipath-maestro-flow/examples/LibraryDeskChat.flow.ts diff --git a/preview/uipath-maestro-flow/examples/LighthouseSignal.flow.ts b/preview/skills/uipath-maestro-flow/examples/LighthouseSignal.flow.ts similarity index 100% rename from preview/uipath-maestro-flow/examples/LighthouseSignal.flow.ts rename to preview/skills/uipath-maestro-flow/examples/LighthouseSignal.flow.ts diff --git a/preview/uipath-maestro-flow/examples/MeteorShowerPages.flow.ts b/preview/skills/uipath-maestro-flow/examples/MeteorShowerPages.flow.ts similarity index 100% rename from preview/uipath-maestro-flow/examples/MeteorShowerPages.flow.ts rename to preview/skills/uipath-maestro-flow/examples/MeteorShowerPages.flow.ts diff --git a/preview/uipath-maestro-flow/examples/NeighborhoodWalkPlanner.flow.ts b/preview/skills/uipath-maestro-flow/examples/NeighborhoodWalkPlanner.flow.ts similarity index 100% rename from preview/uipath-maestro-flow/examples/NeighborhoodWalkPlanner.flow.ts rename to preview/skills/uipath-maestro-flow/examples/NeighborhoodWalkPlanner.flow.ts diff --git a/preview/uipath-maestro-flow/examples/ObservatorySeeing.flow.ts b/preview/skills/uipath-maestro-flow/examples/ObservatorySeeing.flow.ts similarity index 100% rename from preview/uipath-maestro-flow/examples/ObservatorySeeing.flow.ts rename to preview/skills/uipath-maestro-flow/examples/ObservatorySeeing.flow.ts diff --git a/preview/uipath-maestro-flow/examples/OralHistoryDigest.flow.ts b/preview/skills/uipath-maestro-flow/examples/OralHistoryDigest.flow.ts similarity index 100% rename from preview/uipath-maestro-flow/examples/OralHistoryDigest.flow.ts rename to preview/skills/uipath-maestro-flow/examples/OralHistoryDigest.flow.ts diff --git a/preview/uipath-maestro-flow/examples/PlanetariumConfirmation.flow.ts b/preview/skills/uipath-maestro-flow/examples/PlanetariumConfirmation.flow.ts similarity index 100% rename from preview/uipath-maestro-flow/examples/PlanetariumConfirmation.flow.ts rename to preview/skills/uipath-maestro-flow/examples/PlanetariumConfirmation.flow.ts diff --git a/preview/uipath-maestro-flow/examples/PlantNameAdvisor.flow.ts b/preview/skills/uipath-maestro-flow/examples/PlantNameAdvisor.flow.ts similarity index 100% rename from preview/uipath-maestro-flow/examples/PlantNameAdvisor.flow.ts rename to preview/skills/uipath-maestro-flow/examples/PlantNameAdvisor.flow.ts diff --git a/preview/uipath-maestro-flow/examples/PostcardCaption.flow.ts b/preview/skills/uipath-maestro-flow/examples/PostcardCaption.flow.ts similarity index 100% rename from preview/uipath-maestro-flow/examples/PostcardCaption.flow.ts rename to preview/skills/uipath-maestro-flow/examples/PostcardCaption.flow.ts diff --git a/preview/uipath-maestro-flow/examples/PotteryStudioCallback.flow.ts b/preview/skills/uipath-maestro-flow/examples/PotteryStudioCallback.flow.ts similarity index 100% rename from preview/uipath-maestro-flow/examples/PotteryStudioCallback.flow.ts rename to preview/skills/uipath-maestro-flow/examples/PotteryStudioCallback.flow.ts diff --git a/preview/uipath-maestro-flow/examples/RecipeScaler.flow.ts b/preview/skills/uipath-maestro-flow/examples/RecipeScaler.flow.ts similarity index 100% rename from preview/uipath-maestro-flow/examples/RecipeScaler.flow.ts rename to preview/skills/uipath-maestro-flow/examples/RecipeScaler.flow.ts diff --git a/preview/uipath-maestro-flow/examples/SeedPacketReader.flow.ts b/preview/skills/uipath-maestro-flow/examples/SeedPacketReader.flow.ts similarity index 100% rename from preview/uipath-maestro-flow/examples/SeedPacketReader.flow.ts rename to preview/skills/uipath-maestro-flow/examples/SeedPacketReader.flow.ts diff --git a/preview/uipath-maestro-flow/examples/TideTableConverter.flow.ts b/preview/skills/uipath-maestro-flow/examples/TideTableConverter.flow.ts similarity index 100% rename from preview/uipath-maestro-flow/examples/TideTableConverter.flow.ts rename to preview/skills/uipath-maestro-flow/examples/TideTableConverter.flow.ts diff --git a/preview/uipath-maestro-flow/examples/TrailLogSummary.flow.ts b/preview/skills/uipath-maestro-flow/examples/TrailLogSummary.flow.ts similarity index 100% rename from preview/uipath-maestro-flow/examples/TrailLogSummary.flow.ts rename to preview/skills/uipath-maestro-flow/examples/TrailLogSummary.flow.ts diff --git a/preview/uipath-maestro-flow/examples/WorkshopInventory.flow.ts b/preview/skills/uipath-maestro-flow/examples/WorkshopInventory.flow.ts similarity index 100% rename from preview/uipath-maestro-flow/examples/WorkshopInventory.flow.ts rename to preview/skills/uipath-maestro-flow/examples/WorkshopInventory.flow.ts diff --git a/preview/uipath-maestro-flow/examples/bindings.json b/preview/skills/uipath-maestro-flow/examples/bindings.json similarity index 100% rename from preview/uipath-maestro-flow/examples/bindings.json rename to preview/skills/uipath-maestro-flow/examples/bindings.json diff --git a/preview/uipath-maestro-flow/references/CLI-LOOP.md b/preview/skills/uipath-maestro-flow/references/CLI-LOOP.md similarity index 100% rename from preview/uipath-maestro-flow/references/CLI-LOOP.md rename to preview/skills/uipath-maestro-flow/references/CLI-LOOP.md diff --git a/preview/uipath-maestro-flow/references/agent-resources.md b/preview/skills/uipath-maestro-flow/references/agent-resources.md similarity index 100% rename from preview/uipath-maestro-flow/references/agent-resources.md rename to preview/skills/uipath-maestro-flow/references/agent-resources.md diff --git a/preview/uipath-maestro-flow/references/agent.md b/preview/skills/uipath-maestro-flow/references/agent.md similarity index 100% rename from preview/uipath-maestro-flow/references/agent.md rename to preview/skills/uipath-maestro-flow/references/agent.md diff --git a/preview/uipath-maestro-flow/references/agentic-process.md b/preview/skills/uipath-maestro-flow/references/agentic-process.md similarity index 100% rename from preview/uipath-maestro-flow/references/agentic-process.md rename to preview/skills/uipath-maestro-flow/references/agentic-process.md diff --git a/preview/uipath-maestro-flow/references/api-workflow.md b/preview/skills/uipath-maestro-flow/references/api-workflow.md similarity index 100% rename from preview/uipath-maestro-flow/references/api-workflow.md rename to preview/skills/uipath-maestro-flow/references/api-workflow.md diff --git a/preview/uipath-maestro-flow/references/api.md b/preview/skills/uipath-maestro-flow/references/api.md similarity index 100% rename from preview/uipath-maestro-flow/references/api.md rename to preview/skills/uipath-maestro-flow/references/api.md diff --git a/preview/uipath-maestro-flow/references/batch-transform.md b/preview/skills/uipath-maestro-flow/references/batch-transform.md similarity index 100% rename from preview/uipath-maestro-flow/references/batch-transform.md rename to preview/skills/uipath-maestro-flow/references/batch-transform.md diff --git a/preview/uipath-maestro-flow/references/bindings.md b/preview/skills/uipath-maestro-flow/references/bindings.md similarity index 100% rename from preview/uipath-maestro-flow/references/bindings.md rename to preview/skills/uipath-maestro-flow/references/bindings.md diff --git a/preview/uipath-maestro-flow/references/branch.md b/preview/skills/uipath-maestro-flow/references/branch.md similarity index 100% rename from preview/uipath-maestro-flow/references/branch.md rename to preview/skills/uipath-maestro-flow/references/branch.md diff --git a/preview/uipath-maestro-flow/references/brownfield.md b/preview/skills/uipath-maestro-flow/references/brownfield.md similarity index 100% rename from preview/uipath-maestro-flow/references/brownfield.md rename to preview/skills/uipath-maestro-flow/references/brownfield.md diff --git a/preview/uipath-maestro-flow/references/connector-params.md b/preview/skills/uipath-maestro-flow/references/connector-params.md similarity index 100% rename from preview/uipath-maestro-flow/references/connector-params.md rename to preview/skills/uipath-maestro-flow/references/connector-params.md diff --git a/preview/uipath-maestro-flow/references/conversational.md b/preview/skills/uipath-maestro-flow/references/conversational.md similarity index 100% rename from preview/uipath-maestro-flow/references/conversational.md rename to preview/skills/uipath-maestro-flow/references/conversational.md diff --git a/preview/uipath-maestro-flow/references/data-fabric.md b/preview/skills/uipath-maestro-flow/references/data-fabric.md similarity index 100% rename from preview/uipath-maestro-flow/references/data-fabric.md rename to preview/skills/uipath-maestro-flow/references/data-fabric.md diff --git a/preview/uipath-maestro-flow/references/delay.md b/preview/skills/uipath-maestro-flow/references/delay.md similarity index 100% rename from preview/uipath-maestro-flow/references/delay.md rename to preview/skills/uipath-maestro-flow/references/delay.md diff --git a/preview/uipath-maestro-flow/references/document-pipeline.md b/preview/skills/uipath-maestro-flow/references/document-pipeline.md similarity index 100% rename from preview/uipath-maestro-flow/references/document-pipeline.md rename to preview/skills/uipath-maestro-flow/references/document-pipeline.md diff --git a/preview/uipath-maestro-flow/references/error-handling.md b/preview/skills/uipath-maestro-flow/references/error-handling.md similarity index 100% rename from preview/uipath-maestro-flow/references/error-handling.md rename to preview/skills/uipath-maestro-flow/references/error-handling.md diff --git a/preview/uipath-maestro-flow/references/evaluate.md b/preview/skills/uipath-maestro-flow/references/evaluate.md similarity index 100% rename from preview/uipath-maestro-flow/references/evaluate.md rename to preview/skills/uipath-maestro-flow/references/evaluate.md diff --git a/preview/uipath-maestro-flow/references/event-trigger.md b/preview/skills/uipath-maestro-flow/references/event-trigger.md similarity index 100% rename from preview/uipath-maestro-flow/references/event-trigger.md rename to preview/skills/uipath-maestro-flow/references/event-trigger.md diff --git a/preview/uipath-maestro-flow/references/form-trigger.md b/preview/skills/uipath-maestro-flow/references/form-trigger.md similarity index 100% rename from preview/uipath-maestro-flow/references/form-trigger.md rename to preview/skills/uipath-maestro-flow/references/form-trigger.md diff --git a/preview/uipath-maestro-flow/references/hitl.md b/preview/skills/uipath-maestro-flow/references/hitl.md similarity index 100% rename from preview/uipath-maestro-flow/references/hitl.md rename to preview/skills/uipath-maestro-flow/references/hitl.md diff --git a/preview/uipath-maestro-flow/references/http.md b/preview/skills/uipath-maestro-flow/references/http.md similarity index 100% rename from preview/uipath-maestro-flow/references/http.md rename to preview/skills/uipath-maestro-flow/references/http.md diff --git a/preview/uipath-maestro-flow/references/inline-agent.md b/preview/skills/uipath-maestro-flow/references/inline-agent.md similarity index 100% rename from preview/uipath-maestro-flow/references/inline-agent.md rename to preview/skills/uipath-maestro-flow/references/inline-agent.md diff --git a/preview/uipath-maestro-flow/references/ixp.md b/preview/skills/uipath-maestro-flow/references/ixp.md similarity index 100% rename from preview/uipath-maestro-flow/references/ixp.md rename to preview/skills/uipath-maestro-flow/references/ixp.md diff --git a/preview/uipath-maestro-flow/references/loops.md b/preview/skills/uipath-maestro-flow/references/loops.md similarity index 100% rename from preview/uipath-maestro-flow/references/loops.md rename to preview/skills/uipath-maestro-flow/references/loops.md diff --git a/preview/uipath-maestro-flow/references/manual-trigger.md b/preview/skills/uipath-maestro-flow/references/manual-trigger.md similarity index 100% rename from preview/uipath-maestro-flow/references/manual-trigger.md rename to preview/skills/uipath-maestro-flow/references/manual-trigger.md diff --git a/preview/uipath-maestro-flow/references/or-processes.md b/preview/skills/uipath-maestro-flow/references/or-processes.md similarity index 100% rename from preview/uipath-maestro-flow/references/or-processes.md rename to preview/skills/uipath-maestro-flow/references/or-processes.md diff --git a/preview/uipath-maestro-flow/references/parallel-merge.md b/preview/skills/uipath-maestro-flow/references/parallel-merge.md similarity index 100% rename from preview/uipath-maestro-flow/references/parallel-merge.md rename to preview/skills/uipath-maestro-flow/references/parallel-merge.md diff --git a/preview/uipath-maestro-flow/references/placeholder.md b/preview/skills/uipath-maestro-flow/references/placeholder.md similarity index 100% rename from preview/uipath-maestro-flow/references/placeholder.md rename to preview/skills/uipath-maestro-flow/references/placeholder.md diff --git a/preview/uipath-maestro-flow/references/published-function.md b/preview/skills/uipath-maestro-flow/references/published-function.md similarity index 100% rename from preview/uipath-maestro-flow/references/published-function.md rename to preview/skills/uipath-maestro-flow/references/published-function.md diff --git a/preview/uipath-maestro-flow/references/queue.md b/preview/skills/uipath-maestro-flow/references/queue.md similarity index 100% rename from preview/uipath-maestro-flow/references/queue.md rename to preview/skills/uipath-maestro-flow/references/queue.md diff --git a/preview/uipath-maestro-flow/references/return.md b/preview/skills/uipath-maestro-flow/references/return.md similarity index 100% rename from preview/uipath-maestro-flow/references/return.md rename to preview/skills/uipath-maestro-flow/references/return.md diff --git a/preview/uipath-maestro-flow/references/rpa-workflow.md b/preview/skills/uipath-maestro-flow/references/rpa-workflow.md similarity index 100% rename from preview/uipath-maestro-flow/references/rpa-workflow.md rename to preview/skills/uipath-maestro-flow/references/rpa-workflow.md diff --git a/preview/uipath-maestro-flow/references/scheduled-trigger.md b/preview/skills/uipath-maestro-flow/references/scheduled-trigger.md similarity index 100% rename from preview/uipath-maestro-flow/references/scheduled-trigger.md rename to preview/skills/uipath-maestro-flow/references/scheduled-trigger.md diff --git a/preview/uipath-maestro-flow/references/script.md b/preview/skills/uipath-maestro-flow/references/script.md similarity index 100% rename from preview/uipath-maestro-flow/references/script.md rename to preview/skills/uipath-maestro-flow/references/script.md diff --git a/preview/uipath-maestro-flow/references/subflow.md b/preview/skills/uipath-maestro-flow/references/subflow.md similarity index 100% rename from preview/uipath-maestro-flow/references/subflow.md rename to preview/skills/uipath-maestro-flow/references/subflow.md diff --git a/preview/uipath-maestro-flow/references/summarize.md b/preview/skills/uipath-maestro-flow/references/summarize.md similarity index 100% rename from preview/uipath-maestro-flow/references/summarize.md rename to preview/skills/uipath-maestro-flow/references/summarize.md diff --git a/preview/uipath-maestro-flow/references/switch.md b/preview/skills/uipath-maestro-flow/references/switch.md similarity index 100% rename from preview/uipath-maestro-flow/references/switch.md rename to preview/skills/uipath-maestro-flow/references/switch.md diff --git a/preview/uipath-maestro-flow/references/terminate.md b/preview/skills/uipath-maestro-flow/references/terminate.md similarity index 100% rename from preview/uipath-maestro-flow/references/terminate.md rename to preview/skills/uipath-maestro-flow/references/terminate.md diff --git a/preview/uipath-maestro-flow/references/transform.md b/preview/skills/uipath-maestro-flow/references/transform.md similarity index 100% rename from preview/uipath-maestro-flow/references/transform.md rename to preview/skills/uipath-maestro-flow/references/transform.md diff --git a/preview/uipath-maestro-flow/references/voice.md b/preview/skills/uipath-maestro-flow/references/voice.md similarity index 100% rename from preview/uipath-maestro-flow/references/voice.md rename to preview/skills/uipath-maestro-flow/references/voice.md diff --git a/scripts/sync-maestro-sdk-preview.mjs b/scripts/sync-maestro-sdk-preview.mjs index 15905fe022..10e1fa0d82 100644 --- a/scripts/sync-maestro-sdk-preview.mjs +++ b/scripts/sync-maestro-sdk-preview.mjs @@ -13,31 +13,31 @@ const sourceRoot = 'typescript/sdk'; const skillMappings = [ { source: `${sourceRoot}/skill/SKILL.md`, - target: 'preview/uipath-maestro-flow/SKILL.md', + target: 'preview/skills/uipath-maestro-flow/SKILL.md', }, { source: `${sourceRoot}/skill/SKILL-case.md`, - target: 'preview/uipath-maestro-case/SKILL.md', + target: 'preview/skills/uipath-maestro-case/SKILL.md', }, { source: `${sourceRoot}/skill/SKILL-bpmn.md`, - target: 'preview/uipath-maestro-bpmn/SKILL.md', + target: 'preview/skills/uipath-maestro-bpmn/SKILL.md', }, ]; const provenanceFiles = [ - ['preview/uipath-maestro-flow/SKILL.md', 'typescript/sdk/skill/SKILL.md'], - ['preview/uipath-maestro-case/SKILL.md', 'typescript/sdk/skill/SKILL-case.md'], - ['preview/uipath-maestro-bpmn/SKILL.md', 'typescript/sdk/skill/SKILL-bpmn.md'], + ['preview/skills/uipath-maestro-flow/SKILL.md', 'typescript/sdk/skill/SKILL.md'], + ['preview/skills/uipath-maestro-case/SKILL.md', 'typescript/sdk/skill/SKILL-case.md'], + ['preview/skills/uipath-maestro-bpmn/SKILL.md', 'typescript/sdk/skill/SKILL-bpmn.md'], ]; const managedDirectories = [ - 'preview/uipath-maestro-flow/references', - 'preview/uipath-maestro-flow/examples', - 'preview/uipath-maestro-case/references', - 'preview/uipath-maestro-case/examples', - 'preview/uipath-maestro-bpmn/references', - 'preview/uipath-maestro-bpmn/examples', + 'preview/skills/uipath-maestro-flow/references', + 'preview/skills/uipath-maestro-flow/examples', + 'preview/skills/uipath-maestro-case/references', + 'preview/skills/uipath-maestro-case/examples', + 'preview/skills/uipath-maestro-bpmn/references', + 'preview/skills/uipath-maestro-bpmn/examples', ]; const oldSiblingParagraph = [ @@ -190,13 +190,13 @@ function collectMappings(upstreamRoot, refs) { 'bpmn-runtime.md', ].includes(name)) continue; const relative = path.posix.relative(referencesRoot, source); - add(source, `preview/uipath-maestro-flow/references/${relative}`); + add(source, `preview/skills/uipath-maestro-flow/references/${relative}`); } const flowExamplesRoot = `${sourceRoot}/example-eval`; for (const source of gitFiles(upstreamRoot, ref, flowExamplesRoot)) { const relative = path.posix.relative(flowExamplesRoot, source); - add(source, `preview/uipath-maestro-flow/examples/${relative}`); + add(source, `preview/skills/uipath-maestro-flow/examples/${relative}`); } const sharedExamplesRoot = `${sourceRoot}/example`; @@ -204,26 +204,26 @@ function collectMappings(upstreamRoot, refs) { const name = path.posix.basename(source); const relative = path.posix.relative(sharedExamplesRoot, source); if (name.endsWith('.case.ts')) { - add(source, `preview/uipath-maestro-case/examples/${relative}`); + add(source, `preview/skills/uipath-maestro-case/examples/${relative}`); } else if (relative === 'case-bindings.json') { - add(source, 'preview/uipath-maestro-case/examples/bindings.json'); + add(source, 'preview/skills/uipath-maestro-case/examples/bindings.json'); } else if (name.endsWith('.bpmn.ts')) { - add(source, `preview/uipath-maestro-bpmn/examples/${relative}`); + add(source, `preview/skills/uipath-maestro-bpmn/examples/${relative}`); } } } add( `${sourceRoot}/skill/references/case-api.md`, - 'preview/uipath-maestro-case/references/api.md', + 'preview/skills/uipath-maestro-case/references/api.md', ); add( `${sourceRoot}/skill/references/bpmn-api.md`, - 'preview/uipath-maestro-bpmn/references/api.md', + 'preview/skills/uipath-maestro-bpmn/references/api.md', ); for (const [source, target] of [ - [`${sourceRoot}/skill/references/case-runtime.md`, 'preview/uipath-maestro-case/references/case-runtime.md'], - [`${sourceRoot}/skill/references/bpmn-runtime.md`, 'preview/uipath-maestro-bpmn/references/bpmn-runtime.md'], + [`${sourceRoot}/skill/references/case-runtime.md`, 'preview/skills/uipath-maestro-case/references/case-runtime.md'], + [`${sourceRoot}/skill/references/bpmn-runtime.md`, 'preview/skills/uipath-maestro-bpmn/references/bpmn-runtime.md'], ]) { if (refs.some((ref) => gitObjectExists(upstreamRoot, ref, source))) add(source, target); } @@ -406,13 +406,13 @@ function adaptBpmnRuntime(text) { } const snapshotAdaptations = new Map([ - ['preview/uipath-maestro-flow/SKILL.md', adaptFlowSkill], - ['preview/uipath-maestro-case/SKILL.md', adaptCaseSkill], - ['preview/uipath-maestro-bpmn/SKILL.md', adaptBpmnSkill], - ['preview/uipath-maestro-flow/references/api.md', adaptFlowApi], - ['preview/uipath-maestro-case/examples/NotifyOnApproval.case.ts', adaptCaseExample], - ['preview/uipath-maestro-case/references/case-runtime.md', adaptCaseRuntime], - ['preview/uipath-maestro-bpmn/references/bpmn-runtime.md', adaptBpmnRuntime], + ['preview/skills/uipath-maestro-flow/SKILL.md', adaptFlowSkill], + ['preview/skills/uipath-maestro-case/SKILL.md', adaptCaseSkill], + ['preview/skills/uipath-maestro-bpmn/SKILL.md', adaptBpmnSkill], + ['preview/skills/uipath-maestro-flow/references/api.md', adaptFlowApi], + ['preview/skills/uipath-maestro-case/examples/NotifyOnApproval.case.ts', adaptCaseExample], + ['preview/skills/uipath-maestro-case/references/case-runtime.md', adaptCaseRuntime], + ['preview/skills/uipath-maestro-bpmn/references/bpmn-runtime.md', adaptBpmnRuntime], ]); function snapshotAdaptationFor(target) { @@ -470,10 +470,10 @@ function verifySkillBody(skillsRoot, upstreamRoot, newCommit, target, source, ad function verifyExactMappings(skillsRoot, upstreamRoot, newCommit, mappings) { const adaptedTargets = new Map([ - ['preview/uipath-maestro-flow/references/api.md', adaptFlowApi], - ['preview/uipath-maestro-case/examples/NotifyOnApproval.case.ts', adaptCaseExample], - ['preview/uipath-maestro-case/references/case-runtime.md', adaptCaseRuntime], - ['preview/uipath-maestro-bpmn/references/bpmn-runtime.md', adaptBpmnRuntime], + ['preview/skills/uipath-maestro-flow/references/api.md', adaptFlowApi], + ['preview/skills/uipath-maestro-case/examples/NotifyOnApproval.case.ts', adaptCaseExample], + ['preview/skills/uipath-maestro-case/references/case-runtime.md', adaptCaseRuntime], + ['preview/skills/uipath-maestro-bpmn/references/bpmn-runtime.md', adaptBpmnRuntime], ]); const skillTargets = new Set(skillMappings.map(({ target }) => target)); for (const mapping of mappings) { @@ -492,7 +492,7 @@ function verifyExactMappings(skillsRoot, upstreamRoot, newCommit, mappings) { function verifyFrontmatterAndPins(skillsRoot, newPin) { for (const skill of ['flow', 'case', 'bpmn']) { - const target = `preview/uipath-maestro-${skill}/SKILL.md`; + const target = `preview/skills/uipath-maestro-${skill}/SKILL.md`; const text = readText(path.join(skillsRoot, target)); const frontmatter = text.match(/^---\n([\s\S]*?)\n---\n/); if (!frontmatter) fail(`${target} is missing YAML frontmatter`); @@ -510,7 +510,7 @@ function verifyFrontmatterAndPins(skillsRoot, newPin) { function verifyDeadLinks(skillsRoot) { for (const skill of ['flow', 'case', 'bpmn']) { - const skillRoot = path.join(skillsRoot, `preview/uipath-maestro-${skill}`); + const skillRoot = path.join(skillsRoot, `preview/skills/uipath-maestro-${skill}`); const documents = [path.join(skillRoot, 'SKILL.md')]; documents.push( ...walkFiles(path.join(skillRoot, 'references')).filter((file) => file.endsWith('.md')), @@ -545,7 +545,7 @@ function routerRows(text) { } function verifyRouter(skillsRoot, upstreamRoot, newCommit) { - const target = readText(path.join(skillsRoot, 'preview/uipath-maestro-flow/SKILL.md')); + const target = readText(path.join(skillsRoot, 'preview/skills/uipath-maestro-flow/SKILL.md')); const source = gitShow(upstreamRoot, newCommit, `${sourceRoot}/skill/SKILL.md`).toString('utf8'); const rows = routerRows(target); const sourceRows = routerRows(source); @@ -555,19 +555,19 @@ function verifyRouter(skillsRoot, upstreamRoot, newCommit) { for (const row of rows) { const reference = row.match(/\]\(references\/([^)]+)\)/)?.[1]; const example = row.match(/`examples\/([^`]+)`/)?.[1]; - if (!reference || !fs.existsSync(path.join(skillsRoot, 'preview/uipath-maestro-flow/references', reference))) { + if (!reference || !fs.existsSync(path.join(skillsRoot, 'preview/skills/uipath-maestro-flow/references', reference))) { fail(`Flow router row has a missing reference: ${row}`); } - if (!example || !fs.existsSync(path.join(skillsRoot, 'preview/uipath-maestro-flow/examples', example))) { + if (!example || !fs.existsSync(path.join(skillsRoot, 'preview/skills/uipath-maestro-flow/examples', example))) { fail(`Flow router row has a missing example: ${row}`); } } } function verifyStalePaths(skillsRoot) { - const flow = readText(path.join(skillsRoot, 'preview/uipath-maestro-flow/SKILL.md')); - const caseSkill = readText(path.join(skillsRoot, 'preview/uipath-maestro-case/SKILL.md')); - const bpmn = readText(path.join(skillsRoot, 'preview/uipath-maestro-bpmn/SKILL.md')); + const flow = readText(path.join(skillsRoot, 'preview/skills/uipath-maestro-flow/SKILL.md')); + const caseSkill = readText(path.join(skillsRoot, 'preview/skills/uipath-maestro-case/SKILL.md')); + const bpmn = readText(path.join(skillsRoot, 'preview/skills/uipath-maestro-bpmn/SKILL.md')); if (/\bexample\//.test(flow) || /references\/(?:case|bpmn)-api\.md/.test(flow)) { fail('Flow SKILL.md contains an upstream-only path'); } @@ -577,7 +577,7 @@ function verifyStalePaths(skillsRoot) { if (/bpmn-api|\bexample\//.test(bpmn)) { fail('BPMN SKILL.md contains an upstream-only path'); } - for (const file of walkFiles(path.join(skillsRoot, 'preview/uipath-maestro-case/examples'))) { + for (const file of walkFiles(path.join(skillsRoot, 'preview/skills/uipath-maestro-case/examples'))) { if (/case-bindings|\bexample\//.test(readText(file))) { fail(`${normalizeRelative(path.relative(skillsRoot, file))} contains an upstream-only path`); } @@ -618,7 +618,7 @@ export function verifySnapshot({ skillsRoot, upstreamRoot, newCommit, newPin, ma skillsRoot, upstreamRoot, newCommit, - 'preview/uipath-maestro-flow/SKILL.md', + 'preview/skills/uipath-maestro-flow/SKILL.md', `${sourceRoot}/skill/SKILL.md`, adaptFlowSkill, ); @@ -626,7 +626,7 @@ export function verifySnapshot({ skillsRoot, upstreamRoot, newCommit, newPin, ma skillsRoot, upstreamRoot, newCommit, - 'preview/uipath-maestro-case/SKILL.md', + 'preview/skills/uipath-maestro-case/SKILL.md', `${sourceRoot}/skill/SKILL-case.md`, adaptCaseSkill, ); @@ -634,7 +634,7 @@ export function verifySnapshot({ skillsRoot, upstreamRoot, newCommit, newPin, ma skillsRoot, upstreamRoot, newCommit, - 'preview/uipath-maestro-bpmn/SKILL.md', + 'preview/skills/uipath-maestro-bpmn/SKILL.md', `${sourceRoot}/skill/SKILL-bpmn.md`, adaptBpmnSkill, ); diff --git a/skills/uipath-maestro-bpmn/SKILL.md b/skills/uipath-maestro-bpmn/SKILL.md index b0e6f64809..6207b6d882 100644 --- a/skills/uipath-maestro-bpmn/SKILL.md +++ b/skills/uipath-maestro-bpmn/SKILL.md @@ -289,10 +289,14 @@ and honestly surfaced to the user as gaps when asked. and the process structure with the user (AskUserQuestion). 5. **The diagram is mandatory.** Import is diagram-driven — every node needs a `BPMNShape`, every flow a `BPMNEdge`, or it will not appear on the canvas. -6. **Node type is a child element, never an attribute.** Every `uipath:activity` - / `uipath:event` / `uipath:mapping` declares its type as - `` inside the wrapper. Never write - `` — the canvas will not recognize the node. +6. **Preserve the registry's node-type shape.** Most `uipath:activity` / + `uipath:event` / `uipath:mapping` templates declare their type as a nested + ``. Some runtime-authored + templates use the payload's `type` attribute instead; notably, + `Orchestrator.StartAgentJob` is a direct child of `bpmn:ServiceTask` with + ``. Both + declarations are supported. Paste the selected registry template literally + and do not normalize one form into the other. Event extension types (`Intsvc.WaitForEvent`, `Intsvc.EventTrigger`, `Maestro.ReceiveMessageEvent`, `Maestro.SendMessageEvent`) must use ``, including when the BPMN host is task-like such as diff --git a/skills/uipath-maestro-bpmn/validator/bpmn-spec.json b/skills/uipath-maestro-bpmn/validator/bpmn-spec.json index 6e6bd0db81..b4543e92b9 100644 --- a/skills/uipath-maestro-bpmn/validator/bpmn-spec.json +++ b/skills/uipath-maestro-bpmn/validator/bpmn-spec.json @@ -165,46 +165,33 @@ "extensionTag": "uipath:activity", "contextFields": [ { - "name": "releaseKey", - "displayName": "Process", + "name": "name", + "displayName": "Agent", "type": "string", - "required": false, + "required": true, "hidden": true, "isPrimaryKey": false, - "binding": false, + "binding": true, "defaultValue": null, "bindingInfo": { "resource": "process", "resourceKeyPattern": "${metadata.key}", - "propertyAttribute": "Key" + "propertyAttribute": "name" } }, - { - "name": "folderId", - "type": "string", - "required": false, - "hidden": true, - "isPrimaryKey": false, - "binding": false, - "defaultValue": null - }, { "name": "folderPath", "type": "string", - "required": false, - "hidden": true, - "isPrimaryKey": false, - "binding": false, - "defaultValue": null - }, - { - "name": "name", - "type": "string", - "required": false, + "required": true, "hidden": true, "isPrimaryKey": false, - "binding": false, - "defaultValue": null + "binding": true, + "defaultValue": null, + "bindingInfo": { + "resource": "process", + "resourceKeyPattern": "${metadata.key}", + "propertyAttribute": "folderPath" + } } ], "inputPattern": "mergedBody", @@ -218,12 +205,12 @@ "bindingInfo": { "resource": "process", "resourceKeyPattern": "${metadata.key}", - "propertyAttribute": "Key", - "contextField": "releaseKey" + "propertyAttribute": "name", + "contextField": "name" }, "requiresDiscovery": true, "isDynamic": true, - "xmlTemplate": "\n \n \n \n \n \n \n \n \n \n \n \n \n \n {incomingEdge}\n {outgoingEdge}\n", + "xmlTemplate": "\n \n \n \n \n \n \n \n \n {incomingEdge}\n {outgoingEdge}\n", "discoveryNotes": "Uses getSupportedOrchestratorTasks('agent', search) to list available agents" }, "Orchestrator.BusinessRules": { @@ -323,20 +310,24 @@ "extensionTag": "uipath:activity", "contextFields": [ { - "name": "activity", - "displayName": "Activity", + "name": "activityConfigurationVersion", + "displayName": "Activity configuration version", "type": "string", - "required": true, - "hidden": false, - "isPrimaryKey": true, + "required": false, + "hidden": true, + "isPrimaryKey": false, "binding": false, - "defaultValue": null + "defaultValue": "v1", + "enum": [ + "v1" + ], + "description": "Marks the Integration Service activity format used by this template" }, { "name": "connectorKey", "displayName": "Connector key", "type": "string", - "required": false, + "required": true, "hidden": true, "isPrimaryKey": false, "binding": false, @@ -350,7 +341,12 @@ "hidden": true, "isPrimaryKey": false, "binding": true, - "defaultValue": null + "defaultValue": null, + "bindingInfo": { + "resource": "Connection", + "resourceKeyPattern": "${resourceKey}", + "propertyAttribute": "ConnectionId" + } }, { "name": "folderKey", @@ -359,43 +355,85 @@ "required": false, "hidden": true, "isPrimaryKey": false, + "binding": true, + "defaultValue": null, + "bindingInfo": { + "resource": "Connection", + "resourceKeyPattern": "${resourceKey}", + "propertyAttribute": "folderKey" + } + }, + { + "name": "operation", + "displayName": "Operation", + "type": "string", + "required": true, + "hidden": true, + "isPrimaryKey": true, "binding": false, - "defaultValue": null + "defaultValue": null, + "description": "Activity catalog Name used as the runtime operation" + }, + { + "name": "objectName", + "displayName": "Object name", + "type": "string", + "required": true, + "hidden": true, + "isPrimaryKey": false, + "binding": false, + "defaultValue": null, + "description": "Activity catalog ObjectName" }, { "name": "method", "displayName": "Method", "type": "string", - "required": false, + "required": true, "hidden": true, "isPrimaryKey": false, "binding": false, - "defaultValue": null + "defaultValue": null, + "description": "Activity catalog MethodName" }, { "name": "path", "displayName": "Path", "type": "string", + "required": true, + "hidden": true, + "isPrimaryKey": false, + "binding": false, + "defaultValue": null, + "description": "Request path returned by resource metadata" + }, + { + "name": "metadata", + "displayName": "Metadata", + "type": "json", "required": false, "hidden": true, "isPrimaryKey": false, "binding": false, - "defaultValue": null + "defaultValue": {}, + "description": "Optional activity metadata" } ], + "inputFields": [], "inputPattern": "separateInputs", "inputName": "body", "inputTarget": "body", - "outputSource": ".", - "outputType": "custom", - "outputName": "result", + "outputSource": "=response", + "outputType": "jsonSchema", + "outputName": "response", "useRelativeSource": false, "bindingPattern": "connection", "requiresDiscovery": true, "isDynamic": true, - "xmlTemplate": "\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n {incomingEdge}\n {outgoingEdge}\n", - "discoveryNotes": "Uses getConnectorConnections(connectorKey) to list available connections. Activity/connector metadata comes from Integration Service catalog.", - "inputNotes": "isSplitInputs=false in design schema; inputs are sent as separate elements, not merged into a single JSON body", + "xmlTemplate": "\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n {incomingEdge}\n {outgoingEdge}\n", + "discoveryNotes": "Use `uip is activities list --output json` to select an activity. Set operation from Name, then describe ObjectName to get its current path, parameters, and fields.", + "inputNotes": "Add each request field from the selected operation as its own uipath:input. Keep the field's type and target: path, query, body, or file.", + "contextFieldNotes": "connection points to a root binding for a Connection resource. When folder scoping is required, folderKey points to a second binding for the same resourceKey. Use propertyAttribute ConnectionId for connection and folderKey for folderKey. Omit folderKey when the selected activity has no folder scoping, and omit both fields only when it does not use a connection.", "extensionTagNotes": "In isExecution array, NOT in isEvents array, so uses uipath:activity" }, "Intsvc.HttpExecution": { @@ -598,7 +636,7 @@ "bindingPattern": "connection", "requiresDiscovery": true, "isDynamic": false, - "xmlTemplate": "\n \n \n \n \n \n \n {incomingEdge}\n {outgoingEdge}\n", + "xmlTemplate": "\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n {incomingEdge}\n {outgoingEdge}\n", "discoveryNotes": "Uses getConnectorConnections(connectorKey) to list available connections", "inputNotes": "isSplitInputs=false; inputs are defined in input.fields (not jsonBody). Sent as separate elements.", "contextFieldNotes": "Context fields are empty in design schema; all field definitions are in inputFields below", @@ -613,7 +651,12 @@ "isPrimaryKey": false, "binding": true, "defaultValue": null, - "description": "Connection ID for the IS connector (bindable)" + "description": "Connection ID for the IS connector (bindable)", + "bindingInfo": { + "resource": "Connection", + "resourceKeyPattern": "${resourceKey}", + "propertyAttribute": "ConnectionId" + } }, { "name": "connectorKey", @@ -727,7 +770,7 @@ "bindingPattern": "connection", "requiresDiscovery": true, "isDynamic": false, - "xmlTemplate": "\n \n \n \n \n \n \n \n {incomingEdge}\n {outgoingEdge}\n", + "xmlTemplate": "\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n {incomingEdge}\n {outgoingEdge}\n", "discoveryNotes": "Uses getConnectorConnections(connectorKey) to list available connections", "inputNotes": "isSplitInputs=false; inputs defined in input.fields. Sent as separate elements.", "contextFieldNotes": "Context fields are empty in design schema; all field definitions are in inputFields below", @@ -742,7 +785,12 @@ "isPrimaryKey": false, "binding": true, "defaultValue": null, - "description": "Connection ID for the IS connector (bindable)" + "description": "Connection ID for the IS connector (bindable)", + "bindingInfo": { + "resource": "Connection", + "resourceKeyPattern": "${resourceKey}", + "propertyAttribute": "ConnectionId" + } }, { "name": "connectorKey", @@ -1840,8 +1888,8 @@ "isDynamic": false, "xmlTemplate": "\n \n \n \n \n \n {incomingEdge}\n {outgoingEdge}\n", "inputNotes": "Variable mappings use uipath:mapping with direct input/output elements, not a jsonBody pattern", - "bpmnElementNotes": "Also used on bpmn:StartEvent and bpmn:EndEvent for variable mappings. On bpmn:Task with no serviceType, defaults to BPMN.Variables.", - "extensionTagNotes": "Uses uipath:mapping (not uipath:activity). createExtensionTask checks for serviceType=BPMN.Variables on StartEvent/EndEvent/Task and routes to createModdleExtensionVariableMapping." + "bpmnElementNotes": "Also used on bpmn:StartEvent, bpmn:EndEvent, bpmn:ScriptTask, and bpmn:SubProcess for variable mappings. On bpmn:Task with no serviceType, defaults to BPMN.Variables.", + "extensionTagNotes": "Uses uipath:mapping (not uipath:activity). BPMN.Variables mappings are supported on StartEvent, EndEvent, Task, ScriptTask, and SubProcess owners." }, "BPMN.ScriptTask": { "extensionType": "BPMN.ScriptTask", diff --git a/skills/uipath-maestro-flow/SKILL.md b/skills/uipath-maestro-flow/SKILL.md index 517d2dbc29..507526f6b5 100644 --- a/skills/uipath-maestro-flow/SKILL.md +++ b/skills/uipath-maestro-flow/SKILL.md @@ -84,7 +84,7 @@ Guide for creating, editing, validating, debugging, publishing, diagnosing, and 7. **Narrate progress only when requested or clearly opted into.** Otherwise work silently and surface decisions, failures, consent gates, and the final result. When engaged, use one short plain-English line per logical step across CLI calls, shell builtins, edits, and searches; do not narrate flags or JSON structure. See [shared/ux-narration-and-todos.md](references/shared/ux-narration-and-todos.md) §When to engage. 8. **Maintain a user-facing progress list only when tracking or verbosity is requested.** In silent mode there is no user-facing todo list (the agent MAY track privately). When engaged, journeys above trivial complexity get granular step-level todos; counts follow actual work, not a target. Hide registry lookups, parsing, and file reads inside their logical step. See [shared/ux-narration-and-todos.md](references/shared/ux-narration-and-todos.md) for triggers, granularity, thresholds, and pivots. 9. **Each node has exactly one author: Edit/Write or CLI, never both.** CLI-owned nodes are connector activities (`uipath.connector..`), connector triggers (`uipath.connector.trigger..`), wait-for-events (`uipath.connector.event..`, configured like triggers), and managed HTTP (`core.action.http.v2`); add/configure them with `uip maestro flow node add` and `node configure`. All others—triggers, control flow, logic, HITL, patterns, agents, resource nodes, and queues—are user-owned and should be authored directly with `Edit` or `Write`. Never full-file `Write` a flow containing CLI-owned nodes because it can clobber CLI-set `bindings[]` and `inputs.detail`; use `Edit` or configure CLI-owned nodes last. Their `inputs.detail` is a `=jsonString:essentialConfiguration` envelope rejected when hand-authored. Inline-agent CLI is limited to `uip agent init / refresh / validate --inline-in-flow`; the `uipath.agent.autonomous` node is user-owned. Scripting (`python`, `node`, `jq`, `sed`, `awk`, or shell heredocs) is a last resort for user-owned edits and requires explicit approval after explaining state bypass, opaque diffs, and lack of interruption points. See [author/CAPABILITY.md — Node ownership](references/author/CAPABILITY.md#node-ownership--who-authors-the-node) and [author/editing-operations.md — Tool Selection Ladder](references/author/editing-operations.md#tool-selection-ladder). -10. **Batch independent tool calls and chain dependent CLI calls.** A typical greenfield build is three turns: T1 scaffold, pull the registry, and add CLI-owned nodes in one chained `Bash`, alongside independent registry/file reads; T2 read the scaffold while editing/adding the End node and edges; T3 chain configure, validate, and format. Split only when later work depends on stdout or a mutation. See [author/greenfield.md — Three-turn execution map](references/author/greenfield.md#three-turn-execution-map). +10. **Batch independent tool calls and chain dependent CLI calls.** A typical greenfield build is three turns: T1 scaffold, pull the registry, and add CLI-owned nodes in one chained `Bash`, alongside independent registry/file reads; T2 read the scaffold while editing/adding the End node and edges; T3 chain configure, validate, and format. Split only when later work depends on stdout or a mutation. A user-requested snapshot or checkpoint is exactly such a boundary: capture it immediately after the named mutation and before optional `validate`, `format`, or later mutations, even when the commands share one Bash call, and never let a later command failure skip writing it. See [author/greenfield.md — Three-turn execution map](references/author/greenfield.md#three-turn-execution-map). 11. **Cross-node bindings in `=js:` require `$vars.`** Use `=js:$vars..output...`; bare `=js:.output...` resolves to `undefined`. See [variables-and-expressions.md — IS Activity Inputs Require `=js:`](references/shared/variables-and-expressions.md#is-activity-inputs-require-js-critical). 12. **Node and edge IDs must begin with a letter.** Use descriptive camelCase node IDs and `edge____` edge IDs. Reserve UUIDs for the top-level flow `id` and `entryPointId`. diff --git a/skills/uipath-maestro-flow/references/author/brownfield.md b/skills/uipath-maestro-flow/references/author/brownfield.md index 1d8cf5b4b2..56a32e8e4c 100644 --- a/skills/uipath-maestro-flow/references/author/brownfield.md +++ b/skills/uipath-maestro-flow/references/author/brownfield.md @@ -33,6 +33,7 @@ For edits touching multiple top-level arrays, follow [parallel same-file Edit ru |---|---| | **Change a script body or node inputs** | Use `Edit` on `inputs`; do not delete/re-add because node IDs and `$vars` expressions must remain stable. Script nodes must return an object (`return { key: value }`). See [Edit/Write: Update node inputs](editing-operations-json.md#update-node-inputs). | | **Add a node between two existing nodes** | Remove the connecting edge; add the node; wire upstream → new → downstream. See [Edit/Write: Insert a node](editing-operations-json.md#insert-a-node-between-two-existing-nodes). | +| **Move / reorder a node** | Re-wire the edges, then re-point every `$vars.` read inside the moved node (a Decision's `expression`, a script body, input fields) to a node that still runs **before** it; a reference to a node that now runs later evaluates against nothing at runtime (`[400302]`/`[400300]`) even though `flow validate` passes. Nodes the branches rejoin at read the Decision as `$vars..matchedCaseId` (see [decision/impl.md — Outputs](plugins/decision/impl.md#outputs)). See [Edit/Write: Delete an edge](editing-operations-json.md#delete-an-edge) and [Edit/Write: Update node inputs](editing-operations-json.md#update-node-inputs). | | **Add a branch (decision node)** | Remove an edge; add the decision; wire true/false branches. See [Edit/Write: Insert a decision branch](editing-operations-json.md#insert-a-decision-branch). | | **Remove a node** | Remove the node; sweep edges/definitions/variables; reconnect upstream to downstream. See [Edit/Write: Remove a node](editing-operations-json.md#remove-a-node-and-reconnect). | | **Remove an edge** | Find and remove its edge ID. See [Edit/Write: Delete an edge](editing-operations-json.md#delete-an-edge). | diff --git a/skills/uipath-maestro-flow/references/author/greenfield.md b/skills/uipath-maestro-flow/references/author/greenfield.md index 72cafce301..edf0244150 100644 --- a/skills/uipath-maestro-flow/references/author/greenfield.md +++ b/skills/uipath-maestro-flow/references/author/greenfield.md @@ -18,6 +18,8 @@ For complex flows, produce a plan before building. Reference [planning-arch.md]( - The flow is a straightforward linear pipeline (trigger → action → action → end) - The user has already described the exact topology they want +**Skipping the plan never skips the node choice.** Two outcomes (*if … otherwise …*) = a `core.logic.decision` node with both `true`/`false` ports wired — never a Script ternary ([decision/planning.md](plugins/decision/planning.md)); three or more outcomes → `core.logic.switch`; a named external service → the Step 3 registry ladder below. + ### Examples **Plan:** "Build a flow that receives a Jira ticket, classifies it with an AI agent, routes urgent tickets to Slack and non-urgent to a queue, and logs everything to a Google Sheet." @@ -246,6 +248,8 @@ Then pick the first match down this ladder: Manual HTTP is the **bottom of the ladder** — only the search returning no connector authorizes it. Picking it without searching is the brand-name shortcut forbidden by [SKILL.md rule #3](../../SKILL.md#critical-rules-universal). +**Branching is a node, not a Script.** A requirement phrased as *if … otherwise …* (two outcomes) is a `core.logic.decision` node with both `true`/`false` ports wired ([decision/planning.md](plugins/decision/planning.md)); three or more outcomes → `core.logic.switch`. Do not fold the branch into a Script ternary — the branch must exist as a node in the graph. + ### Document-extraction step — route it to IxP (runs even when full planning is skipped) Pulling **named fields out of documents** (PDFs, scans, receipts, invoices, contracts, forms) is a document-extraction step — its node is an **IxP node** (`uipath.ixp.*`). It's easy to miss because the fetch/post steps around it are ordinary nodes, but the extraction in between is its own node. **Always land a node for it — never leave it out:** diff --git a/skills/uipath-maestro-flow/references/author/plugins/decision/impl.md b/skills/uipath-maestro-flow/references/author/plugins/decision/impl.md index 77be35d290..3afeef1f27 100644 --- a/skills/uipath-maestro-flow/references/author/plugins/decision/impl.md +++ b/skills/uipath-maestro-flow/references/author/plugins/decision/impl.md @@ -53,10 +53,15 @@ $vars.lookupUser.output.user !== null Output ports: `true` and `false`. Both branches must be wired. See [editing-operations.md](../../editing-operations.md) for edge add procedures. +## Outputs + +A Decision has **no `.output`**. Downstream nodes read which branch ran as `$vars..matchedCaseId` (`"true"` or `"false"`) or `$vars..matchedCase` (the branch label) — e.g. `$vars.checkStatus.matchedCaseId === "true"`. `$vars..output.matchedCaseId` is undefined and faults the reader at runtime (`[300501]`). + ## Debug | Error | Cause | Fix | | --- | --- | --- | | Expression does not evaluate to boolean | Expression returns non-boolean value | Ensure expression uses comparison operators (`===`, `>`, etc.) | | `$vars.nodeId` is undefined | Upstream node not connected or wrong ID | Check edges and node IDs | +| `[400302] Error evaluating outgoing flow expression from gateway` | `expression` reads a node that runs *after* the Decision (typical after moving the Decision earlier in the flow) | Point `expression` at a node that still runs before the Decision (e.g. the HTTP node's `$vars..output…`), not at the node the branches lead to | | Only one branch wired | Missing true or false edge | Add the missing edge — both branches are required | diff --git a/skills/uipath-maestro-flow/references/author/plugins/merge/planning.md b/skills/uipath-maestro-flow/references/author/plugins/merge/planning.md index 374957dfb9..aaf580036e 100644 --- a/skills/uipath-maestro-flow/references/author/plugins/merge/planning.md +++ b/skills/uipath-maestro-flow/references/author/plugins/merge/planning.md @@ -10,6 +10,7 @@ Use a Merge node to synchronize two or more parallel branches before continuing. - Use it when parallel branches must rejoin or after one node forks into multiple downstream branches. - Do not use it for sequential pipelines; wire nodes directly. +- Do not use it to join the `true`/`false` branches of one Decision (or the cases of one Switch); wire both ports straight into the next node, which accepts several incoming edges. A Merge waits for ALL inputs and a Decision emits exactly one, so the run would hang forever; `flow validate` refuses this shape. ## Ports diff --git a/tests/README.md b/tests/README.md index abddabdff2..b7324fa911 100644 --- a/tests/README.md +++ b/tests/README.md @@ -190,10 +190,16 @@ runner. The image build passes the package credential as exists only for the external nightly caller during migration. Regular nightly and smoke jobs continue to use `skills-image:latest`. -`flow-v2-preview.yaml` runs the three `preview/uipath-maestro-{flow,case,bpmn}` -builder-SDK skills as the ONLY skill catalog, shadowing the shipped v1 skills of -the same name, so a run measures the Flow v2 authoring path rather than a mix of -both generations. Narrowing `plugins.path` to `preview/` drops the automatic +`flow-v2-preview.yaml` runs the three `preview/skills/uipath-maestro-{flow,case,bpmn}` +builder-SDK skills as the ONLY skill catalog, so a run measures the Flow v2 +authoring path rather than a mix of both generations. `preview/` is a Claude Code +**plugin root** (`preview/.claude-plugin/plugin.json` + `preview/skills//SKILL.md`), +which is the one layout every harness loads: Claude Code requires it, the +Delegate SDK appends `/skills` to it, Codex and Antigravity accept it. Skills load +as `uipath-preview:uipath-maestro-flow` (the repo-root catalog is `uipath:`). Never +point `plugins.path` at a bare directory of skill folders: Claude Code loads +nothing from it and says so only as a per-task WARNING in task.log (every v2 run +08-20 → 09-03 ran that way). Narrowing `plugins.path` to `preview/` drops the automatic repo-root bind mount, so the root is remounted explicitly; the image also needs runtime npm auth for the `@uipath` scope. Login state mounts at `/.uipath`, identical to `nightly.yaml`. Confirm that mount resolves before a full run, or diff --git a/tests/docker/Dockerfile b/tests/docker/Dockerfile index 1d1067191b..ea7c343d7b 100644 --- a/tests/docker/Dockerfile +++ b/tests/docker/Dockerfile @@ -1,6 +1,6 @@ # syntax=docker/dockerfile:1.7 # Custom image for UiPath skills smoke tests -# Extends coder-eval-agent with UiPath CLI tools pre-installed +# Extends coder-eval-agent with UiPath CLI tools and the Flow SDK pre-installed ARG CODER_EVAL_IMAGE=coder-eval-agent:latest FROM ${CODER_EVAL_IMAGE} @@ -17,6 +17,11 @@ ARG NPM_AUTH_TOKEN= # and ETARGETs. A pin freezes the CLI; the tools ride the train's tag. ARG CLI_VERSION=dev +# The builder SDK is released independently from the CLI. Follow its rolling +# release just as CLI_VERSION follows the live CLI train; callers can still +# override this with an exact version for a reproducible comparison. +ARG FLOW_SDK_VERSION=latest + # Floor on the number of tool plugins that must install for the image to be # usable. Discovery returns the CLI's full whitelist; a few may lack a build on # the current train (e.g. model-hub-tool has no `-dev` build), so individual @@ -28,7 +33,10 @@ ARG MIN_TOOLS=25 # CLI, sync would try to re-align tools through the release-channel path (which # can't resolve `-dev` tool builds — see below) and would undo the plugins baked # here. Belt-and-suspenders with the eval harness env. -ENV UIPATH_CLI_DISABLE_VERSION_SYNC=1 +ENV UIPATH_CLI_DISABLE_VERSION_SYNC=1 \ + PREVIEW_FLOW_SDK_ROOT=/opt/preview-flow-sdk \ + PREVIEW_FLOW_SDK_ASSETS_ROOT=/opt/preview-flow-sdk-assets \ + UIP_MAESTRO_REGISTRY_HOME=/opt/preview-flow-sdk-assets/registry # Install the UiPath CLI shell from its public npm release and pre-bake EVERY # whitelisted tool plugin from GitHub Packages. Stable CLI releases such as the @@ -95,6 +103,17 @@ RUN --mount=type=secret,id=npm_auth_token \ if [ "$public_cli" = false ]; then \ npm install -g "@uipath/cli@${CLI_VERSION}"; \ fi && \ + mkdir -p "$PREVIEW_FLOW_SDK_ROOT" && \ + printf '%s\n' '{"private":true,"type":"module"}' \ + > "$PREVIEW_FLOW_SDK_ROOT/package.json" && \ + npm install --prefix "$PREVIEW_FLOW_SDK_ROOT" \ + --save-exact --no-audit --no-fund --userconfig "$npmrc" \ + "@uipath/flow-sdk@${FLOW_SDK_VERSION}" && \ + resolved_sdk_version="$(node -p \ + "require('$PREVIEW_FLOW_SDK_ROOT/node_modules/@uipath/flow-sdk/package.json').version")" && \ + printf '%s\n' "$resolved_sdk_version" > "$PREVIEW_FLOW_SDK_ROOT/flow-sdk.version" && \ + echo "Installed @uipath/flow-sdk@${resolved_sdk_version}" && \ + test -x "$PREVIEW_FLOW_SDK_ROOT/node_modules/.bin/flow-sdk" && \ uip --version && \ uip tools search --output json > /tmp/uip-tools-search.json && \ tools="$(grep -oE '@uipath/[a-z0-9-]+-tool' /tmp/uip-tools-search.json | sort -u || true)" && \ @@ -132,6 +151,51 @@ ENV DOTNET_ROOT=/usr/share/dotnet \ DOTNET_NOLOGO=1 \ DOTNET_CLI_TELEMETRY_OPTOUT=1 +# Install the released connector catalog through the product CLI. The stable +# links keep the authoring paths independent of the catalog's content hash. +# +# Where the pull lands is the CLI's business, and it moved: up to +# maestro-tool 1.202.0-dev.8425 `uip maestro registry pull` unpacked a snapshot +# under $UIP_MAESTRO_REGISTRY_HOME and wrote `current.json` there; from +# dev.8461 (cli#3823) it delegates to the Flow SDK, which caches the library +# under ~/.uipath/cache/flow-sdk/library// and writes no current.json. +# This step reads BOTH: the old pointer when the CLI still writes it, otherwise +# `uip maestro registry path` (the CLI's own answer to "where is it") — and then +# records the image's provenance pointer itself, so the per-task +# stage-preview-sdk-workspace.sh and the run scripts keep one contract +# ($UIP_MAESTRO_REGISTRY_HOME/current.json: libraryHash + snapshotPath) no matter +# which layout the installed CLI produced. Hard-coding either layout is what +# broke the 2026-09-01 image build the moment the @dev tool moved. +RUN cd "$PREVIEW_FLOW_SDK_ROOT" && \ + uip maestro registry pull --force --output json && \ + if [ -f "$UIP_MAESTRO_REGISTRY_HOME/current.json" ]; then \ + registry_snapshot="$(jq -er '.snapshotPath' "$UIP_MAESTRO_REGISTRY_HOME/current.json")"; \ + else \ + library_json="$(readlink -f "$(uip maestro registry path --library-json)")" && \ + registry_snapshot="$(dirname "$library_json")" && \ + library_hash="$(uip maestro registry path --sha)" && \ + test -n "$library_hash" && \ + mkdir -p "$UIP_MAESTRO_REGISTRY_HOME" && \ + jq -n --arg libraryHash "$library_hash" --arg snapshotPath "$registry_snapshot" \ + '{libraryHash: $libraryHash, snapshotPath: $snapshotPath}' \ + > "$UIP_MAESTRO_REGISTRY_HOME/current.json"; \ + fi && \ + test -f "$registry_snapshot/library-json/index.json" && \ + test -f "$registry_snapshot/library-md/index.json" && \ + mkdir -p "$PREVIEW_FLOW_SDK_ASSETS_ROOT" && \ + ln -s "$registry_snapshot/library-json" "$PREVIEW_FLOW_SDK_ASSETS_ROOT/library-json" && \ + ln -s "$registry_snapshot/library-md" "$PREVIEW_FLOW_SDK_ASSETS_ROOT/library-md" + +ENV FLOW_SDK_LIBRARY_JSON=/opt/preview-flow-sdk-assets/library-json \ + FLOW_SDK_LIBRARY_MD=/opt/preview-flow-sdk-assets/library-md + +# Prove that the installed SDK resolves without registry credentials and that +# the product CLI can check real source against it. +COPY tests/fixtures/maestro-flow-sdk-smoke/ /opt/maestro-flow-sdk-smoke/ +RUN cd /opt/maestro-flow-sdk-smoke && \ + ln -s ${PREVIEW_FLOW_SDK_ROOT}/node_modules node_modules && \ + uip maestro flow check Smoke.flow.ts --source + # Pre-install Python packages needed by coded-agent sandbox fixtures so agents # don't trigger a live uv-sync download (which takes 1-2 min and causes retry # loops in the turn timeline). uipath-langchain is declared in the diff --git a/tests/experiments/flow-v2-preview.yaml b/tests/experiments/flow-v2-preview.yaml index 6b9dbf9006..a10480421d 100644 --- a/tests/experiments/flow-v2-preview.yaml +++ b/tests/experiments/flow-v2-preview.yaml @@ -15,12 +15,10 @@ defaults: sandbox: driver: docker docker: - image: skills-codex:latest + image: skills-image:latest network: bridge env_passthrough_extra: - SKILLS_REPO_PATH - # GitHub Packages auth for `npm install @uipath/flow-sdk`. - - NODE_AUTH_TOKEN - UIPATH_CLI_DISABLE_VERSION_SYNC # Parity with nightly.yaml. Inert unless a Delegate runtime sets them. - UIPATH_CLI_ENABLE_ENV_AUTH @@ -46,25 +44,17 @@ defaults: allowed_tools: ["Skill", "Bash", "Read", "Write", "Edit", "Glob", "Grep"] plugins: # PREVIEW ONLY. Also the source of the automatic repo-path bind mount. + # Must stay the plugin ROOT (holds .claude-plugin/ + skills/), never preview/skills. - type: "local" path: "$SKILLS_REPO_PATH/preview" ignore_patterns: [] pre_run: - # Runtime npm auth for the @uipath scope. The shared image ships none: its - # build-time npmrc carries a literal token and is deleted in the same layer. - # @uipath/flow-sdk is published only to GitHub Packages, so without this - # every in-sandbox `npm install @uipath/flow-sdk` 404s against the public - # registry and each compile fails with it. Kept here rather than in - # tests/docker/Dockerfile so npm resolution is unchanged for every other - # suite. Written to $HOME, which is where npm resolves userconfig, so it - # holds whatever HOME the runner forwards. ${NODE_AUTH_TOKEN} stays literal: - # npm expands it at read time, so no token is written to disk. - - command: |- - mkdir -p "$HOME" && printf '%s\n' \ - '@uipath:registry=https://npm.pkg.github.com/' \ - '//npm.pkg.github.com/:_authToken=${NODE_AUTH_TOKEN}' \ - > "$HOME/.npmrc" + # Link the image's @uipath/flow-sdk and connector registry into the workspace, + # the same setup nightly.yaml runs. Unconditional here: a preview run always + # authors through the SDK. Replaces the old in-sandbox `npm install`, which the + # shared image made unnecessary when it started baking the package. + - command: bash "$SKILLS_REPO_PATH/tests/scripts/stage-preview-sdk-workspace.sh" timeout: 30 post_run: diff --git a/tests/experiments/nightly.yaml b/tests/experiments/nightly.yaml index 82acd6575b..4906f1d8d3 100644 --- a/tests/experiments/nightly.yaml +++ b/tests/experiments/nightly.yaml @@ -15,6 +15,7 @@ defaults: image: skills-image:latest env_passthrough_extra: - SKILLS_REPO_PATH + - MAESTRO_FLOW_SDK_SETUP - BEDROCK_MODEL - TASK_PARALLELISM - UIPATH_CLI_ENABLE_ENV_AUTH @@ -52,6 +53,13 @@ defaults: api_base: CODEX_BASE_URL api_key: CODEX_API_KEY + pre_run: + - command: |- + if [ "${MAESTRO_FLOW_SDK_SETUP:-0}" = "1" ]; then + bash "$SKILLS_REPO_PATH/tests/scripts/stage-preview-sdk-workspace.sh" + fi + timeout: 30 + # Sandbox cleanup between tasks. Tasks that upload Studio Web solutions # wire cleanup_solutions.py into their own post_run — that must run before # this step (still needs npm-installed CLIs). diff --git a/tests/fixtures/maestro-flow-sdk-smoke/Smoke.flow.ts b/tests/fixtures/maestro-flow-sdk-smoke/Smoke.flow.ts new file mode 100644 index 0000000000..1448d4c1e7 --- /dev/null +++ b/tests/fixtures/maestro-flow-sdk-smoke/Smoke.flow.ts @@ -0,0 +1,11 @@ +import { flow, out, script, types } from '@uipath/flow-sdk'; + +export default flow('sdk-smoke') + .name('SdkSmoke') + .version('1.0.0') + .output({ message: types.string }) + .step('makeMessage', script({ + code: 'return { message: "ready" };', + })) + .return({ message: out('makeMessage', 'message') }) + .build(); diff --git a/tests/fixtures/maestro-flow-sdk-smoke/package.json b/tests/fixtures/maestro-flow-sdk-smoke/package.json new file mode 100644 index 0000000000..e986b24bba --- /dev/null +++ b/tests/fixtures/maestro-flow-sdk-smoke/package.json @@ -0,0 +1,4 @@ +{ + "private": true, + "type": "module" +} diff --git a/tests/scripts/compose-skill-flavor.test.mjs b/tests/scripts/compose-skill-flavor.test.mjs index f373cdb352..988bb4374e 100644 --- a/tests/scripts/compose-skill-flavor.test.mjs +++ b/tests/scripts/compose-skill-flavor.test.mjs @@ -1188,7 +1188,7 @@ test("pack builds complete, marker-free default and custom npm packages", (t) => ); const customPublishDryRun = invokeNpm( repo, - ["publish", selectedStudioWeb, "--dry-run"], + ["publish", selectedStudioWeb, "--dry-run", "--tag", "preview"], { npm_config_userconfig: forcedPublicUserConfig }, ); assert.equal( @@ -1489,7 +1489,16 @@ test("root npm publish dry-run keeps the old default-only behavior and restores addPackageManifest(repo); const canonicalBefore = treeFileBytes(join(repo, "skills")); - const result = runNpm(repo, "publish", "--dry-run", "--json", "--access", "public"); + const result = runNpm( + repo, + "publish", + "--dry-run", + "--json", + "--access", + "public", + "--tag", + "test", + ); assert.equal(result.status, 0, result.stderr || result.stdout); assert.match(`${result.stdout}\n${result.stderr}`, /@uipath\/skills|"name"\s*:\s*"@uipath\/skills"/); assert.doesNotMatch(`${result.stdout}\n${result.stderr}`, /skills-studioweb/); diff --git a/tests/scripts/stage-preview-sdk-workspace.sh b/tests/scripts/stage-preview-sdk-workspace.sh new file mode 100755 index 0000000000..d3c9da0b03 --- /dev/null +++ b/tests/scripts/stage-preview-sdk-workspace.sh @@ -0,0 +1,89 @@ +#!/usr/bin/env bash +set -euo pipefail + +sdk_root=${PREVIEW_FLOW_SDK_ROOT:-/opt/preview-flow-sdk} +assets_root=${PREVIEW_FLOW_SDK_ASSETS_ROOT:-/opt/preview-flow-sdk-assets} +package_dir=$sdk_root/node_modules/@uipath/flow-sdk +library_json=${FLOW_SDK_LIBRARY_JSON:-$assets_root/library-json} +registry_root=${UIP_MAESTRO_REGISTRY_HOME:-$assets_root/registry} +registry_current=$registry_root/current.json + +[[ -f "$package_dir/package.json" ]] || { + echo "stage-preview-sdk-workspace: missing $package_dir/package.json" >&2 + exit 1 +} +[[ ! -e node_modules ]] || { + echo "stage-preview-sdk-workspace: node_modules already exists" >&2 + exit 1 +} +[[ -f "$library_json/index.json" ]] || { + echo "stage-preview-sdk-workspace: missing connector library index at $library_json" >&2 + exit 1 +} +[[ -f "$registry_current" ]] || { + echo "stage-preview-sdk-workspace: missing connector registry provenance at $registry_current" >&2 + exit 1 +} + +node - "$library_json/index.json" <<'NODE' +const fs = require('node:fs'); +const index = JSON.parse(fs.readFileSync(process.argv[2], 'utf8')); +const required = 'uipath.connector.uipath-uipath-dataservice.query-entity-records'; +if (!index.entries?.some(({ nodeType }) => nodeType === required)) { + throw new Error(`connector library is missing ${required}`); +} +NODE + +sdk_version=$(node - "$package_dir/package.json" <<'NODE' +const fs = require('node:fs'); +console.log(JSON.parse(fs.readFileSync(process.argv[2], 'utf8')).version); +NODE +) +sdk_gitref=$(node - "$package_dir/package.json" <<'NODE' +const fs = require('node:fs'); +console.log(JSON.parse(fs.readFileSync(process.argv[2], 'utf8')).gitref || ''); +NODE +) +registry_hash=$(node - "$registry_current" <<'NODE' +const fs = require('node:fs'); +const {libraryHash} = JSON.parse(fs.readFileSync(process.argv[2], 'utf8')); +if (typeof libraryHash !== 'string' || !libraryHash) { + throw new Error('connector registry provenance is missing libraryHash'); +} +console.log(libraryHash); +NODE +) + +ln -s "$sdk_root/node_modules" node_modules +node - "$sdk_version" "$sdk_gitref" "$registry_hash" <<'NODE' +const fs = require('node:fs'); +const version = process.argv[2]; +const gitref = process.argv[3]; +const connectorLibraryHash = process.argv[4]; +let packageJson = {}; +if (fs.existsSync('package.json')) { + packageJson = JSON.parse(fs.readFileSync('package.json', 'utf8')); +} +packageJson.private = true; +packageJson.type = 'module'; +packageJson.devDependencies = { + ...(packageJson.devDependencies || {}), + '@uipath/flow-sdk': version, +}; +fs.writeFileSync('package.json', `${JSON.stringify(packageJson, null, 2)}\n`); +fs.writeFileSync( + 'preview-sdk-provenance.json', + `${JSON.stringify({ + package: '@uipath/flow-sdk', + version, + gitref: gitref || null, + connector_library: true, + connector_library_hash: connectorLibraryHash, + }, null, 2)}\n`, +); +NODE + +node -e "import('@uipath/flow-sdk')" +uip maestro flow compile --help >/dev/null +printf 'stage-preview-sdk-workspace: @uipath/flow-sdk@%s, connector library %s\n' \ + "$sdk_version" "$registry_hash" diff --git a/tests/scripts/stage_shared.sh b/tests/scripts/stage_shared.sh index 7f06af4c0c..916fc3e3c6 100755 --- a/tests/scripts/stage_shared.sh +++ b/tests/scripts/stage_shared.sh @@ -19,7 +19,26 @@ # mutates the checked-out tree, not the committed repo. set -euo pipefail -cd "$(git rev-parse --show-toplevel)" +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "$(git -C "$script_dir" rev-parse --show-toplevel)" + +manifest=${STAGE_SHARED_MANIFEST:-} +if [ -n "$manifest" ]; then + : > "$manifest" +fi + +search_root=${STAGE_SHARED_ROOT:-tests/tasks} +# Compare resolved paths so `tests/tasks/../../etc` cannot pass the guard. +resolved_root=$(realpath -m -- "$search_root") +tasks_root=$(realpath -m -- "$PWD/tests/tasks") +case "$resolved_root" in + "$tasks_root" | "$tasks_root"/*) ;; + *) + echo "stage_shared: STAGE_SHARED_ROOT must be under tests/tasks: $search_root" >&2 + exit 2 + ;; +esac +search_root=${resolved_root#"$PWD"/} count=0 while IFS= read -r f; do @@ -32,8 +51,11 @@ while IFS= read -r f; do [ -d "$src" ] || continue # group has no _shared -> nothing to stage [ -e "$dir/_shared" ] && continue # already present (idempotent / hand-authored) cp -R "$src" "$dir/_shared" + if [ -n "$manifest" ]; then + printf '%s\n' "$PWD/$dir/_shared" >> "$manifest" + fi count=$((count + 1)) -done < <(git grep -lE 'from _shared|import _shared' -- 'tests/tasks/**/*.py') +done < <(git grep -lE 'from _shared|import _shared' -- "$search_root/**/*.py") # Pass 2: graders that put a `_shared` DIR itself on sys.path and then import # its modules bare (e.g. `sys.path.insert(0, .../"..", "_shared")` followed by diff --git a/tests/scripts/sync-maestro-sdk-preview.test.mjs b/tests/scripts/sync-maestro-sdk-preview.test.mjs index e4bd66eaea..b3577b4f9e 100644 --- a/tests/scripts/sync-maestro-sdk-preview.test.mjs +++ b/tests/scripts/sync-maestro-sdk-preview.test.mjs @@ -160,19 +160,19 @@ function createInitialSnapshot(skillsRoot, upstreamRoot, pin) { [ 'uipath-maestro-flow', 'typescript/sdk/skill/SKILL.md', - 'preview/uipath-maestro-flow/SKILL.md', + 'preview/skills/uipath-maestro-flow/SKILL.md', adaptFlowSkill, ], [ 'uipath-maestro-case', 'typescript/sdk/skill/SKILL-case.md', - 'preview/uipath-maestro-case/SKILL.md', + 'preview/skills/uipath-maestro-case/SKILL.md', adaptCaseSkill, ], [ 'uipath-maestro-bpmn', 'typescript/sdk/skill/SKILL-bpmn.md', - 'preview/uipath-maestro-bpmn/SKILL.md', + 'preview/skills/uipath-maestro-bpmn/SKILL.md', adaptBpmnSkill, ], ]; @@ -186,52 +186,52 @@ function createInitialSnapshot(skillsRoot, upstreamRoot, pin) { write( skillsRoot, - 'preview/uipath-maestro-flow/references/api.md', + 'preview/skills/uipath-maestro-flow/references/api.md', adaptFlowApi(read(upstreamRoot, 'typescript/sdk/skill/references/api.md')), ); write( skillsRoot, - 'preview/uipath-maestro-flow/references/guide.md', + 'preview/skills/uipath-maestro-flow/references/guide.md', read(upstreamRoot, 'typescript/sdk/skill/references/guide.md'), ); write( skillsRoot, - 'preview/uipath-maestro-case/references/api.md', + 'preview/skills/uipath-maestro-case/references/api.md', read(upstreamRoot, 'typescript/sdk/skill/references/case-api.md'), ); write( skillsRoot, - 'preview/uipath-maestro-bpmn/references/api.md', + 'preview/skills/uipath-maestro-bpmn/references/api.md', read(upstreamRoot, 'typescript/sdk/skill/references/bpmn-api.md'), ); write( skillsRoot, - 'preview/uipath-maestro-flow/examples/Foo.flow.ts', + 'preview/skills/uipath-maestro-flow/examples/Foo.flow.ts', read(upstreamRoot, 'typescript/sdk/example-eval/Foo.flow.ts'), ); write( skillsRoot, - 'preview/uipath-maestro-flow/examples/bindings.json', + 'preview/skills/uipath-maestro-flow/examples/bindings.json', read(upstreamRoot, 'typescript/sdk/example-eval/bindings.json'), ); write( skillsRoot, - 'preview/uipath-maestro-case/examples/Other.case.ts', + 'preview/skills/uipath-maestro-case/examples/Other.case.ts', read(upstreamRoot, 'typescript/sdk/example/Other.case.ts'), ); write( skillsRoot, - 'preview/uipath-maestro-case/examples/NotifyOnApproval.case.ts', + 'preview/skills/uipath-maestro-case/examples/NotifyOnApproval.case.ts', adaptCaseExample(read(upstreamRoot, 'typescript/sdk/example/NotifyOnApproval.case.ts')), ); write( skillsRoot, - 'preview/uipath-maestro-case/examples/bindings.json', + 'preview/skills/uipath-maestro-case/examples/bindings.json', read(upstreamRoot, 'typescript/sdk/example/case-bindings.json'), ); write( skillsRoot, - 'preview/uipath-maestro-bpmn/examples/NotifyChannel.bpmn.ts', + 'preview/skills/uipath-maestro-bpmn/examples/NotifyChannel.bpmn.ts', read(upstreamRoot, 'typescript/sdk/example/NotifyChannel.bpmn.ts'), ); } @@ -302,56 +302,56 @@ test('syncSnapshots three-way merges drift and reapplies only snapshot adaptatio { oldPin, newPin, changed: true }, ); assert.equal( - read(skillsRoot, 'preview/uipath-maestro-flow/references/api.md'), + read(skillsRoot, 'preview/skills/uipath-maestro-flow/references/api.md'), '# Flow API v2\n\nWorked example: `examples/Foo.flow.ts`\n', ); assert.equal( - fs.existsSync(path.join(skillsRoot, 'preview/uipath-maestro-flow/references/guide.md')), + fs.existsSync(path.join(skillsRoot, 'preview/skills/uipath-maestro-flow/references/guide.md')), false, ); assert.equal( - read(skillsRoot, 'preview/uipath-maestro-flow/references/new.md'), + read(skillsRoot, 'preview/skills/uipath-maestro-flow/references/new.md'), '# New reference\n', ); assert.equal( - read(skillsRoot, 'preview/uipath-maestro-flow/examples/Foo.flow.ts'), + read(skillsRoot, 'preview/skills/uipath-maestro-flow/examples/Foo.flow.ts'), 'export const value = 2;\n', ); assert.match( - read(skillsRoot, 'preview/uipath-maestro-flow/SKILL.md'), + read(skillsRoot, 'preview/skills/uipath-maestro-flow/SKILL.md'), /\| Script action \|.*`examples\/Foo\.flow\.ts`/, ); assert.match( - read(skillsRoot, 'preview/uipath-maestro-case/SKILL.md'), + read(skillsRoot, 'preview/skills/uipath-maestro-case/SKILL.md'), /New upstream Case guidance/, ); assert.match( - read(skillsRoot, 'preview/uipath-maestro-bpmn/SKILL.md'), + read(skillsRoot, 'preview/skills/uipath-maestro-bpmn/SKILL.md'), /# BPMN fixture v2[\s\S]*`examples\/NotifyChannel\.bpmn\.ts`/, ); assert.doesNotMatch( - read(skillsRoot, 'preview/uipath-maestro-bpmn/SKILL.md'), + read(skillsRoot, 'preview/skills/uipath-maestro-bpmn/SKILL.md'), /representative process/, ); assert.equal( - read(skillsRoot, 'preview/uipath-maestro-case/references/case-runtime.md'), + read(skillsRoot, 'preview/skills/uipath-maestro-case/references/case-runtime.md'), '# Case runtime\n\nSee [Case API](api.md).\n', ); assert.equal( - read(skillsRoot, 'preview/uipath-maestro-bpmn/references/bpmn-runtime.md'), + read(skillsRoot, 'preview/skills/uipath-maestro-bpmn/references/bpmn-runtime.md'), '# BPMN runtime\n\nSee [BPMN API](api.md).\n', ); assert.equal( - fs.existsSync(path.join(skillsRoot, 'preview/uipath-maestro-flow/references/case-runtime.md')), + fs.existsSync(path.join(skillsRoot, 'preview/skills/uipath-maestro-flow/references/case-runtime.md')), false, ); assert.equal( - fs.existsSync(path.join(skillsRoot, 'preview/uipath-maestro-flow/references/bpmn-runtime.md')), + fs.existsSync(path.join(skillsRoot, 'preview/skills/uipath-maestro-flow/references/bpmn-runtime.md')), false, ); for (const skill of ['flow', 'case', 'bpmn']) { assert.match( - read(skillsRoot, `preview/uipath-maestro-${skill}/SKILL.md`), + read(skillsRoot, `preview/skills/uipath-maestro-${skill}/SKILL.md`), new RegExp(`@ ${newPin}\\. Canonical source`), ); } diff --git a/tests/scripts/test_stage_preview_sdk_workspace.py b/tests/scripts/test_stage_preview_sdk_workspace.py new file mode 100644 index 0000000000..9625c10f5e --- /dev/null +++ b/tests/scripts/test_stage_preview_sdk_workspace.py @@ -0,0 +1,89 @@ +import json +import os +import subprocess +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[2] +STAGE_SCRIPT = REPO_ROOT / "tests/scripts/stage-preview-sdk-workspace.sh" +REQUIRED_CONNECTOR = "uipath.connector.uipath-uipath-dataservice.query-entity-records" + + +def _write_executable(path: Path, body: str) -> None: + path.write_text(body) + path.chmod(0o755) + + +def test_stages_credential_free_sdk_workspace(tmp_path: Path) -> None: + sdk_root = tmp_path / "sdk" + package_dir = sdk_root / "node_modules/@uipath/flow-sdk" + package_dir.mkdir(parents=True) + (package_dir / "package.json").write_text( + json.dumps( + { + "name": "@uipath/flow-sdk", + "version": "3.20.0", + "gitref": "efd27ce4c90ad76fc7f3c9b67f4920997b2cf0a8", + "type": "module", + "exports": "./index.js", + } + ) + ) + (package_dir / "index.js").write_text("export {};\n") + + assets_root = tmp_path / "assets" + library_json = assets_root / "typescript/sdk/lib/library-json" + library_json.mkdir(parents=True) + (library_json / "index.json").write_text( + json.dumps({"entries": [{"nodeType": REQUIRED_CONNECTOR}]}) + ) + registry_root = assets_root / "registry" + registry_root.mkdir() + registry_hash = "d34db33f" + (registry_root / "current.json").write_text( + json.dumps({"libraryHash": registry_hash}) + ) + + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + _write_executable( + bin_dir / "uip", + "#!/bin/sh\n[ \"$*\" = \"maestro flow compile --help\" ]\n", + ) + workspace = tmp_path / "workspace" + workspace.mkdir() + (workspace / "package.json").write_text( + json.dumps({"scripts": {"keep": "true"}}) + ) + env = { + **os.environ, + "PATH": f"{bin_dir}:{os.environ['PATH']}", + "PREVIEW_FLOW_SDK_ROOT": str(sdk_root), + "PREVIEW_FLOW_SDK_ASSETS_ROOT": str(assets_root), + "UIP_MAESTRO_REGISTRY_HOME": str(registry_root), + "FLOW_SDK_LIBRARY_JSON": str(library_json), + } + + completed = subprocess.run( + ["bash", str(STAGE_SCRIPT)], + cwd=workspace, + env=env, + check=True, + capture_output=True, + text=True, + ) + + assert "@uipath/flow-sdk@3.20.0" in completed.stdout + assert (workspace / "node_modules").is_symlink() + assert (workspace / "node_modules").resolve() == sdk_root / "node_modules" + package_json = json.loads((workspace / "package.json").read_text()) + assert package_json["scripts"] == {"keep": "true"} + assert package_json["devDependencies"] == {"@uipath/flow-sdk": "3.20.0"} + assert "flowSdk" not in package_json + assert json.loads((workspace / "preview-sdk-provenance.json").read_text()) == { + "package": "@uipath/flow-sdk", + "version": "3.20.0", + "gitref": "efd27ce4c90ad76fc7f3c9b67f4920997b2cf0a8", + "connector_library": True, + "connector_library_hash": registry_hash, + } diff --git a/tests/scripts/test_stage_shared.py b/tests/scripts/test_stage_shared.py new file mode 100644 index 0000000000..727fdd3b6f --- /dev/null +++ b/tests/scripts/test_stage_shared.py @@ -0,0 +1,71 @@ +"""Regression tests for staging shared checker helpers.""" + +import shutil +import subprocess +import sys +from pathlib import Path + + +SCRIPT = Path(__file__).with_name("stage_shared.sh") + + +def test_stage_shared_anchors_to_checkout_and_stages_nested_checkers(tmp_path): + repo = tmp_path / "skills" + script = repo / "tests" / "scripts" / "stage_shared.sh" + shared = repo / "tests" / "tasks" / "example" / "_shared" + shallow = repo / "tests" / "tasks" / "example" / "scenario" + nested = repo / "tests" / "tasks" / "example" / "nested" / "scenario" + unrelated_shared = repo / "tests" / "tasks" / "unrelated" / "_shared" + unrelated_task = repo / "tests" / "tasks" / "unrelated" / "scenario" + + script.parent.mkdir(parents=True) + shared.mkdir(parents=True) + shallow.mkdir(parents=True) + nested.mkdir(parents=True) + unrelated_shared.mkdir(parents=True) + unrelated_task.mkdir(parents=True) + shutil.copy2(SCRIPT, script) + (shared / "__init__.py").write_text("", encoding="utf-8") + (shared / "checker.py").write_text("VALUE = 42\n", encoding="utf-8") + checker = "from _shared.checker import VALUE\nassert VALUE == 42\n" + (shallow / "check.py").write_text(checker, encoding="utf-8") + (nested / "check.py").write_text(checker, encoding="utf-8") + (unrelated_shared / "__init__.py").write_text("", encoding="utf-8") + (unrelated_shared / "checker.py").write_text("VALUE = 42\n", encoding="utf-8") + (unrelated_task / "check.py").write_text(checker, encoding="utf-8") + + subprocess.run(["git", "init", "--quiet", str(repo)], check=True) + subprocess.run(["git", "-C", str(repo), "add", "."], check=True) + + manifest = tmp_path / "staged-paths.txt" + first = subprocess.run( + [ + "env", + f"STAGE_SHARED_MANIFEST={manifest}", + "STAGE_SHARED_ROOT=tests/tasks/example", + "bash", + str(script), + ], + cwd=tmp_path, + check=True, + capture_output=True, + text=True, + ) + second = subprocess.run( + ["env", "STAGE_SHARED_ROOT=tests/tasks/example", "bash", str(script)], + cwd=tmp_path, + check=True, + capture_output=True, + text=True, + ) + + assert "co-located _shared into 2 task dir(s)" in first.stdout + assert "co-located _shared into 0 task dir(s)" in second.stdout + assert manifest.read_text(encoding="utf-8").splitlines() == [ + str(nested / "_shared"), + str(shallow / "_shared"), + ] + for task in (shallow, nested): + assert (task / "_shared" / "checker.py").read_text(encoding="utf-8") == "VALUE = 42\n" + subprocess.run([sys.executable, str(task / "check.py")], cwd=tmp_path, check=True) + assert not (unrelated_task / "_shared").exists() diff --git a/tests/tasks/uipath-maestro-bpmn/_shared/bpmn_check.py b/tests/tasks/uipath-maestro-bpmn/_shared/bpmn_check.py index d4eeb3f0b6..9aff8ca0a8 100644 --- a/tests/tasks/uipath-maestro-bpmn/_shared/bpmn_check.py +++ b/tests/tasks/uipath-maestro-bpmn/_shared/bpmn_check.py @@ -71,7 +71,33 @@ def text_content(element: ET.Element) -> str: def has_uipath_extension(element: ET.Element, token: str) -> bool: ext = element.find("bpmn:extensionElements", NS) - return ext is not None and token in ET.tostring(ext, encoding="unicode") + if ext is not None and token in ET.tostring(ext, encoding="unicode"): + return True + return any( + token in ET.tostring(child, encoding="unicode") + for child in element + if child.tag.startswith(f"{{{NS['uipath']}}}") + ) + + +def has_typed_uipath_extension( + element: ET.Element, extension_name: str, type_value: str +) -> bool: + """Match both registry-supported UiPath payload type declarations.""" + payloads = list(element.findall(f"uipath:{extension_name}", NS)) + ext = element.find("bpmn:extensionElements", NS) + if ext is not None: + payloads.extend(ext.findall(f"uipath:{extension_name}", NS)) + + for payload in payloads: + if payload.attrib.get("type") == type_value: + return True + if any( + type_elem.attrib.get("value") == type_value + for type_elem in payload.findall("uipath:type", NS) + ): + return True + return False def require_di_for_visible_elements(root: ET.Element) -> None: diff --git a/tests/tasks/uipath-maestro-bpmn/_shared/test_bpmn_check.py b/tests/tasks/uipath-maestro-bpmn/_shared/test_bpmn_check.py new file mode 100644 index 0000000000..233316a57b --- /dev/null +++ b/tests/tasks/uipath-maestro-bpmn/_shared/test_bpmn_check.py @@ -0,0 +1,118 @@ +"""Unit tests for shared Maestro BPMN checker contracts.""" + +from __future__ import annotations + +import json +import os +import sys +import xml.etree.ElementTree as ET +from pathlib import Path + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from bpmn_check import ( # noqa: E402 + NS, + has_typed_uipath_extension, + has_uipath_extension, +) + + +def _service_task(payload: str) -> ET.Element: + return ET.fromstring( + f'{payload}' + ) + + +def test_typed_extension_accepts_nested_type_child() -> None: + task = _service_task( + """ + + + + + + """ + ) + + assert has_typed_uipath_extension(task, "activity", "Orchestrator.StartJob") + assert has_uipath_extension(task, "Orchestrator.StartJob") + + +def test_typed_extension_accepts_registry_direct_type_attribute() -> None: + task = _service_task( + '' + ) + + assert has_typed_uipath_extension(task, "activity", "Orchestrator.StartAgentJob") + assert has_uipath_extension(task, "Orchestrator.StartAgentJob") + + +def test_typed_extension_rejects_wrong_wrapper_or_type() -> None: + task = _service_task( + '' + ) + + assert not has_typed_uipath_extension(task, "activity", "Orchestrator.StartAgentJob") + assert not has_typed_uipath_extension(task, "event", "Orchestrator.StartJob") + + +def test_bundled_start_agent_contract_matches_runtime_registry() -> None: + repo_root = Path(__file__).parents[4] + spec_path = repo_root / "skills/uipath-maestro-bpmn/validator/bpmn-spec.json" + spec = json.loads(spec_path.read_text(encoding="utf-8")) + start_agent = spec["extensionTypes"]["Orchestrator.StartAgentJob"] + + assert [field["name"] for field in start_agent["contextFields"]] == [ + "name", + "folderPath", + ] + assert start_agent["bindingInfo"]["contextField"] == "name" + assert '" not in start_agent["xmlTemplate"] + assert "releaseKey" not in start_agent["xmlTemplate"] + + +def test_bundled_intsvc_activity_contract_matches_cli_manifest() -> None: + repo_root = Path(__file__).parents[4] + spec_path = repo_root / "skills/uipath-maestro-bpmn/validator/bpmn-spec.json" + spec = json.loads(spec_path.read_text(encoding="utf-8")) + activity = spec["extensionTypes"]["Intsvc.ActivityExecution"] + + assert [field["name"] for field in activity["contextFields"]] == [ + "activityConfigurationVersion", + "connectorKey", + "connection", + "folderKey", + "operation", + "objectName", + "method", + "path", + "metadata", + ] + assert activity["outputName"] == "response" + assert activity["outputType"] == "jsonSchema" + assert activity["outputSource"] == "=response" + assert activity["xmlTemplate"].startswith(" None: + repo_root = Path(__file__).parents[4] + spec_path = repo_root / "skills/uipath-maestro-bpmn/validator/bpmn-spec.json" + spec = json.loads(spec_path.read_text(encoding="utf-8")) + + for extension_type in ("Intsvc.EventTrigger", "Intsvc.WaitForEvent"): + event = spec["extensionTypes"][extension_type] + connection = next( + field for field in event["inputFields"] if field["name"] == "connectionId" + ) + assert connection["bindingInfo"] == { + "resource": "Connection", + "resourceKeyPattern": "${resourceKey}", + "propertyAttribute": "ConnectionId", + } + assert "" in event["xmlTemplate"] + assert "=bindings.{connectionBindingId}" in event["xmlTemplate"] diff --git a/tests/tasks/uipath-maestro-bpmn/author/check_simple_approval_bpmn.py b/tests/tasks/uipath-maestro-bpmn/author/check_simple_approval_bpmn.py index db48444c9a..7209a2c695 100644 --- a/tests/tasks/uipath-maestro-bpmn/author/check_simple_approval_bpmn.py +++ b/tests/tasks/uipath-maestro-bpmn/author/check_simple_approval_bpmn.py @@ -2,6 +2,7 @@ import os import sys +import xml.etree.ElementTree as ET sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from _shared.bpmn_check import ( # noqa: E402 @@ -18,6 +19,15 @@ ) +def root_variable_identifiers(process: ET.Element) -> set[str]: + return { + identifier + for var in process.findall("bpmn:extensionElements/uipath:variables/*", NS) + for identifier in (var.attrib.get("id"), var.attrib.get("name")) + if identifier + } + + def main() -> None: path, root = parse_bpmn("ExpenseApprovalBpmn") @@ -57,13 +67,9 @@ def main() -> None: process = root.find("bpmn:process", NS) if process is None: fail("missing bpmn:process element") - variable_names = { - var.attrib.get("name") - for var in process.findall("bpmn:extensionElements/uipath:variables/*", NS) - if var.attrib.get("name") - } + variable_identifiers = root_variable_identifiers(process) for required in ("expenseId", "amount", "decision"): - if required not in variable_names: + if required not in variable_identifiers: fail(f"missing root variable named {required!r}") migration_versions = { diff --git a/tests/tasks/uipath-maestro-bpmn/author/simple_approval_bpmn.yaml b/tests/tasks/uipath-maestro-bpmn/author/simple_approval_bpmn.yaml index 4cea88bdcf..1c6a7f4a78 100644 --- a/tests/tasks/uipath-maestro-bpmn/author/simple_approval_bpmn.yaml +++ b/tests/tasks/uipath-maestro-bpmn/author/simple_approval_bpmn.yaml @@ -39,15 +39,19 @@ initial_prompt: | - Do not upload, publish, deploy, debug, or run the process. success_criteria: - - type: file_exists + - type: run_command description: "BPMN source was created" - path: "ExpenseApprovalBpmn/ExpenseApprovalBpmn.bpmn" + command: "python3 -c \"from pathlib import Path; import sys; sys.exit(0 if any(Path('.').rglob('ExpenseApprovalBpmn.bpmn')) else 'ExpenseApprovalBpmn.bpmn missing')\"" + timeout: 30 + expected_exit_code: 0 weight: 1.0 pass_threshold: 1.0 - - type: file_exists - description: "Project metadata was created" - path: "ExpenseApprovalBpmn/project.uiproj" + - type: run_command + description: "Editable project source or metadata was created beside the BPMN" + command: "python3 -c \"from pathlib import Path; import sys; b=next(Path('.').rglob('ExpenseApprovalBpmn.bpmn'), None); sys.exit(0 if b and ((b.parent / 'project.uiproj').is_file() or b.with_suffix('.bpmn.ts').is_file()) else 'project.uiproj or ExpenseApprovalBpmn.bpmn.ts missing beside BPMN')\"" + timeout: 30 + expected_exit_code: 0 weight: 1.0 pass_threshold: 1.0 diff --git a/tests/tasks/uipath-maestro-bpmn/author/test_simple_approval_bpmn.py b/tests/tasks/uipath-maestro-bpmn/author/test_simple_approval_bpmn.py new file mode 100644 index 0000000000..801df09293 --- /dev/null +++ b/tests/tasks/uipath-maestro-bpmn/author/test_simple_approval_bpmn.py @@ -0,0 +1,46 @@ +import os +import sys +import xml.etree.ElementTree as ET + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from check_simple_approval_bpmn import NS, root_variable_identifiers # noqa: E402 + + +def _variable_identifiers(payload: str) -> set[str]: + root = ET.fromstring( + f'' + f"{payload}" + "" + ) + return root_variable_identifiers(root) + + +def test_variable_identity_accepts_sdk_ids_with_descriptive_names() -> None: + identifiers = _variable_identifiers( + '' + '' + '' + ) + + assert {"expenseId", "amount", "decision"} <= identifiers + + +def test_variable_identity_retains_legacy_name_compatibility() -> None: + identifiers = _variable_identifiers( + '' + '' + '' + ) + + assert {"expenseId", "amount", "decision"} <= identifiers + + +def test_variable_identity_rejects_unrelated_ids_and_names() -> None: + identifiers = _variable_identifiers( + '' + '' + ) + + assert not {"expenseId", "amount", "decision"} <= identifiers diff --git a/tests/tasks/uipath-maestro-bpmn/connector/integration_service_boundary.yaml b/tests/tasks/uipath-maestro-bpmn/connector/integration_service_boundary.yaml index 8f1c2d25fe..b7048acd79 100644 --- a/tests/tasks/uipath-maestro-bpmn/connector/integration_service_boundary.yaml +++ b/tests/tasks/uipath-maestro-bpmn/connector/integration_service_boundary.yaml @@ -31,9 +31,11 @@ initial_prompt: | - Do not upload, publish, deploy, debug, or run the process. success_criteria: - - type: file_exists + - type: run_command description: "BPMN source was created" - path: "SlackDigestBoundaryBpmn/SlackDigestBoundaryBpmn.bpmn" + command: "python3 -c \"from pathlib import Path; import sys; sys.exit(0 if any(Path('.').rglob('SlackDigestBoundaryBpmn.bpmn')) else 'SlackDigestBoundaryBpmn.bpmn missing')\"" + timeout: 30 + expected_exit_code: 0 weight: 1.0 pass_threshold: 1.0 diff --git a/tests/tasks/uipath-maestro-bpmn/debug/debug_workflow_mocked/debug_workflow_mocked.yaml b/tests/tasks/uipath-maestro-bpmn/debug/debug_workflow_mocked/debug_workflow_mocked.yaml index b09fd6d170..e060fae3b2 100644 --- a/tests/tasks/uipath-maestro-bpmn/debug/debug_workflow_mocked/debug_workflow_mocked.yaml +++ b/tests/tasks/uipath-maestro-bpmn/debug/debug_workflow_mocked/debug_workflow_mocked.yaml @@ -61,13 +61,11 @@ success_criteria: weight: 1.5 pass_threshold: 1.0 - - type: file_contains + - type: run_command description: "Summary reports the runtime variable values only obtainable from the debug session" - path: "summary.md" - includes: - - "1875" - - "AutoApproved" - - "Tier2" + command: "python3 -c \"from pathlib import Path; import sys; texts=[p.read_text(encoding='utf-8') for p in Path('.').rglob('summary.md')]; required=('1875', 'AutoApproved', 'Tier2'); sys.exit(0 if any(all(value in text for value in required) for text in texts) else 'no summary.md contains all runtime values')\"" + timeout: 30 + expected_exit_code: 0 weight: 3.0 pass_threshold: 1.0 diff --git a/tests/tasks/uipath-maestro-bpmn/multi_node/wiki_pageviews/wiki_pageviews.yaml b/tests/tasks/uipath-maestro-bpmn/multi_node/wiki_pageviews/wiki_pageviews.yaml index 75ee12fdd2..bf1a84b630 100644 --- a/tests/tasks/uipath-maestro-bpmn/multi_node/wiki_pageviews/wiki_pageviews.yaml +++ b/tests/tasks/uipath-maestro-bpmn/multi_node/wiki_pageviews/wiki_pageviews.yaml @@ -53,7 +53,7 @@ initial_prompt: | success_criteria: - type: run_command description: "BPMN source file was created" - command: "python3 -c \"from pathlib import Path; import sys; paths=[Path('WikiPageviews/WikiPageviews.bpmn'), Path('WikiPageviewsSolution/WikiPageviews/WikiPageviews.bpmn')]; sys.exit(0 if any(p.is_file() for p in paths) else 'WikiPageviews.bpmn missing in direct or solution-wrapper layout')\"" + command: "python3 -c \"from pathlib import Path; import sys; sys.exit(0 if any(Path('.').rglob('WikiPageviews.bpmn')) else 'WikiPageviews.bpmn missing')\"" timeout: 30 expected_exit_code: 0 weight: 1.5 @@ -61,7 +61,7 @@ success_criteria: - type: run_command description: "Package operate metadata was created" - command: "python3 -c \"from pathlib import Path; import sys; paths=[Path('WikiPageviews/operate.json'), Path('WikiPageviewsSolution/WikiPageviews/operate.json')]; sys.exit(0 if any(p.is_file() for p in paths) else 'operate.json missing in direct or solution-wrapper layout')\"" + command: "python3 -c \"from pathlib import Path; import sys; b=next(Path('.').rglob('WikiPageviews.bpmn'), None); sys.exit(0 if b and (b.parent / 'operate.json').is_file() else 'operate.json missing beside WikiPageviews.bpmn')\"" timeout: 30 expected_exit_code: 0 weight: 1.0 @@ -69,7 +69,7 @@ success_criteria: - type: run_command description: "The invalid-article path returns the fixed error string" - command: "python3 -c \"from pathlib import Path; import sys; paths=[Path('WikiPageviews/WikiPageviews.bpmn'), Path('WikiPageviewsSolution/WikiPageviews/WikiPageviews.bpmn')]; texts=[p.read_text(encoding='utf-8') for p in paths if p.is_file()]; sys.exit(0 if any('Article not found' in t for t in texts) else 'Article not found missing from WikiPageviews.bpmn')\"" + command: "python3 -c \"from pathlib import Path; import sys; texts=[p.read_text(encoding='utf-8') for p in Path('.').rglob('WikiPageviews.bpmn')]; sys.exit(0 if any('Article not found' in t for t in texts) else 'Article not found missing from WikiPageviews.bpmn')\"" timeout: 30 expected_exit_code: 0 weight: 1.0 diff --git a/tests/tasks/uipath-maestro-bpmn/nodes/check_contract_variant_wrappers.py b/tests/tasks/uipath-maestro-bpmn/nodes/check_contract_variant_wrappers.py index 8e25ffba04..881ae6a6f9 100644 --- a/tests/tasks/uipath-maestro-bpmn/nodes/check_contract_variant_wrappers.py +++ b/tests/tasks/uipath-maestro-bpmn/nodes/check_contract_variant_wrappers.py @@ -9,6 +9,7 @@ NS, elements, fail, + has_typed_uipath_extension, parse_bpmn, require_di_for_visible_elements, require_no_private_connector_values, @@ -16,22 +17,11 @@ ) -def has_typed_extension(element: ET.Element, extension_name: str, type_value: str) -> bool: - ext = element.find("bpmn:extensionElements", NS) - if ext is None: - return False - for payload in ext.findall(f"uipath:{extension_name}", NS): - type_elem = payload.find("uipath:type", NS) - if type_elem is not None and type_elem.attrib.get("value") == type_value: - return True - return False - - def require_wrapper(root: ET.Element, wrapper: str, extension_name: str, type_value: str) -> None: matches = [ elem for elem in elements(root, wrapper) - if has_typed_extension(elem, extension_name, type_value) + if has_typed_uipath_extension(elem, extension_name, type_value) ] if not matches: fail(f"missing bpmn:{wrapper} with {type_value} uipath:{extension_name} shell") @@ -42,6 +32,15 @@ def require_uipath_element(root: ET.Element, local_name: str, description: str) fail(f"missing {description}") +def has_preserve_only_generic_activity(root: ET.Element) -> bool: + if root.findall(".//uipath:Activity", NS): + return True + return any( + has_typed_uipath_extension(task, "activity", "uipath:Activity") + for task in elements(root, "serviceTask") + ) + + def require_uipath_attr_value( root: ET.Element, local_name: str, attr_name: str, expected_value: str, description: str ) -> None: @@ -85,7 +84,8 @@ def main() -> None: root, "scriptVersion", "value", "v2", "preserved legacy scriptVersion" ) require_uipath_element(root, "caseManagement", "preserve-only uipath:caseManagement payload") - require_uipath_element(root, "Activity", "preserve-only generic uipath:Activity payload") + if not has_preserve_only_generic_activity(root): + fail("missing preserve-only generic uipath:Activity payload") require_no_private_connector_values(root) require_sequence_integrity(root) diff --git a/tests/tasks/uipath-maestro-bpmn/nodes/hitl_rpa_wrappers.yaml b/tests/tasks/uipath-maestro-bpmn/nodes/hitl_rpa_wrappers.yaml index 6566a293b2..4775bce1b6 100644 --- a/tests/tasks/uipath-maestro-bpmn/nodes/hitl_rpa_wrappers.yaml +++ b/tests/tasks/uipath-maestro-bpmn/nodes/hitl_rpa_wrappers.yaml @@ -22,9 +22,11 @@ initial_prompt: | - Do not upload, publish, deploy, debug, or run the process. success_criteria: - - type: file_exists + - type: run_command description: "BPMN source was created" - path: "EmployeeAccessBpmn/EmployeeAccessBpmn.bpmn" + command: "python3 -c \"from pathlib import Path; import sys; sys.exit(0 if any(Path('.').rglob('EmployeeAccessBpmn.bpmn')) else 'EmployeeAccessBpmn.bpmn missing')\"" + timeout: 30 + expected_exit_code: 0 weight: 1.0 pass_threshold: 1.0 diff --git a/tests/tasks/uipath-maestro-bpmn/nodes/test_contract_variant_wrappers.py b/tests/tasks/uipath-maestro-bpmn/nodes/test_contract_variant_wrappers.py new file mode 100644 index 0000000000..59e3bd2033 --- /dev/null +++ b/tests/tasks/uipath-maestro-bpmn/nodes/test_contract_variant_wrappers.py @@ -0,0 +1,42 @@ +import os +import sys +import xml.etree.ElementTree as ET + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from check_contract_variant_wrappers import NS, has_preserve_only_generic_activity + + +def _root(payload: str) -> ET.Element: + return ET.fromstring( + f'' + f'{payload}' + "" + ) + + +def test_accepts_legacy_literal_generic_activity() -> None: + root = _root('synthetic') + + assert has_preserve_only_generic_activity(root) + + +def test_accepts_sdk_authored_generic_activity_type() -> None: + root = _root( + '' + '' + "" + ) + + assert has_preserve_only_generic_activity(root) + + +def test_rejects_unrelated_activity_type() -> None: + root = _root( + '' + '' + "" + ) + + assert not has_preserve_only_generic_activity(root) diff --git a/tests/tasks/uipath-maestro-bpmn/patterns/high_volume_batch/check_greenfield.py b/tests/tasks/uipath-maestro-bpmn/patterns/high_volume_batch/check_greenfield.py index bcf46baeeb..2583a0c58f 100755 --- a/tests/tasks/uipath-maestro-bpmn/patterns/high_volume_batch/check_greenfield.py +++ b/tests/tasks/uipath-maestro-bpmn/patterns/high_volume_batch/check_greenfield.py @@ -44,8 +44,19 @@ def main() -> None: fail(f"no gateway is reachable from {sub_id} — the block never reaches a verdict") verdict = verdicts[0] + # A BPMN message throw is the native way to send a summary. Count it as + # work on the path alongside task-shaped activities; requiring a sendTask + # would grade an implementation detail rather than the requested behavior. + message_throws = [ + event + for event in elements(root, "intermediateThrowEvent") + if event.find(f"./{{{BPMN_NS}}}messageEventDefinition") is not None + ] work_ids = ids( - elements(root, "serviceTask") + elements(root, "scriptTask") + elements(root, "sendTask") + elements(root, "serviceTask") + + elements(root, "scriptTask") + + elements(root, "sendTask") + + message_throws ) on_path = { n for n in downstream & work_ids diff --git a/tests/tasks/uipath-maestro-case/_shared/case_check.py b/tests/tasks/uipath-maestro-case/_shared/case_check.py index 7418cb06b5..51c4ac0a10 100644 --- a/tests/tasks/uipath-maestro-case/_shared/case_check.py +++ b/tests/tasks/uipath-maestro-case/_shared/case_check.py @@ -31,10 +31,45 @@ def find_caseplan(pattern: str = "**/caseplan.json") -> str: ) if not matches: _fail(f"No caseplan.json found matching {pattern}") - if len(matches) > 1: - joined = "\n - ".join(matches) - _fail(f"Multiple caseplan.json files match {pattern!r}:\n - {joined}") - return matches[0] + if len(matches) == 1: + return matches[0] + + parsed: list[tuple[str, dict | None]] = [] + for path in matches: + try: + with open(path, encoding="utf-8") as f: + value = json.load(f) + parsed.append((path, value if isinstance(value, dict) else None)) + except (OSError, json.JSONDecodeError, UnicodeDecodeError): + parsed.append((path, None)) + + substantive = [ + (path, plan) + for path, plan in parsed + if plan is not None and len(plan.get("nodes") or []) > 1 + ] + husks = [ + (path, plan) + for path, plan in parsed + if plan is not None and len(plan.get("nodes") or []) <= 1 + ] + if len(substantive) == 1 and len(substantive) + len(husks) == len(parsed): + return substantive[0][0] + + candidates = substantive or parsed + if all(plan is not None for _, plan in candidates): + signatures = { + json.dumps(plan, sort_keys=True, separators=(",", ":")) + for _, plan in candidates + } + if len(signatures) == 1: + return min( + (path for path, _ in candidates), + key=lambda path: (path.count(os.sep), len(path), path), + ) + + joined = "\n - ".join(matches) + _fail(f"Multiple distinct caseplan.json files match {pattern!r}:\n - {joined}") def read_caseplan(path: str | None = None) -> dict: @@ -43,6 +78,35 @@ def read_caseplan(path: str | None = None) -> dict: return json.load(f) +def is_non_required(item: dict) -> bool: + """Accept the Case SDK's omitted default and the explicit false form.""" + value = item.get("isRequired") + return value is None or value is False + + +def registry_audit_entries(payload: object) -> list[dict]: + """Read equivalent flat and enveloped registry-audit representations.""" + if isinstance(payload, list): + entries = payload + elif isinstance(payload, dict): + entries = next( + ( + payload[key] + for key in ("resolutions", "resources") + if isinstance(payload.get(key), list) + ), + None, + ) + if entries is None: + _fail("registry-resolved.json has no resolutions/resources list") + else: + _fail("registry-resolved.json must be a list or object envelope") + + if not all(isinstance(entry, dict) for entry in entries): + _fail("registry-resolved.json entries must be objects") + return entries + + def iter_tasks(plan: dict): """Yield every task dict from every Stage node (and legacy ExceptionStage). @@ -216,6 +280,15 @@ def find_node_by_label(plan: dict, label: str) -> dict: _fail(f"no node with data.label={label!r}; available labels: {labels}") +def selected_stage_ids(rule: dict) -> list[str]: + """Return canonical V30 stage references with legacy-schema compatibility.""" + selected = rule.get("selectedStageIds") + if isinstance(selected, list): + return [value for value in selected if isinstance(value, str) and value] + legacy_selected = rule.get("selectedStageId") + return [legacy_selected] if isinstance(legacy_selected, str) and legacy_selected else [] + + def stage_transitions(plan: dict) -> list[dict]: """Stage→stage transitions derived from entry/exit conditions. @@ -224,7 +297,7 @@ def stage_transitions(plan: dict) -> list[dict]: exists when EITHER: - ``Y``'s ``entryConditions`` carries a ``selected-stage-completed`` / - ``selected-stage-exited`` rule with ``selectedStageId == X``, OR + ``selected-stage-exited`` rule with ``X`` in ``selectedStageIds``, OR - ``X``'s ``exitConditions`` carries ``exitToStageId == Y``. ``case-entered`` entries are NOT transitions — their source is the case @@ -244,8 +317,7 @@ def stage_transitions(plan: dict) -> list[dict]: for cond in iter_stage_entry_conditions(node): for group in cond.get("rules") or []: for rule in group or []: - src = (rule or {}).get("selectedStageId") - if src: + for src in selected_stage_ids(rule or {}): pairs.add((src, nid)) for cond in iter_stage_exit_conditions(node): dst = cond.get("exitToStageId") @@ -416,7 +488,9 @@ def _get_ci(mapping: Any, *candidate_keys: str, default: Any = None) -> Any: def find_project_dir(pattern: str = "**/project.uiproj") -> str: """Return the directory holding the Case `project.uiproj`. Filters by ``ProjectType`` so a sibling Agent / RPA / Coded project in the same - solution does not collide with the Case project we want to debug. + solution does not collide with the Case project we want to debug. When a + preview author leaves both a substantive project and a generated + trigger-only scaffold, select the substantive project. """ candidates = sorted( p for p in glob.glob(pattern, recursive=True) if "/.venv/" not in p @@ -432,6 +506,12 @@ def find_project_dir(pattern: str = "**/project.uiproj") -> str: f"\n - {joined}" ) if len(case_projects) > 1: + selected, husks = _split_case_project_husks(case_projects) + if selected is not None: + print( + f"note: ignoring {len(husks)} abandoned Case project scaffold(s)" + ) + return os.path.dirname(selected) joined = "\n - ".join(case_projects) _fail( f"Multiple Case projects match {pattern!r} — refusing to guess:" @@ -442,7 +522,9 @@ def find_project_dir(pattern: str = "**/project.uiproj") -> str: def find_solution_dir(pattern: str = "**/*.uipx") -> str: """Return the directory holding the ``*.uipx`` solution manifest. - Used as ``--solution-folder`` for ``uip solution resources refresh``. + Used as ``--solution-folder`` for ``uip solution resources refresh``. A + solution whose Case project is substantive wins over generated solutions + that contain only a trigger scaffold. """ matches = sorted( p for p in glob.glob(pattern, recursive=True) if "/.venv/" not in p @@ -450,11 +532,69 @@ def find_solution_dir(pattern: str = "**/*.uipx") -> str: if not matches: _fail(f"No solution manifest found matching {pattern}") if len(matches) > 1: + selected, husks = _split_case_solution_husks(matches) + if selected is not None: + print( + f"note: ignoring {len(husks)} abandoned Case solution scaffold(s)" + ) + return os.path.dirname(selected) joined = "\n - ".join(matches) _fail(f"Multiple solution manifests match {pattern!r}:\n - {joined}") return os.path.dirname(matches[0]) +def _split_case_project_husks( + project_uiprojs: list[str], +) -> tuple[str | None, list[str]]: + counts = [ + (path, _caseplan_node_count(os.path.dirname(path))) + for path in project_uiprojs + ] + return _split_case_husks(counts) + + +def _split_case_solution_husks( + solution_uipxs: list[str], +) -> tuple[str | None, list[str]]: + counts = [ + (path, _caseplan_node_count(os.path.dirname(path))) + for path in solution_uipxs + ] + return _split_case_husks(counts) + + +def _split_case_husks( + counts: list[tuple[str, int | None]], +) -> tuple[str | None, list[str]]: + substantive = [path for path, count in counts if count is None or count > 1] + husks = [path for path, count in counts if count is not None and count <= 1] + if len(substantive) != 1: + return None, [] + selected = substantive[0] + selected_count = next(count for path, count in counts if path == selected) + if selected_count is None: + return None, [] + return selected, husks + + +def _caseplan_node_count(root: str) -> int | None: + """Return the node count for one unambiguous Case plan under ``root``.""" + plans = sorted( + path + for path in glob.glob(os.path.join(root, "**/caseplan.json"), recursive=True) + if "/.venv/" not in path + ) + if len(plans) != 1: + return None + try: + with open(plans[0], encoding="utf-8") as f: + plan = json.load(f) + except (OSError, json.JSONDecodeError, UnicodeDecodeError): + return None + nodes = plan.get("nodes") if isinstance(plan, dict) else None + return len(nodes) if isinstance(nodes, list) else None + + def run_debug( *, timeout: int = 540, diff --git a/tests/tasks/uipath-maestro-case/_shared/rule_variable_check.py b/tests/tasks/uipath-maestro-case/_shared/rule_variable_check.py index 77ce94efd7..f6f8893fe6 100644 --- a/tests/tasks/uipath-maestro-case/_shared/rule_variable_check.py +++ b/tests/tasks/uipath-maestro-case/_shared/rule_variable_check.py @@ -40,6 +40,7 @@ assert_tasks_nested, find_stages, get_bindings, + selected_stage_ids, get_case_exit_conditions, get_variables, iter_tasks, @@ -209,12 +210,12 @@ def check_rule_mapping(plan: dict) -> list[str]: ) continue if rname in ("selected-stage-completed", "selected-stage-exited"): - sid = rule.get("selectedStageId") - if sid and sid not in stage_ids: - issues.append( - f"[{scope}] {owner}: {rname} selectedStageId={sid!r} " - f"does not match any stage id" - ) + for sid in selected_stage_ids(rule): + if sid not in stage_ids: + issues.append( + f"[{scope}] {owner}: {rname} selectedStageIds contains {sid!r}, " + f"which does not match any stage id" + ) if rname == "selected-tasks-completed": for tid in rule.get("selectedTasksIds") or []: if tid not in task_ids: diff --git a/tests/tasks/uipath-maestro-case/_shared/test_case_check.py b/tests/tasks/uipath-maestro-case/_shared/test_case_check.py index e8e5c4bbfd..c7f412d6e8 100644 --- a/tests/tasks/uipath-maestro-case/_shared/test_case_check.py +++ b/tests/tasks/uipath-maestro-case/_shared/test_case_check.py @@ -10,6 +10,7 @@ """ import os +import json import subprocess import sys @@ -20,12 +21,134 @@ from case_check import ( # noqa: E402 _get_ci, collect_outputs, + find_caseplan, + find_project_dir, + find_solution_dir, + is_non_required, partition_return_to_origin_conditions, + registry_audit_entries, run_debug, ) import case_check # noqa: E402 +def _write_caseplan(path, nodes): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps({"nodes": nodes}), encoding="utf-8") + + +def _write_case_project(root, solution, project, nodes): + project_dir = root / solution / project + _write_caseplan(project_dir / "caseplan.json", nodes) + (project_dir / "project.uiproj").write_text( + json.dumps({"ProjectType": "CaseManagement"}), encoding="utf-8" + ) + (root / solution / f"{solution}.uipx").write_text("{}", encoding="utf-8") + return project_dir + + +def test_find_caseplan_prefers_one_substantive_plan_over_scaffold_husk(tmp_path, monkeypatch): + authored = tmp_path / "Case" / "Case" / "caseplan.json" + husk = tmp_path / "CaseSolution" / "Case" / "caseplan.json" + _write_caseplan(authored, [{"id": "trigger"}, {"id": "stage"}]) + _write_caseplan(husk, [{"id": "trigger"}]) + monkeypatch.chdir(tmp_path) + + assert find_caseplan() == os.path.join("Case", "Case", "caseplan.json") + + +def test_find_caseplan_accepts_identical_packaging_copy(tmp_path, monkeypatch): + authored = tmp_path / "caseplan.json" + packaged = tmp_path / "CaseSolution" / "Case" / "caseplan.json" + nodes = [{"id": "trigger"}, {"id": "stage"}] + _write_caseplan(authored, nodes) + _write_caseplan(packaged, nodes) + monkeypatch.chdir(tmp_path) + + assert find_caseplan() == "caseplan.json" + + +def test_find_caseplan_rejects_distinct_substantive_plans(tmp_path, monkeypatch): + _write_caseplan(tmp_path / "First" / "caseplan.json", [{"id": "trigger"}, {"id": "first"}]) + _write_caseplan(tmp_path / "Second" / "caseplan.json", [{"id": "trigger"}, {"id": "second"}]) + monkeypatch.chdir(tmp_path) + + with pytest.raises(SystemExit, match="Multiple distinct caseplan.json"): + find_caseplan() + + +def test_project_and_solution_discovery_prefer_substantive_case_over_husk( + tmp_path, monkeypatch, capsys +): + substantive = _write_case_project( + tmp_path, + "Case", + "Case", + [{"id": "trigger"}, {"id": "stage"}], + ) + _write_case_project( + tmp_path, + "CaseSolution", + "Case", + [{"id": "trigger"}], + ) + monkeypatch.chdir(tmp_path) + + assert find_project_dir() == os.path.relpath(substantive, tmp_path) + assert find_solution_dir() == "Case" + notes = capsys.readouterr().out + assert "ignoring 1 abandoned Case project scaffold(s)" in notes + assert "ignoring 1 abandoned Case solution scaffold(s)" in notes + + +def test_project_discovery_still_refuses_distinct_substantive_cases( + tmp_path, monkeypatch +): + _write_case_project( + tmp_path, + "FirstSolution", + "First", + [{"id": "trigger"}, {"id": "first"}], + ) + _write_case_project( + tmp_path, + "SecondSolution", + "Second", + [{"id": "trigger"}, {"id": "second"}], + ) + monkeypatch.chdir(tmp_path) + + with pytest.raises(SystemExit, match="Multiple Case projects match"): + find_project_dir() + with pytest.raises(SystemExit, match="Multiple solution manifests match"): + find_solution_dir() + + +def test_non_required_accepts_omitted_or_explicit_false(): + assert is_non_required({}) + assert is_non_required({"isRequired": False}) + + +def test_non_required_rejects_required_or_invalid_values(): + assert not is_non_required({"isRequired": True}) + assert not is_non_required({"isRequired": "false"}) + + +def test_registry_audit_entries_accepts_flat_and_named_envelopes(): + entries = [{"task": "Post Invoice"}] + + assert registry_audit_entries(entries) == entries + assert registry_audit_entries({"resolutions": entries}) == entries + assert registry_audit_entries({"resources": entries}) == entries + + +def test_registry_audit_entries_rejects_missing_or_non_object_entries(): + with pytest.raises(SystemExit, match="no resolutions/resources list"): + registry_audit_entries({"status": "resolved"}) + with pytest.raises(SystemExit, match="entries must be objects"): + registry_audit_entries({"resources": ["Post Invoice"]}) + + def test_get_ci_reads_camelcase_and_pascalcase(): assert _get_ci({"finalStatus": "Completed"}, "finalStatus", "FinalStatus") == "Completed" assert _get_ci({"FinalStatus": "Completed"}, "finalStatus", "FinalStatus") == "Completed" diff --git a/tests/tasks/uipath-maestro-case/_shared/test_same_ground_corpus.py b/tests/tasks/uipath-maestro-case/_shared/test_same_ground_corpus.py new file mode 100644 index 0000000000..db3607cc80 --- /dev/null +++ b/tests/tasks/uipath-maestro-case/_shared/test_same_ground_corpus.py @@ -0,0 +1,46 @@ +"""Permanent cross-loop grading contracts for Maestro Case eval tasks.""" + +from __future__ import annotations + +import re +from pathlib import Path + + +CASE_TASKS = Path(__file__).resolve().parent.parent + + +def _block(text: str, key: str) -> str: + match = re.search(rf"(?ms)^{re.escape(key)}:[^\n]*\n(.*?)(?=^[A-Za-z_][\w-]*:|\Z)", text) + return match.group(0) if match else "" + + +def _criterion_blocks(text: str) -> list[tuple[str, str]]: + section = _block(text, "success_criteria") + return [ + (match.group(1), match.group(0)) + for match in re.finditer( + r"(?ms)^ - type:\s*([^\s#]+).*?(?=^ - type:|^[A-Za-z_][\w-]*:|\Z)", + section, + ) + ] + + +def _tagged_tasks() -> list[tuple[str, str]]: + tasks = [] + for path in sorted(CASE_TASKS.rglob("*.yaml")): + text = path.read_text() + if "uipath-maestro-case" in _block(text, "tags"): + tasks.append((path.relative_to(CASE_TASKS).as_posix(), text)) + return tasks + + +def test_skill_telemetry_is_advisory_across_the_case_corpus() -> None: + offenders = set() + for relative, text in _tagged_tasks(): + for criterion_type, criterion in _criterion_blocks(text): + if criterion_type != "skill_triggered": + continue + threshold = re.search(r"(?m)^\s+pass_threshold:\s*([0-9.]+)", criterion) + if threshold is None or float(threshold.group(1)) > 0: + offenders.add(relative) + assert offenders == set() diff --git a/tests/tasks/uipath-maestro-case/athena_cm_event/athena_cm_event.yaml b/tests/tasks/uipath-maestro-case/athena_cm_event/athena_cm_event.yaml index 9b10fea62a..701301e765 100644 --- a/tests/tasks/uipath-maestro-case/athena_cm_event/athena_cm_event.yaml +++ b/tests/tasks/uipath-maestro-case/athena_cm_event/athena_cm_event.yaml @@ -47,11 +47,11 @@ initial_prompt: | success_criteria: - type: skill_triggered - description: "Agent invoked the uipath-maestro-case skill" + description: "Advisory: agent invoked the uipath-maestro-case skill" skill_name: "uipath-maestro-case" expected_skill: "uipath-maestro-case" weight: 1.0 - pass_threshold: 1.0 + pass_threshold: 0.0 - type: command_executed description: "Agent ran case validation (advisory; outcome is graded below)" diff --git a/tests/tasks/uipath-maestro-case/athena_cm_event/check_athena_cm_event_case.py b/tests/tasks/uipath-maestro-case/athena_cm_event/check_athena_cm_event_case.py index 6491465bc7..f719b6311c 100644 --- a/tests/tasks/uipath-maestro-case/athena_cm_event/check_athena_cm_event_case.py +++ b/tests/tasks/uipath-maestro-case/athena_cm_event/check_athena_cm_event_case.py @@ -17,6 +17,7 @@ get_case_exit_conditions, get_variables, read_caseplan, + selected_stage_ids, ) @@ -91,9 +92,14 @@ def iter_rules(conditions: Iterable[dict]) -> Iterable[dict]: def has_rule(conditions: Iterable[dict], rule_name: str, **expected: object) -> bool: + def matches(rule: dict, field: str, value: object) -> bool: + if field == "selectedStageId": + return value in selected_stage_ids(rule) + return rule.get(field) == value + return any( rule.get("rule") == rule_name - and all(rule.get(field) == value for field, value in expected.items()) + and all(matches(rule, field, value) for field, value in expected.items()) for rule in iter_rules(conditions) ) diff --git a/tests/tasks/uipath-maestro-case/build_from_planner_sdd/build_from_planner_sdd.yaml b/tests/tasks/uipath-maestro-case/build_from_planner_sdd/build_from_planner_sdd.yaml index f22dd29736..f4a7db3faf 100644 --- a/tests/tasks/uipath-maestro-case/build_from_planner_sdd/build_from_planner_sdd.yaml +++ b/tests/tasks/uipath-maestro-case/build_from_planner_sdd/build_from_planner_sdd.yaml @@ -54,11 +54,11 @@ initial_prompt: | success_criteria: - type: skill_triggered - description: "Agent invoked the uipath-maestro-case skill (not planner Lane A)" + description: "Advisory: agent invoked the uipath-maestro-case skill (not planner Lane A)" skill_name: "uipath-maestro-case" expected_skill: "uipath-maestro-case" weight: 2.0 - pass_threshold: 1.0 + pass_threshold: 0.0 - type: llm_judge description: "Final reply names both unresolved resource identities as deferred to the build machine" diff --git a/tests/tasks/uipath-maestro-case/connector_features/connection_parity/connection_parity.yaml b/tests/tasks/uipath-maestro-case/connector_features/connection_parity/connection_parity.yaml index e4182eefd0..03f3b7be01 100644 --- a/tests/tasks/uipath-maestro-case/connector_features/connection_parity/connection_parity.yaml +++ b/tests/tasks/uipath-maestro-case/connector_features/connection_parity/connection_parity.yaml @@ -125,11 +125,11 @@ initial_prompt: | success_criteria: - type: skill_triggered - description: "Agent invoked the uipath-maestro-case skill" + description: "Advisory: agent invoked the uipath-maestro-case skill" skill_name: "uipath-maestro-case" expected_skill: "uipath-maestro-case" weight: 1.0 - pass_threshold: 1.0 + pass_threshold: 0.0 - type: run_command description: "uip maestro case validate passes on the generated caseplan.json" diff --git a/tests/tasks/uipath-maestro-case/connector_features/connector_activity/connector_activity.yaml b/tests/tasks/uipath-maestro-case/connector_features/connector_activity/connector_activity.yaml index 346e45c453..dac23e2446 100644 --- a/tests/tasks/uipath-maestro-case/connector_features/connector_activity/connector_activity.yaml +++ b/tests/tasks/uipath-maestro-case/connector_features/connector_activity/connector_activity.yaml @@ -119,11 +119,11 @@ initial_prompt: | success_criteria: - type: skill_triggered - description: "Agent invoked the uipath-maestro-case skill" + description: "Advisory: agent invoked the uipath-maestro-case skill" skill_name: "uipath-maestro-case" expected_skill: "uipath-maestro-case" weight: 1.0 - pass_threshold: 1.0 + pass_threshold: 0.0 - type: command_executed description: "Agent pulled the case registry before resolving the connector" diff --git a/tests/tasks/uipath-maestro-case/connector_features/connector_wait/check_connector_wait.py b/tests/tasks/uipath-maestro-case/connector_features/connector_wait/check_connector_wait.py index a8ecfe9181..b994f177f1 100644 --- a/tests/tasks/uipath-maestro-case/connector_features/connector_wait/check_connector_wait.py +++ b/tests/tasks/uipath-maestro-case/connector_features/connector_wait/check_connector_wait.py @@ -18,6 +18,7 @@ from _shared.case_check import ( # noqa: E402 assert_task_type_present, first_rule_of_condition, + is_non_required, iter_tasks, read_caseplan, task_is_skeleton, @@ -94,7 +95,7 @@ def main(): f"FAIL: entry rule {slot}[{entry.get('name')!r}].elementId " f"must be {expected_element_id!r}; got {entry.get('elementId')!r}" ) - if event_task.get("isRequired") is not False: + if not is_non_required(event_task): sys.exit( "FAIL: event-triggered placeholder process should stay non-required; " f"got isRequired={event_task.get('isRequired')!r}" diff --git a/tests/tasks/uipath-maestro-case/connector_features/connector_wait/connector_wait.yaml b/tests/tasks/uipath-maestro-case/connector_features/connector_wait/connector_wait.yaml index 3d13672bd7..96da11fcc7 100644 --- a/tests/tasks/uipath-maestro-case/connector_features/connector_wait/connector_wait.yaml +++ b/tests/tasks/uipath-maestro-case/connector_features/connector_wait/connector_wait.yaml @@ -152,11 +152,11 @@ initial_prompt: | success_criteria: - type: skill_triggered - description: "Agent invoked the uipath-maestro-case skill" + description: "Advisory: agent invoked the uipath-maestro-case skill" skill_name: "uipath-maestro-case" expected_skill: "uipath-maestro-case" weight: 1.0 - pass_threshold: 1.0 + pass_threshold: 0.0 - type: command_executed description: "Agent fetched the connector-trigger spec to resolve typeId/connectionId" diff --git a/tests/tasks/uipath-maestro-case/diagnose/extract_alias_collision/extract_alias_collision.yaml b/tests/tasks/uipath-maestro-case/diagnose/extract_alias_collision/extract_alias_collision.yaml index c161895a34..59503b8994 100644 --- a/tests/tasks/uipath-maestro-case/diagnose/extract_alias_collision/extract_alias_collision.yaml +++ b/tests/tasks/uipath-maestro-case/diagnose/extract_alias_collision/extract_alias_collision.yaml @@ -54,11 +54,11 @@ initial_prompt: | success_criteria: - type: skill_triggered - description: "Agent invoked the uipath-maestro-case skill" + description: "Advisory: agent invoked the uipath-maestro-case skill" skill_name: "uipath-maestro-case" expected_skill: "uipath-maestro-case" weight: 1.0 - pass_threshold: 1.0 + pass_threshold: 0.0 - type: run_command description: "uip maestro case validate still passes after the repair" diff --git a/tests/tasks/uipath-maestro-case/diagnose/fix_broken_caseplan/check_fix_broken_caseplan.py b/tests/tasks/uipath-maestro-case/diagnose/fix_broken_caseplan/check_fix_broken_caseplan.py index d39f51e004..9e9d8c3e41 100755 --- a/tests/tasks/uipath-maestro-case/diagnose/fix_broken_caseplan/check_fix_broken_caseplan.py +++ b/tests/tasks/uipath-maestro-case/diagnose/fix_broken_caseplan/check_fix_broken_caseplan.py @@ -43,6 +43,7 @@ find_triggers, get_case_exit_conditions, iter_tasks, + selected_stage_ids, read_caseplan, ) @@ -147,12 +148,12 @@ def main(): for cond in (stage.get("data") or {}).get("entryConditions") or []: for group in cond.get("rules") or []: for rule in group or []: - sid = (rule or {}).get("selectedStageId") - if sid and sid not in node_ids: - _fail( - f"stage {_label(stage)!r} entry condition references " - f"non-existent stage id {sid!r}" - ) + for sid in selected_stage_ids(rule or {}): + if sid not in node_ids: + _fail( + f"stage {_label(stage)!r} entry condition references " + f"non-existent stage id {sid!r}" + ) if not _has_path(plan, primary["Intake"]["id"], primary["Decision"]["id"]): _fail( "Decision is not reachable from Intake through condition-derived " diff --git a/tests/tasks/uipath-maestro-case/diagnose/fix_broken_caseplan/fix_broken_caseplan.yaml b/tests/tasks/uipath-maestro-case/diagnose/fix_broken_caseplan/fix_broken_caseplan.yaml index 344c5d39f1..b76e8bebe2 100644 --- a/tests/tasks/uipath-maestro-case/diagnose/fix_broken_caseplan/fix_broken_caseplan.yaml +++ b/tests/tasks/uipath-maestro-case/diagnose/fix_broken_caseplan/fix_broken_caseplan.yaml @@ -48,11 +48,11 @@ initial_prompt: | success_criteria: - type: skill_triggered - description: "Agent invoked the uipath-maestro-case skill" + description: "Advisory: agent invoked the uipath-maestro-case skill" skill_name: "uipath-maestro-case" expected_skill: "uipath-maestro-case" weight: 1.0 - pass_threshold: 1.0 + pass_threshold: 0.0 - type: command_executed description: "Agent ran the validate → fix loop (validate at least twice: read errors, then confirm the fix)" diff --git a/tests/tasks/uipath-maestro-case/diagnose/formal_arg_slot_ids/formal_arg_slot_ids.yaml b/tests/tasks/uipath-maestro-case/diagnose/formal_arg_slot_ids/formal_arg_slot_ids.yaml index 85b08c809f..cc4e179e5b 100644 --- a/tests/tasks/uipath-maestro-case/diagnose/formal_arg_slot_ids/formal_arg_slot_ids.yaml +++ b/tests/tasks/uipath-maestro-case/diagnose/formal_arg_slot_ids/formal_arg_slot_ids.yaml @@ -40,11 +40,11 @@ initial_prompt: | success_criteria: - type: skill_triggered - description: "Agent invoked the uipath-maestro-case skill" + description: "Advisory: agent invoked the uipath-maestro-case skill" skill_name: "uipath-maestro-case" expected_skill: "uipath-maestro-case" weight: 1.0 - pass_threshold: 1.0 + pass_threshold: 0.0 - type: run_command description: "uip maestro case validate still passes after the repair" diff --git a/tests/tasks/uipath-maestro-case/diagnose/missing_task_entry_rule/missing_task_entry_rule.yaml b/tests/tasks/uipath-maestro-case/diagnose/missing_task_entry_rule/missing_task_entry_rule.yaml index 4c7db878ce..d5ba6ba00c 100644 --- a/tests/tasks/uipath-maestro-case/diagnose/missing_task_entry_rule/missing_task_entry_rule.yaml +++ b/tests/tasks/uipath-maestro-case/diagnose/missing_task_entry_rule/missing_task_entry_rule.yaml @@ -38,11 +38,11 @@ initial_prompt: | success_criteria: - type: skill_triggered - description: "Agent invoked the uipath-maestro-case skill" + description: "Advisory: agent invoked the uipath-maestro-case skill" skill_name: "uipath-maestro-case" expected_skill: "uipath-maestro-case" weight: 1.0 - pass_threshold: 1.0 + pass_threshold: 0.0 - type: run_command description: "uip maestro case validate still passes after the repair" diff --git a/tests/tasks/uipath-maestro-case/diagnose/undeclared_gate_vars/undeclared_gate_vars.yaml b/tests/tasks/uipath-maestro-case/diagnose/undeclared_gate_vars/undeclared_gate_vars.yaml index efadbf586a..5802f83c05 100644 --- a/tests/tasks/uipath-maestro-case/diagnose/undeclared_gate_vars/undeclared_gate_vars.yaml +++ b/tests/tasks/uipath-maestro-case/diagnose/undeclared_gate_vars/undeclared_gate_vars.yaml @@ -39,11 +39,11 @@ initial_prompt: | success_criteria: - type: skill_triggered - description: "Agent invoked the uipath-maestro-case skill" + description: "Advisory: agent invoked the uipath-maestro-case skill" skill_name: "uipath-maestro-case" expected_skill: "uipath-maestro-case" weight: 1.0 - pass_threshold: 1.0 + pass_threshold: 0.0 - type: run_command description: "uip maestro case validate passes on the repaired caseplan.json" diff --git a/tests/tasks/uipath-maestro-case/e2e_expense_runnable/check_expense_runnable_structure.py b/tests/tasks/uipath-maestro-case/e2e_expense_runnable/check_expense_runnable_structure.py index 3eae587737..b68e6585f4 100644 --- a/tests/tasks/uipath-maestro-case/e2e_expense_runnable/check_expense_runnable_structure.py +++ b/tests/tasks/uipath-maestro-case/e2e_expense_runnable/check_expense_runnable_structure.py @@ -35,6 +35,7 @@ get_case_exit_conditions, iter_tasks, read_caseplan, + selected_stage_ids, ) EXPECTED_CASEPLAN = os.path.join( @@ -218,8 +219,8 @@ def main(): rname = rule.get("rule") if rname == "required-stages-completed" and ce.get("marksCaseComplete") is True: happy = True - if rname in ("selected-stage-completed", "selected-stage-exited") and rule.get("selectedStageId") in terminal_ids: - terminal_seen.add(rule.get("selectedStageId")) + if rname in ("selected-stage-completed", "selected-stage-exited"): + terminal_seen.update(terminal_ids & set(selected_stage_ids(rule))) if not happy: _fail("missing happy-path case-exit 'required-stages-completed' with marksCaseComplete=true") missing_term = [name for name, node in terminal.items() if node["id"] not in terminal_seen] diff --git a/tests/tasks/uipath-maestro-case/e2e_expense_runnable/test_check_expense_runnable_structure.py b/tests/tasks/uipath-maestro-case/e2e_expense_runnable/test_check_expense_runnable_structure.py index 643a777eeb..758258c559 100644 --- a/tests/tasks/uipath-maestro-case/e2e_expense_runnable/test_check_expense_runnable_structure.py +++ b/tests/tasks/uipath-maestro-case/e2e_expense_runnable/test_check_expense_runnable_structure.py @@ -13,6 +13,7 @@ _assert_bindings_v2_metadata, _assert_required_external_bindings, ) +from _shared.case_check import selected_stage_ids, stage_transitions # noqa: E402 class BindingsV2MetadataTests(unittest.TestCase): @@ -78,6 +79,52 @@ def test_turn_timeout_covers_the_full_e2e_task_budget(self) -> None: self.assertEqual(limits["turn_timeout"], limits["task_timeout"]) +class StageTransitionTests(unittest.TestCase): + def test_selected_stage_ids_prefers_v30_plural_and_accepts_legacy_singular(self) -> None: + self.assertEqual( + selected_stage_ids({"selectedStageIds": ["first", "second"]}), + ["first", "second"], + ) + self.assertEqual(selected_stage_ids({"selectedStageId": "legacy"}), ["legacy"]) + + def test_reads_v30_plural_and_legacy_singular_stage_ids(self) -> None: + plan = { + "nodes": [ + { + "id": "target", + "type": "case-management:Stage", + "data": { + "entryConditions": [ + { + "rules": [ + [ + { + "rule": "selected-stage-completed", + "selectedStageIds": ["first", "second"], + }, + { + "rule": "selected-stage-exited", + "selectedStageId": "legacy", + }, + ] + ] + } + ] + }, + } + ] + } + + self.assertEqual( + stage_transitions(plan), + [ + {"source": "first", "target": "target"}, + {"source": "legacy", "target": "target"}, + {"source": "second", "target": "target"}, + ], + ) + + def _valid_resources() -> list[dict]: """bindings_v2.json resources whose deployed names all match the tenant.""" return [ diff --git a/tests/tasks/uipath-maestro-case/e2e_outcome_external/outcome_external_e2e.yaml b/tests/tasks/uipath-maestro-case/e2e_outcome_external/outcome_external_e2e.yaml index af697daa78..f27d2f8e75 100644 --- a/tests/tasks/uipath-maestro-case/e2e_outcome_external/outcome_external_e2e.yaml +++ b/tests/tasks/uipath-maestro-case/e2e_outcome_external/outcome_external_e2e.yaml @@ -104,11 +104,11 @@ initial_prompt: | success_criteria: - type: skill_triggered - description: "Agent invoked the uipath-maestro-case skill" + description: "Advisory: agent invoked the uipath-maestro-case skill" skill_name: "uipath-maestro-case" expected_skill: "uipath-maestro-case" weight: 1.0 - pass_threshold: 1.0 + pass_threshold: 0.0 # Advisory: convention adherence, not the objective. "Not if it validated" — a case # that validates while notifying nobody has not achieved anything. diff --git a/tests/tasks/uipath-maestro-case/edit/bound_task_add_remove/bound_task_add_remove.yaml b/tests/tasks/uipath-maestro-case/edit/bound_task_add_remove/bound_task_add_remove.yaml index 6d9ef380b8..504c04155f 100644 --- a/tests/tasks/uipath-maestro-case/edit/bound_task_add_remove/bound_task_add_remove.yaml +++ b/tests/tasks/uipath-maestro-case/edit/bound_task_add_remove/bound_task_add_remove.yaml @@ -46,11 +46,11 @@ initial_prompt: | success_criteria: - type: skill_triggered - description: "Agent invoked the uipath-maestro-case skill" + description: "Advisory: agent invoked the uipath-maestro-case skill" skill_name: "uipath-maestro-case" expected_skill: "uipath-maestro-case" weight: 1.0 - pass_threshold: 1.0 + pass_threshold: 0.0 - type: run_command description: "uip maestro case validate passes on the edited caseplan.json" diff --git a/tests/tasks/uipath-maestro-case/edit/remove_stage/check_remove_stage.py b/tests/tasks/uipath-maestro-case/edit/remove_stage/check_remove_stage.py index eb71dbd93a..725a4296d1 100644 --- a/tests/tasks/uipath-maestro-case/edit/remove_stage/check_remove_stage.py +++ b/tests/tasks/uipath-maestro-case/edit/remove_stage/check_remove_stage.py @@ -25,6 +25,7 @@ iter_stage_exit_conditions, iter_tasks, read_caseplan, + selected_stage_ids, start_debug, ) @@ -47,9 +48,9 @@ def _dangling_stage_refs(plan): for cond in iter_stage_entry_conditions(node): for group in cond.get("rules") or []: for rule in group or []: - sid = (rule or {}).get("selectedStageId") - if sid and sid not in node_ids: - dangling.append((label, sid)) + for sid in selected_stage_ids(rule or {}): + if sid not in node_ids: + dangling.append((label, sid)) for cond in iter_stage_exit_conditions(node): dst = cond.get("exitToStageId") if dst and dst not in node_ids: diff --git a/tests/tasks/uipath-maestro-case/golden_rebuild/cm_golden_expense/check_cm_golden_case.py b/tests/tasks/uipath-maestro-case/golden_rebuild/cm_golden_expense/check_cm_golden_case.py index 552ab4cfc8..8c96849c8d 100644 --- a/tests/tasks/uipath-maestro-case/golden_rebuild/cm_golden_expense/check_cm_golden_case.py +++ b/tests/tasks/uipath-maestro-case/golden_rebuild/cm_golden_expense/check_cm_golden_case.py @@ -36,6 +36,7 @@ get_sla_rules, iter_tasks, read_caseplan, + selected_stage_ids, ) EXPECTED_CASEPLAN = os.path.join("CMGoldenExpense", "CMGoldenExpense", "caseplan.json") @@ -186,7 +187,7 @@ def main(): happy = True if ( name in ("selected-stage-completed", "selected-stage-exited") - and rule.get("selectedStageId") == stage4_id + and stage4_id in selected_stage_ids(rule) and case_exit.get("marksCaseComplete") is not True ): expression = rule.get("conditionExpression") diff --git a/tests/tasks/uipath-maestro-case/golden_rebuild/cm_golden_expense/check_cm_golden_semantics.py b/tests/tasks/uipath-maestro-case/golden_rebuild/cm_golden_expense/check_cm_golden_semantics.py index 17db535074..fe9d387210 100644 --- a/tests/tasks/uipath-maestro-case/golden_rebuild/cm_golden_expense/check_cm_golden_semantics.py +++ b/tests/tasks/uipath-maestro-case/golden_rebuild/cm_golden_expense/check_cm_golden_semantics.py @@ -26,7 +26,7 @@ sys.path.insert( 0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) ) -from _shared.case_check import find_stages, read_caseplan # noqa: E402 +from _shared.case_check import find_stages, read_caseplan, selected_stage_ids # noqa: E402 EXPECTED_CASEPLAN = os.path.join("CMGoldenExpense", "CMGoldenExpense", "caseplan.json") FIXTURE_SDD = os.path.join(os.path.dirname(os.path.abspath(__file__)), "fixtures", "sdd.md") @@ -370,9 +370,7 @@ def _selected_tasks(rule: dict, task_ids: dict) -> tuple[tuple[str, str], ...]: def _selected_stages(rule: dict, stage_ids: dict) -> tuple[str, ...]: - ids = list(rule.get("selectedStagesIds") or []) - if rule.get("selectedStageId"): - ids.append(rule["selectedStageId"]) + ids = selected_stage_ids(rule) logical = [] for stage_id in ids: if stage_id not in stage_ids: diff --git a/tests/tasks/uipath-maestro-case/hitl/check_boolean_routing.py b/tests/tasks/uipath-maestro-case/hitl/check_boolean_routing.py new file mode 100644 index 0000000000..bd98c0b162 --- /dev/null +++ b/tests/tasks/uipath-maestro-case/hitl/check_boolean_routing.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +"""Check both semantic boolean branches without pinning authoring-loop spelling.""" + +from __future__ import annotations + +import json +import re +import sys +from pathlib import Path +from typing import Any + + +BOOLEAN_BRANCH = re.compile( + r"^(?:=js:)?(?:vars\.)?approved={2,3}(true|false)$", + re.IGNORECASE, +) + + +def branch_values(value: Any) -> set[bool]: + found: set[bool] = set() + if isinstance(value, dict): + expression = value.get("conditionExpression") + if isinstance(expression, str): + match = BOOLEAN_BRANCH.fullmatch(re.sub(r"\s+", "", expression)) + if match: + found.add(match.group(1).lower() == "true") + for child in value.values(): + found.update(branch_values(child)) + elif isinstance(value, list): + for child in value: + found.update(branch_values(child)) + return found + + +def main() -> None: + if len(sys.argv) != 2: + sys.exit("usage: check_boolean_routing.py ") + path = Path(sys.argv[1]) + plan = json.loads(path.read_text(encoding="utf-8")) + found = branch_values(plan) + missing = {True, False} - found + if missing: + sys.exit(f"FAIL: missing approved boolean branch(es): {sorted(missing)}") + print(f"OK: {path} contains approved true and false branches") + + +if __name__ == "__main__": + main() diff --git a/tests/tasks/uipath-maestro-case/hitl/quality_01_action_schema_design.yaml b/tests/tasks/uipath-maestro-case/hitl/quality_01_action_schema_design.yaml index 234e43a2eb..520869cc39 100644 --- a/tests/tasks/uipath-maestro-case/hitl/quality_01_action_schema_design.yaml +++ b/tests/tasks/uipath-maestro-case/hitl/quality_01_action_schema_design.yaml @@ -3,8 +3,8 @@ description: > Quality test: agent maps a business description to a correctly-shaped `action` task — taskTitle, priority, recipient, and the planned input/ output fields. Verifies that even when the action-app is unresolved - (skeleton path), the agent still records the planning fields in the - resolution audit and writes valid JSON. Tests action-task field design (priority enum, + (skeleton path), the agent records the intended contract in report.json + and writes valid JSON. Tests action-task field design (priority enum, recipient handling, taskTitle requirement). tags: [uipath-maestro-case, integration, mode:build, lifecycle:generate, shape:multi-node, node:hitl] initial_prompt: | @@ -181,13 +181,11 @@ initial_prompt: | Important: - The `uip` CLI is already available. - - Treat the generated plan as pre-approved — do not block on any HARD STOP + - Treat any generated plan as pre-approved — do not block on any HARD STOP / AskUserQuestion approval step. - If the action-app or the process resource is unresolved, write a - skeleton task per the skeleton-tasks reference. That task's - `tasks/registry-resolved.json` entry must still capture taskTitle, - priority, recipient, and the intended inputs/outputs in a fenced - wiring-notes block. + skeleton task per the skeleton-tasks reference. `report.json` must still + capture taskTitle, priority, recipient, and the intended inputs/outputs. - Do NOT run `uip maestro case debug`. **REQUIRED — before finishing, write report.json at the sandbox root**: @@ -224,32 +222,8 @@ success_criteria: weight: 2.0 pass_threshold: 1.0 - - type: run_command - description: "the task-title is captured (caseplan on the resolved path, registry audit on the skeleton path)" - command: "grep -ri 'task-title\\|taskTitle\\|Task Title' tasks/registry-resolved.json VendorPOReviewCase/VendorPOReviewCase/caseplan.json" - timeout: 10 - expected_exit_code: 0 - weight: 2.0 - pass_threshold: 1.0 - - - type: run_command - description: "priority: High is captured (caseplan or registry audit)" - command: "grep -ri 'priority.*high' tasks/registry-resolved.json VendorPOReviewCase/VendorPOReviewCase/caseplan.json" - timeout: 10 - expected_exit_code: 0 - weight: 1.5 - pass_threshold: 1.0 - - - type: run_command - description: "the recipient email is captured (caseplan or registry audit)" - command: "grep -ri 'po-approvals@corp.com' tasks/registry-resolved.json VendorPOReviewCase/VendorPOReviewCase/caseplan.json" - timeout: 10 - expected_exit_code: 0 - weight: 1.5 - pass_threshold: 1.0 - - type: json_check - description: "report.json schema has at least 3 input fields, 1 output field, priority High" + description: "report.json preserves the HITL task contract" path: "report.json" assertions: - expression: "validation_passed" @@ -264,8 +238,14 @@ success_criteria: - expression: "priority" operator: equals expected: "High" - weight: 2.5 - pass_threshold: 0.75 + - expression: "task_title" + operator: equals + expected: "Please review this PO and approve or reject" + - expression: "recipient" + operator: equals + expected: "po-approvals@corp.com" + weight: 7.5 + pass_threshold: 0.85 run_limits: expected_turns: 66 diff --git a/tests/tasks/uipath-maestro-case/hitl/quality_03_boolean_decision.yaml b/tests/tasks/uipath-maestro-case/hitl/quality_03_boolean_decision.yaml index 8fdb57cae4..f601942991 100644 --- a/tests/tasks/uipath-maestro-case/hitl/quality_03_boolean_decision.yaml +++ b/tests/tasks/uipath-maestro-case/hitl/quality_03_boolean_decision.yaml @@ -258,12 +258,11 @@ success_criteria: weight: 1.5 pass_threshold: 1.0 - - type: file_contains - description: "Both branch expressions reference the approved boolean" - path: "VendorApprovalCase/VendorApprovalCase/caseplan.json" - includes: - - "vars.approved === true" - - "vars.approved === false" + - type: run_command + description: "Both semantic boolean branches reference approved" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-case/hitl/check_boolean_routing.py VendorApprovalCase/VendorApprovalCase/caseplan.json" + timeout: 30 + expected_exit_code: 0 weight: 3.0 pass_threshold: 1.0 diff --git a/tests/tasks/uipath-maestro-case/hitl/smoke_03_multi_outcome_routing.yaml b/tests/tasks/uipath-maestro-case/hitl/smoke_03_multi_outcome_routing.yaml index 7da10b3f8f..a221006a24 100644 --- a/tests/tasks/uipath-maestro-case/hitl/smoke_03_multi_outcome_routing.yaml +++ b/tests/tasks/uipath-maestro-case/hitl/smoke_03_multi_outcome_routing.yaml @@ -195,12 +195,11 @@ success_criteria: weight: 2.5 pass_threshold: 1.0 - - type: file_contains - description: "Both branch expressions present on Review's exits" - path: "LeaveRequestCase/LeaveRequestCase/caseplan.json" - includes: - - "vars.approved === true" - - "vars.approved === false" + - type: run_command + description: "Both semantic boolean branches are present on Review's exits" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-case/hitl/check_boolean_routing.py LeaveRequestCase/LeaveRequestCase/caseplan.json" + timeout: 30 + expected_exit_code: 0 weight: 2.0 pass_threshold: 1.0 diff --git a/tests/tasks/uipath-maestro-case/hitl/test_check_boolean_routing.py b/tests/tasks/uipath-maestro-case/hitl/test_check_boolean_routing.py new file mode 100644 index 0000000000..c98cd97abc --- /dev/null +++ b/tests/tasks/uipath-maestro-case/hitl/test_check_boolean_routing.py @@ -0,0 +1,28 @@ +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from check_boolean_routing import branch_values # noqa: E402 + + +def test_accepts_equivalent_case_expression_spellings(): + plan = { + "conditions": [ + {"conditionExpression": "=js:vars.approved === true"}, + {"conditionExpression": "approved == false"}, + ] + } + + assert branch_values(plan) == {True, False} + + +def test_does_not_accept_unrelated_boolean_expressions(): + plan = { + "conditions": [ + {"conditionExpression": "=js:vars.other === true"}, + {"conditionExpression": "=js:vars.approved !== false"}, + ] + } + + assert branch_values(plan) == set() diff --git a/tests/tasks/uipath-maestro-case/logging/logging_issue_log.yaml b/tests/tasks/uipath-maestro-case/logging/logging_issue_log.yaml index 91dec2bafa..ad12278b8a 100644 --- a/tests/tasks/uipath-maestro-case/logging/logging_issue_log.yaml +++ b/tests/tasks/uipath-maestro-case/logging/logging_issue_log.yaml @@ -131,8 +131,9 @@ initial_prompt: | Run `uip maestro case validate TimerLogCase/TimerLogCase/caseplan.json --output json`. Validation is expected to fail due to the invalid exitType; that is not a reason to stop. The task is NOT complete until validate has run - and the build's issue log is on disk. Follow the skill's own logging contract - for when and how that log is written. + and `tasks/build-issues.md` is on disk. Keep that file as an append-only build + journal: include a heading naming TimerLogCase, a `## Journal` table with the + validation failure, and a completed summary of the recorded issue. success_criteria: - type: run_command diff --git a/tests/tasks/uipath-maestro-case/multi_stages_tasks/case_exit_full/check_case_exit_full.py b/tests/tasks/uipath-maestro-case/multi_stages_tasks/case_exit_full/check_case_exit_full.py index b96b9935b2..26709d71d6 100644 --- a/tests/tasks/uipath-maestro-case/multi_stages_tasks/case_exit_full/check_case_exit_full.py +++ b/tests/tasks/uipath-maestro-case/multi_stages_tasks/case_exit_full/check_case_exit_full.py @@ -13,9 +13,11 @@ find_stages, first_rule_of_condition, get_case_exit_conditions, + is_non_required, iter_stage_entry_conditions, iter_stage_exit_conditions, read_caseplan, + selected_stage_ids, start_debug, ) @@ -46,7 +48,8 @@ def main(): } for name, (stage, want) in expected_required.items(): got = _is_required(stage) - if got is not want: + matches = got is True if want else is_non_required(stage.get("data") or {}) + if not matches: sys.exit( f"FAIL: stage {name!r} isRequired should be {want}; got {got!r}" ) @@ -122,9 +125,7 @@ def main(): elif marks is False: non_completing_rules.add(rname) if rname in ("selected-stage-exited", "selected-stage-completed"): - sid = rule.get("selectedStageId") - if sid: - selected_stage_by_rule.setdefault(rname, set()).add(sid) + selected_stage_by_rule.setdefault(rname, set()).update(selected_stage_ids(rule)) if "required-stages-completed" not in completing_rules: sys.exit( diff --git a/tests/tasks/uipath-maestro-case/multi_stages_tasks/exception_stage/check_exception_stage.py b/tests/tasks/uipath-maestro-case/multi_stages_tasks/exception_stage/check_exception_stage.py index 5625644840..2c52fb98b5 100644 --- a/tests/tasks/uipath-maestro-case/multi_stages_tasks/exception_stage/check_exception_stage.py +++ b/tests/tasks/uipath-maestro-case/multi_stages_tasks/exception_stage/check_exception_stage.py @@ -15,6 +15,7 @@ iter_stage_exit_conditions, partition_return_to_origin_conditions, read_caseplan, + selected_stage_ids, ) @@ -77,10 +78,10 @@ def _is_secondary(n): f"FAIL: 'Issues' interrupting rule should be 'selected-stage-exited'; " f"got {issues_rule and issues_rule.get('rule')!r}" ) - if issues_rule.get("selectedStageId") != process["id"]: + if process["id"] not in selected_stage_ids(issues_rule): sys.exit( f"FAIL: 'Issues' rule.selectedStageId should be Process id " - f"({process['id']}), got {issues_rule.get('selectedStageId')!r}" + f"({process['id']}), got {selected_stage_ids(issues_rule)!r}" ) critical_entry = list(iter_stage_entry_conditions(critical)) @@ -93,10 +94,10 @@ def _is_secondary(n): f"FAIL: 'Critical' interrupting rule should be 'selected-stage-exited'; " f"got {critical_rule and critical_rule.get('rule')!r}" ) - if critical_rule.get("selectedStageId") != process["id"]: + if process["id"] not in selected_stage_ids(critical_rule): sys.exit( f"FAIL: 'Critical' rule.selectedStageId should be Process id " - f"({process['id']}), got {critical_rule.get('selectedStageId')!r}" + f"({process['id']}), got {selected_stage_ids(critical_rule)!r}" ) for label, node in (("Issues", issues), ("Critical", critical)): @@ -147,7 +148,7 @@ def _is_secondary(n): critical_case_exits = [] for cond in get_case_exit_conditions(plan): rule = first_rule_of_condition(cond) or {} - if rule.get("selectedStageId") == critical["id"]: + if critical["id"] in selected_stage_ids(rule): critical_case_exits.append(cond) if not critical_case_exits: sys.exit("FAIL: terminal secondary stage 'Critical' needs a root case-exit row") diff --git a/tests/tasks/uipath-maestro-case/multi_stages_tasks/fan_in_join/check_fan_in_join.py b/tests/tasks/uipath-maestro-case/multi_stages_tasks/fan_in_join/check_fan_in_join.py index e011b6c227..2afc5fb320 100644 --- a/tests/tasks/uipath-maestro-case/multi_stages_tasks/fan_in_join/check_fan_in_join.py +++ b/tests/tasks/uipath-maestro-case/multi_stages_tasks/fan_in_join/check_fan_in_join.py @@ -15,6 +15,7 @@ first_rule_of_condition, iter_stage_entry_conditions, read_caseplan, + selected_stage_ids, start_debug, task_is_skeleton, ) @@ -71,9 +72,7 @@ def main(): sys.exit( f"FAIL: Join entry rule must be 'selected-stage-completed'; got {rule.get('rule')!r}" ) - sid = rule.get("selectedStageId") - if sid: - referenced_stage_ids.add(sid) + referenced_stage_ids.update(selected_stage_ids(rule)) if not {validate["id"], enrich["id"]}.issubset(referenced_stage_ids): sys.exit( diff --git a/tests/tasks/uipath-maestro-case/multi_stages_tasks/linear_three_stages/check_linear_three_stages.py b/tests/tasks/uipath-maestro-case/multi_stages_tasks/linear_three_stages/check_linear_three_stages.py index 4f86856f06..1ee3634bcb 100644 --- a/tests/tasks/uipath-maestro-case/multi_stages_tasks/linear_three_stages/check_linear_three_stages.py +++ b/tests/tasks/uipath-maestro-case/multi_stages_tasks/linear_three_stages/check_linear_three_stages.py @@ -16,6 +16,7 @@ get_variables, iter_stage_entry_conditions, iter_tasks, + is_non_required, read_caseplan, start_debug, ) @@ -110,7 +111,7 @@ def _by_label(label: str) -> dict | None: f"FAIL: 'Hold For 1 Hour' skipCondition should reference skipReview; " f"got {skip!r}" ) - if notify.get("isRequired") is not False: + if not is_non_required(notify): sys.exit( f"FAIL: 'Notify Reviewer' should have isRequired=false (task-level " f"flag explicitly set); got {notify.get('isRequired')!r}" diff --git a/tests/tasks/uipath-maestro-case/multi_stages_tasks/task_dependency_chain/check_task_dependency_chain.py b/tests/tasks/uipath-maestro-case/multi_stages_tasks/task_dependency_chain/check_task_dependency_chain.py index 0653a3a5e1..66a7014626 100644 --- a/tests/tasks/uipath-maestro-case/multi_stages_tasks/task_dependency_chain/check_task_dependency_chain.py +++ b/tests/tasks/uipath-maestro-case/multi_stages_tasks/task_dependency_chain/check_task_dependency_chain.py @@ -14,6 +14,7 @@ iter_stage_entry_conditions, iter_stage_exit_conditions, read_caseplan, + selected_stage_ids, start_debug, task_is_skeleton, ) @@ -60,6 +61,12 @@ def _task_entry_rule(task: dict) -> str | None: return None +def _task_is_non_required(task: dict) -> bool: + # The Case SDK omits isRequired for the default non-required state. + value = task.get("isRequired") + return value is None or value is False + + def main(): plan = read_caseplan() @@ -115,10 +122,12 @@ def main(): f"FAIL: Done should have entry rule 'selected-stage-completed'; " f"got {sorted(r.get('rule') for r in done_entry_rules if r.get('rule'))}" ) - if finalize["id"] not in {r.get("selectedStageId") for r in done_entry_rules}: + if finalize["id"] not in { + stage_id for rule in done_entry_rules for stage_id in selected_stage_ids(rule) + }: sys.exit( f"FAIL: Done selected-stage-completed entry must reference Finalize id " - f"{finalize['id']!r}; got {[r.get('selectedStageId') for r in done_entry_rules]}" + f"{finalize['id']!r}; got {[selected_stage_ids(r) for r in done_entry_rules]}" ) first_step = _stage_task_by_label(process, "First Step") @@ -155,7 +164,7 @@ def main(): (first_rule_of_condition(condition) or {}).get("rule") for condition in (optional_audit.get("entryConditions") or []) ] - if adhoc_rules != ["adhoc"] or optional_audit.get("isRequired") is not False: + if adhoc_rules != ["adhoc"] or not _task_is_non_required(optional_audit): sys.exit( "FAIL: Optional Audit must be an adhoc-only, non-required task; " f"rules={adhoc_rules!r}, isRequired={optional_audit.get('isRequired')!r}" diff --git a/tests/tasks/uipath-maestro-case/multi_stages_tasks/task_dependency_chain/test_check_task_dependency_chain.py b/tests/tasks/uipath-maestro-case/multi_stages_tasks/task_dependency_chain/test_check_task_dependency_chain.py new file mode 100644 index 0000000000..8df39d3c90 --- /dev/null +++ b/tests/tasks/uipath-maestro-case/multi_stages_tasks/task_dependency_chain/test_check_task_dependency_chain.py @@ -0,0 +1,16 @@ +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from check_task_dependency_chain import _task_is_non_required # noqa: E402 + + +def test_non_required_accepts_the_sdk_default_or_explicit_false(): + assert _task_is_non_required({}) + assert _task_is_non_required({"isRequired": False}) + + +def test_non_required_rejects_required_or_invalid_values(): + assert not _task_is_non_required({"isRequired": True}) + assert not _task_is_non_required({"isRequired": "false"}) diff --git a/tests/tasks/uipath-maestro-case/publish/publish_upload/publish_upload.yaml b/tests/tasks/uipath-maestro-case/publish/publish_upload/publish_upload.yaml index d69cbda021..aa439926e6 100644 --- a/tests/tasks/uipath-maestro-case/publish/publish_upload/publish_upload.yaml +++ b/tests/tasks/uipath-maestro-case/publish/publish_upload/publish_upload.yaml @@ -77,11 +77,11 @@ initial_prompt: | success_criteria: - type: skill_triggered - description: "Agent invoked the uipath-maestro-case skill" + description: "Advisory: agent invoked the uipath-maestro-case skill" skill_name: "uipath-maestro-case" expected_skill: "uipath-maestro-case" weight: 1.0 - pass_threshold: 1.0 + pass_threshold: 0.0 - type: run_command description: "uip maestro case validate passes on the generated caseplan.json" diff --git a/tests/tasks/uipath-maestro-case/registry_handoff_sdd_only/check_registry_handoff.py b/tests/tasks/uipath-maestro-case/registry_handoff_sdd_only/check_registry_handoff.py index 7d68fd5e64..8461f828be 100644 --- a/tests/tasks/uipath-maestro-case/registry_handoff_sdd_only/check_registry_handoff.py +++ b/tests/tasks/uipath-maestro-case/registry_handoff_sdd_only/check_registry_handoff.py @@ -1,6 +1,11 @@ import json +import os +import sys from pathlib import Path +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +from _shared.case_check import registry_audit_entries # noqa: E402 + EXPECTED = { "FinancialPostingFunction": { @@ -15,16 +20,6 @@ }, } STAGE = "Resolve Resources" -REQUIRED_KEYS = { - "stage", - "task", - "taskType", - "cacheFile", - "searchQuery", - "matches", - "selected", - "rationale", -} def load_json(path: Path): @@ -34,7 +29,7 @@ def load_json(path: Path): registry_path = Path("tasks/registry-resolved.json") entries = load_json(registry_path) -assert isinstance(entries, list), "registry-resolved.json must be a list" +entries = registry_audit_entries(entries) assert len(entries) == len(EXPECTED), ( f"expected one fresh audit entry per SDD task, got {len(entries)}" ) @@ -45,34 +40,36 @@ def load_json(path: Path): matching_entries = [ entry for entry in entries - if str(entry.get("searchQuery", "")).strip() == name + if str(entry.get("searchQuery") or entry.get("resolvedResource") or "").strip() + == name ] assert len(matching_entries) == 1, ( f"expected one registry audit entry for {name}, got {len(matching_entries)}" ) entry = matching_entries[0] - assert REQUIRED_KEYS <= entry.keys(), ( - f"{name} audit entry is missing {sorted(REQUIRED_KEYS - entry.keys())}" - ) - assert str(entry.get("rationale", "")).strip(), ( - f"{name} audit entry has no selection rationale" - ) - assert entry.get("stage") == STAGE, ( - f"{name} audit entry is associated with stage {entry.get('stage')!r}" - ) - assert entry.get("task") == expected["task"], ( - f"{name} audit entry is associated with task {entry.get('task')!r}" + if "stage" in entry: + assert entry["stage"] == STAGE, ( + f"{name} audit entry is associated with stage {entry['stage']!r}" + ) + task_name = entry.get("task") or entry.get("taskName") + assert task_name == expected["task"], ( + f"{name} audit entry is associated with task {task_name!r}" ) assert entry.get("taskType") == expected["task_type"], ( f"{name} used taskType {entry.get('taskType')!r}" ) - assert entry.get("cacheFile") == expected["cache_file"], ( - f"{name} did not record cacheFile {expected['cache_file']}" - ) + if "cacheFile" in entry: + assert entry["cacheFile"] == expected["cache_file"], ( + f"{name} did not record cacheFile {expected['cache_file']}" + ) + assert str(entry.get("rationale") or "").strip() or isinstance( + entry.get("resolution"), dict + ), f"{name} audit entry has no resolution evidence" - selected = entry.get("selected") + selected = entry.get("selected") or entry.get("resourceIdentity") assert isinstance(selected, dict), f"missing selected result for {name}" - assert selected.get("name") == name, f"selected the wrong resource for {name}" + selected_name = selected.get("name") or entry.get("resolvedResource") + assert selected_name == name, f"selected the wrong resource for {name}" cached_resources = load_json(cache_root / expected["cache_file"]) exact_matches = [ @@ -89,18 +86,18 @@ def load_json(path: Path): # Also tolerant of ephemeral debug-deploy churn: we require the SELECTED entry to be a # live exact-name match and every recorded candidate to be exact-name, not a byte-for- # byte copy of the whole cache set. - recorded = entry.get("matches") or [] - assert recorded, f"{name} audit recorded no matches" - # The skill's discovery search is substring-based, so `matches` may carry near-name - # candidates (e.g. a `V2` sibling) alongside the exact hit. Correctness is proven - # by `selected` below (exact-name + live entityKey); here we only require the exact-name - # match to be PRESENT among the recorded candidates, not that every candidate is exact. - assert any( - str(m.get("name", "")).strip().casefold() == name.casefold() for m in recorded - ), f"{name} audit recorded no exact-name match" - assert isinstance(selected, dict), f"missing selected result for {name}" + recorded = entry.get("matches") + if recorded is not None: + assert recorded, f"{name} audit recorded no matches" + # The skill's discovery search is substring-based, so `matches` may carry + # near-name candidates alongside the exact hit. Correctness is proven by + # `selected`; here we only require the exact-name match to be present. + assert any( + str(m.get("name", "")).strip().casefold() == name.casefold() + for m in recorded + ), f"{name} audit recorded no exact-name match" assert ( - str(selected.get("name", "")).strip().casefold() == name.casefold() + str(selected_name).strip().casefold() == name.casefold() and str(selected.get("entityKey")) in cache_keys ), f"selected result for {name} is not a live exact-name cache entry" @@ -110,10 +107,15 @@ def load_json(path: Path): # flat (folder/folderType) or nested (folders[0].fullyQualifiedName/.type). fol = selected.get("folders") or [] sel_folder = str( - selected.get("folder") or (fol[0].get("fullyQualifiedName") if fol else "") + selected.get("folder") + or selected.get("folderPath") + or entry.get("folderPath") + or (fol[0].get("fullyQualifiedName") if fol else "") ).casefold() sel_ftype = str( - selected.get("folderType") or (fol[0].get("type") if fol else "") + selected.get("folderType") + or selected.get("organizationUnitType") + or (fol[0].get("type") if fol else "") ).casefold() assert sel_ftype != "debugsolution" and "debug_" not in sel_folder, ( f"{name} resolved to an ephemeral debug-solution deploy " diff --git a/tests/tasks/uipath-maestro-case/safety_guard/artifact_safety_guard.yaml b/tests/tasks/uipath-maestro-case/safety_guard/artifact_safety_guard.yaml index d26718c1b7..4625d6a8a2 100644 --- a/tests/tasks/uipath-maestro-case/safety_guard/artifact_safety_guard.yaml +++ b/tests/tasks/uipath-maestro-case/safety_guard/artifact_safety_guard.yaml @@ -188,9 +188,9 @@ success_criteria: # --- Rule 13 guards: no interpreter/stream-editor/helper-script assembly of a skill artifact --- # crypto.randomUUID() UUID subprocess (no fs, no redirection) is NOT matched by either pattern. - type: command_not_executed - description: "Rule 13: no inline interpreter or stream-editor manipulation of a skill artifact (sed -i, perl -i, gawk in-place, jq on/into an artifact, python open(), node -e ...fs. / require('fs'))" + description: "Rule 13: no inline interpreter or stream-editor writes to a skill artifact (read-only inspection is allowed)" tool_name: "Bash" - command_pattern: '(sed\s+-i|perl\s+-i|awk\b[^\n]*-i\s*inplace|jq\b[^\n]*([<>]|caseplan\.json|registry-resolved\.json|bindings_v2\.json|entry-points\.json)|python[0-9]?\b[^\n]*open\s*\(|(node|nodejs)\s+(-e|--eval)\b[^\n]*(fs\.|require\s*\(\s*[''"](node:)?fs[''"]))' + command_pattern: '(sed\s+-i|perl\s+-i|awk\b[^\n]*-i\s*inplace|python[0-9]?\b[^\n]*(open\s*\([^)]*,[^)]*[wax]|\.write(_text|_bytes)?\s*\()|(node|nodejs)\s+(-e|--eval)\b[^\n]*(fs\.|require\s*\(\s*[''"](node:)?fs[''"]\s*\)\s*\.)(write|append|truncate|rm|unlink|rename))' weight: 2.0 pass_threshold: 1.0 diff --git a/tests/tasks/uipath-maestro-case/sla_response/at_risk_enter_stage/check_at_risk_enter_stage.py b/tests/tasks/uipath-maestro-case/sla_response/at_risk_enter_stage/check_at_risk_enter_stage.py index eef4531a94..a5a36d0d4d 100644 --- a/tests/tasks/uipath-maestro-case/sla_response/at_risk_enter_stage/check_at_risk_enter_stage.py +++ b/tests/tasks/uipath-maestro-case/sla_response/at_risk_enter_stage/check_at_risk_enter_stage.py @@ -13,6 +13,7 @@ assert_stage_count, fail, iter_sla_status_change, + is_non_required, label_of, read_plan, secondary_stages, @@ -33,7 +34,7 @@ def main() -> None: lane = lanes[0] lane_data = lane["data"] - if lane_data.get("isRequired") is not False: + if not is_non_required(lane_data): fail( f"lane {label_of(lane)!r} has isRequired={lane_data.get('isRequired')!r}; an SLA lane " "must stay out of the happy-path required-stages-completed set" diff --git a/tests/tasks/uipath-maestro-case/sla_response/case_breach_oversight/check_case_breach_oversight.py b/tests/tasks/uipath-maestro-case/sla_response/case_breach_oversight/check_case_breach_oversight.py index c94f5dd9ef..0ecd0ddc8e 100644 --- a/tests/tasks/uipath-maestro-case/sla_response/case_breach_oversight/check_case_breach_oversight.py +++ b/tests/tasks/uipath-maestro-case/sla_response/case_breach_oversight/check_case_breach_oversight.py @@ -19,6 +19,7 @@ fail, is_secondary, iter_sla_status_change, + is_non_required, label_of, read_plan, ) @@ -46,7 +47,7 @@ def main() -> None: "non-interrupting SLA oversight lane still stays `secondary` — promoting it to a " "regular stage would make it required for case completion" ) - if lane["data"].get("isRequired") is not False: + if not is_non_required(lane["data"]): fail( f"lane {label_of(lane)!r} has isRequired={lane['data'].get('isRequired')!r}; an " "oversight lane must stay out of the required-stages-completed set" diff --git a/tests/tasks/uipath-maestro-case/sla_response/case_breach_takeover/check_case_breach_takeover.py b/tests/tasks/uipath-maestro-case/sla_response/case_breach_takeover/check_case_breach_takeover.py index 2ab719570f..6787afa7d0 100644 --- a/tests/tasks/uipath-maestro-case/sla_response/case_breach_takeover/check_case_breach_takeover.py +++ b/tests/tasks/uipath-maestro-case/sla_response/case_breach_takeover/check_case_breach_takeover.py @@ -13,6 +13,7 @@ assert_stage_count, fail, iter_sla_status_change, + is_non_required, label_of, read_plan, secondary_stages, @@ -31,7 +32,7 @@ def main() -> None: f"{[label_of(n) for n in lanes]}" ) lane = lanes[0] - if lane["data"].get("isRequired") is not False: + if not is_non_required(lane["data"]): fail(f"lane {label_of(lane)!r} must be isRequired False") hits = [(n, c, r) for n, c, r in iter_sla_status_change(plan) if n is lane] diff --git a/tests/tasks/uipath-maestro-case/triggers/event_placeholder/event_placeholder.yaml b/tests/tasks/uipath-maestro-case/triggers/event_placeholder/event_placeholder.yaml index c9e4729b21..4f6f2a26a2 100644 --- a/tests/tasks/uipath-maestro-case/triggers/event_placeholder/event_placeholder.yaml +++ b/tests/tasks/uipath-maestro-case/triggers/event_placeholder/event_placeholder.yaml @@ -102,11 +102,11 @@ initial_prompt: | success_criteria: - type: skill_triggered - description: "Agent invoked the uipath-maestro-case skill" + description: "Advisory: agent invoked the uipath-maestro-case skill" skill_name: "uipath-maestro-case" expected_skill: "uipath-maestro-case" weight: 1.0 - pass_threshold: 1.0 + pass_threshold: 0.0 - type: run_command description: "uip maestro case validate passes on the generated caseplan.json" diff --git a/tests/tasks/uipath-maestro-flow/_shared/check_inline_agent.py b/tests/tasks/uipath-maestro-flow/_shared/check_inline_agent.py index bfff5fc65c..c832bf5fb6 100644 --- a/tests/tasks/uipath-maestro-flow/_shared/check_inline_agent.py +++ b/tests/tasks/uipath-maestro-flow/_shared/check_inline_agent.py @@ -3,12 +3,13 @@ to a production bar, not left on the toy scaffold defaults. Usage (from a task's run_command, cwd = sandbox root): - python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/check_inline_agent.py + python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/check_inline_agent.py [--check ] [] Shell-style glob for the inline agent.json. The inline agent dir is a UUID, so the path is not statically knowable — pass e.g. "EmailTriage/EmailTriage/*/agent.json". Defaults to "**/agent.json" if omitted. + One of ``exists``, ``model``, or ``quality`` (the default). Asserts, on the first matching agent.json (excluding generated .agent-builder/): 1. settings.model is set and is NOT the stale scaffold default gpt-4o-2024-11-20. @@ -54,8 +55,23 @@ } +def _parse_args() -> tuple[str, str]: + args = sys.argv[1:] + scope = "quality" + if args[:1] == ["--check"]: + if len(args) < 2 or args[1] not in {"exists", "model", "quality"}: + print("FAIL: --check must be one of exists, model, or quality") + raise SystemExit(1) + scope = args[1] + args = args[2:] + if len(args) > 1: + print("FAIL: expected at most one agent.json glob") + raise SystemExit(1) + return scope, args[0] if args else "**/agent.json" + + def main() -> int: - pattern = sys.argv[1] if len(sys.argv) > 1 else "**/agent.json" + scope, pattern = _parse_args() paths = [ path for path in glob.glob(pattern, recursive=True) @@ -66,6 +82,10 @@ def main() -> int: return 1 path = min(paths) + if scope == "exists": + print(f"OK ({path}): inline agent sidecar exists") + return 0 + try: with open(path) as source: agent = json.load(source) @@ -81,6 +101,13 @@ def main() -> int: elif model == SCAFFOLD_MODEL: errs.append(f"settings.model not overridden ({model})") + if scope == "model": + if errs: + print(f"FAIL ({path}): " + "; ".join(errs)) + return 1 + print(f"OK ({path}): model={model}") + return 0 + sys_msgs = [m.get("content", "") for m in agent.get("messages", []) if m.get("role") == "system"] prompt = (sys_msgs[0] if sys_msgs else "").strip() if prompt.lower() in PLACEHOLDER_PROMPTS or len(prompt) < MIN_PROMPT_LEN: diff --git a/tests/tasks/uipath-maestro-flow/_shared/check_simulated_hitl.py b/tests/tasks/uipath-maestro-flow/_shared/check_simulated_hitl.py new file mode 100644 index 0000000000..63b43c4f9c --- /dev/null +++ b/tests/tasks/uipath-maestro-flow/_shared/check_simulated_hitl.py @@ -0,0 +1,171 @@ +#!/usr/bin/env python3 +"""Check source semantics for the simulated HITL tasks.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +from typing import Any + +from flow_check import find_flow_file + + +def fail(message: str) -> None: + raise SystemExit(f"FAIL: {message}") + + +def load_flow() -> tuple[dict[str, Any], list[dict[str, Any]]]: + path = Path(find_flow_file()) + try: + flow = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + fail(f"could not read {path}: {error}") + nodes = flow.get("nodes") + if not isinstance(nodes, list): + fail(f"{path} has no nodes array") + return flow, nodes + + +def hitl_nodes(nodes: list[dict[str, Any]]) -> list[dict[str, Any]]: + return [ + node + for node in nodes + if str(node.get("type", "")).startswith("uipath.human-in-the-loop") + ] + + +def check_quick_form(_flow: dict[str, Any], nodes: list[dict[str, Any]]) -> None: + candidates = [ + node + for node in hitl_nodes(nodes) + if node.get("type") == "uipath.human-in-the-loop.quick-form" + or ( + node.get("type") == "uipath.human-in-the-loop" + and (node.get("inputs") or {}).get("type") == "quick" + ) + ] + if not candidates: + fail( + "need an inline HITL quick form: either the quick-form node type or " + "the base HITL node with inputs.type='quick'" + ) + print("OK: flow contains an inline HITL quick form") + + +def field_text(field: dict[str, Any], key: str) -> str: + value = field.get(key) + return value.lower() if isinstance(value, str) else "" + + +def check_schema(flow: dict[str, Any], nodes: list[dict[str, Any]]) -> None: + candidates = hitl_nodes(nodes) + if not candidates: + fail("need a HITL quickform node") + schema = (candidates[0].get("inputs") or {}).get("schema", {}) + fields = schema.get("fields", []) + directions = {field.get("direction") for field in fields} + if "input" not in directions: + fail("need an input-direction (read-only) field") + if not directions.intersection({"output", "inOut"}): + fail("need an output/inOut (fill-in) field") + rendered = json.dumps(schema).lower() + if "approve" not in rendered or "reject" not in rendered: + fail("need Approve and Reject outcomes") + print("OK: HITL schema has input/output fields and Approve/Reject outcomes") + + +def check_priority(_flow: dict[str, Any], nodes: list[dict[str, Any]]) -> None: + candidates = hitl_nodes(nodes) + if not candidates: + fail("need a HITL quickform node") + if "high" not in json.dumps(candidates[0]).lower(): + fail("priority should be High") + print("OK: HITL priority is High") + + +def check_expense(flow: dict[str, Any], nodes: list[dict[str, Any]]) -> None: + edges = flow.get("edges") + if not isinstance(edges, list): + fail("Flow must contain edges[]") + candidates = hitl_nodes(nodes) + if len(candidates) != 1: + fail(f"need exactly 1 HITL node, found {len(candidates)}") + hitl = candidates[0] + hitl_id = hitl.get("id") + schema = (hitl.get("inputs") or {}).get("schema", {}) + fields = schema.get("fields", []) + outcomes = schema.get("outcomes", []) or [] + if not fields: + fail("HITL needs fields") + if not any( + (field_text(field, "id") == "amount" or "amount" in field_text(field, "label")) + and field.get("type") == "number" + for field in fields + ): + fail("amount must be a number field") + decision_fields = [ + field + for field in fields + if field.get("direction") in {"output", "inOut"} + and any( + key in field_text(field, "id") + for key in ("approval", "approved", "decision") + ) + ] + outcome_names = { + str(outcome.get("name") or outcome.get("id") or "").lower() + for outcome in outcomes + } + has_decision = any(field.get("type") == "boolean" for field in decision_fields) + has_outcomes = any("approve" in name for name in outcome_names) and any( + "reject" in name for name in outcome_names + ) + if not (has_decision or has_outcomes): + fail("need a boolean decision field or approve/reject outcomes") + reason_keys = ("reason", "comment", "explanation", "justification", "note") + if not any( + field.get("direction") in {"output", "inOut"} + and field.get("type") in {"text", "string", "textarea"} + and any( + key in field_text(field, "id") or key in field_text(field, "label") + for key in reason_keys + ) + for field in fields + ): + fail("need a text reason output field") + if not any( + edge.get("sourceNodeId") == hitl_id + and edge.get("sourcePort") in {"completed", "outcome-completed"} + for edge in edges + ): + fail("HITL completed handle must be wired") + scripts = [ + str((node.get("inputs") or {}).get("script", "")) + for node in nodes + if node.get("type") == "core.action.script" + ] + expected = f"$vars.{hitl_id}.output" + if not any(expected in script for script in scripts): + fail(f"downstream script must read HITL output via {expected}") + print("OK: expense HITL schema, routing, and downstream output access are correct") + + +CHECKS = { + "expense": check_expense, + "priority": check_priority, + "quick-form": check_quick_form, + "schema": check_schema, +} + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("check", choices=sorted(CHECKS)) + args = parser.parse_args() + flow, nodes = load_flow() + CHECKS[args.check](flow, nodes) + + +if __name__ == "__main__": + main() diff --git a/tests/tasks/uipath-maestro-flow/_shared/cleanup_solutions.py b/tests/tasks/uipath-maestro-flow/_shared/cleanup_solutions.py index d7e9c29bfa..f89f82bed4 100644 --- a/tests/tasks/uipath-maestro-flow/_shared/cleanup_solutions.py +++ b/tests/tasks/uipath-maestro-flow/_shared/cleanup_solutions.py @@ -4,7 +4,9 @@ Wired in via ``post_run`` in flow e2e task YAMLs. Runs from the sandbox CWD after evaluation completes; finds every ``.uipx`` file under it, reads ``SolutionId``, and best-effort deletes each via ``uip solution delete``. -``.uipx`` files without a ``SolutionId`` are skipped. +``.uipx`` files without a ``SolutionId`` are skipped. Ids that the INTERIM +Overwrite rotation in ``flow_check.py`` moved away from are read from the +``.rotated-solution-ids`` sidecar beside each ``.uipx`` and deleted too. Cleanup policy is controlled by the ``FLOW_E2E_CLEANUP`` env var: @@ -28,6 +30,29 @@ logging.basicConfig(level=logging.INFO, format="cleanup_solutions: %(message)s") logger = logging.getLogger(__name__) +ROTATED_SOLUTION_IDS_SIDECAR = ".rotated-solution-ids" # written by flow_check.py + + +def _solution_ids(uipx_path: str) -> list[tuple[str, str]]: + """``(solution_id, source)`` pairs for one ``.uipx``: its current id plus any + ids the Overwrite rotation left on the tenant (sidecar, one per line).""" + ids: list[tuple[str, str]] = [] + with open(uipx_path) as f: + data = json.load(f) + sid = data.get("SolutionId") + if sid: + ids.append((sid, uipx_path)) + else: + logger.info("no SolutionId in %s", uipx_path) + sidecar = os.path.join(os.path.dirname(uipx_path), ROTATED_SOLUTION_IDS_SIDECAR) + if os.path.isfile(sidecar): + with open(sidecar) as f: + for line in f: + rotated = line.strip() + if rotated and all(rotated != known for known, _ in ids): + ids.append((rotated, sidecar)) + return ids + def _resolve_policy() -> str: policy = os.environ.get("FLOW_E2E_CLEANUP", "always").lower() @@ -53,21 +78,19 @@ def main() -> int: not_uploaded: list[str] = [] failed: list[str] = [] - for path in paths: + targets: list[tuple[str, str]] = [] + for uipx_path in paths: try: - with open(path) as f: - data = json.load(f) + ids = _solution_ids(uipx_path) except (OSError, json.JSONDecodeError) as e: - logger.warning("could not read %s: %s", path, e) - skipped.append(path) - continue - - sid = data.get("SolutionId") - if not sid: - logger.info("no SolutionId in %s, skipping", path) - skipped.append(path) + logger.warning("could not read %s: %s", uipx_path, e) + skipped.append(uipx_path) continue + if not ids: + skipped.append(uipx_path) + targets.extend(ids) + for sid, path in targets: if policy == "never": logger.info( "FLOW_E2E_CLEANUP=never; preserving %s (delete later with: uip solution delete %s)", diff --git a/tests/tasks/uipath-maestro-flow/_shared/flow_check.py b/tests/tasks/uipath-maestro-flow/_shared/flow_check.py index ba3024e26c..325094fa48 100644 --- a/tests/tasks/uipath-maestro-flow/_shared/flow_check.py +++ b/tests/tasks/uipath-maestro-flow/_shared/flow_check.py @@ -34,6 +34,7 @@ from __future__ import annotations import glob +import hashlib import json import math import os @@ -53,6 +54,7 @@ # the exact payload (finalStatus, elementExecutions, globals, incidents) that the # checker saw — which is otherwise ephemeral and unrecoverable post-run. _LAST_DEBUG_RAW: str | None = None +_LAST_DEBUG_STDERR: str | None = None # Variable ids the most recent :func:`run_debug` bound as INPUTS (via ``--inputs`` # / ``--attachment``). Stashed because the runtime returns every global — `in` and @@ -106,9 +108,18 @@ # 5xx, matching the CLI's own "retry once before reporting") that keeps a third # full-length attempt out of every criterion ceiling. `retries=3` still works. _DEFAULT_RETRIES_TIMEOUT = 240 +# One backend read of a faulted instance's incidents; never worth more of the criterion budget. +_INCIDENTS_TIMEOUT_SECONDS = 60 _DEFAULT_RETRIES = 2 _DEFAULT_BACKOFF_SECONDS = 5.0 +# ``finalStatus: Completed`` with NO ``variables`` key (or a ``variablesError``) +# means the CLI's post-completion outputs fetch failed — infrastructure to retry, +# not an empty result to grade (a flow with no outputs still returns the key). +# Retries and the CLI-side fix: UiPath/cli#3929. +_VARIABLES_UNREADABLE_ATTEMPTS = 2 +_STDERR_CAPTURE_TAIL_CHARS = 4000 + # A `uip maestro flow debug` run can die on a transient server-side error — a # gateway timeout / 5xx while polling the debug instance, which the CLI reports # as `Result:Failure`, `ErrorCode:server_error`, `Retry:RetryLater` (the CLI's @@ -125,11 +136,112 @@ ) +def _rglob_pruned(pattern: str) -> list[str]: + """``glob.glob(pattern, recursive=True)`` from the CWD, but never descending + into ``node_modules`` (the preview workspace symlinks the baked SDK tree + there, and ``glob`` follows directory symlinks under ``**``).""" + if "**" not in pattern: + return sorted(glob.glob(pattern, recursive=True)) + prefix, _, suffix = pattern.partition("**") + root = prefix.rstrip("/") or "." + suffix = suffix.lstrip("/") + matches: list[str] = [] + for dirpath, dirnames, _ in os.walk(root, followlinks=True): + dirnames[:] = [d for d in dirnames if d != "node_modules"] + matches.extend(glob.glob(os.path.join(glob.escape(dirpath), suffix))) + if root == ".": + matches = [m[2:] if m.startswith("./") else m for m in matches] + return sorted(set(matches)) + + def _output_blob(result: subprocess.CompletedProcess) -> str: """Both streams, lowercased, for case-insensitive marker matching.""" return f"{result.stdout}\n{result.stderr}".lower() +# INTERIM — Studio Web answers 400/1001 when an EXISTING solution is overwritten +# (UiPath/cli#3938 has the evidence). `flow debug` #2 of a project is an +# overwrite, so on exactly that signature the bundled SolutionId is rotated and +# the debug retried ONCE as a new import. Delete this block when Overwrite works. +_OVERWRITE_STUCK_ISSUE = "UiPath/cli#3938" +_OVERWRITE_STUCK_MARKER = "overwrite failed (400)" +_OVERWRITE_STUCK_CODE = re.compile(r'\\?"code\\?":\s*\\?"1001\\?"') +# Since UiPath/cli#3951 the CLI reports the refusal as +# Context: {"HttpStatus": 400, "Stage": "overwrite-solution", "ErrorCode": "1001"} +# instead of echoing the raw body; both envelopes must rotate. +_OVERWRITE_STUCK_STAGE = "overwrite-solution" +_OVERWRITE_STUCK_ATTEMPTS = 2 # the failed overwrite + one import-as-new retry +_SOLUTION_ID_RE = re.compile(r'("SolutionId"\s*:\s*")([0-9a-fA-F-]{36})(")') +# Rotated-away ids, one per line, beside the `.uipx`; `cleanup_solutions.py` +# deletes them alongside the current id so the rotation leaks nothing. +ROTATED_SOLUTION_IDS_SIDECAR = ".rotated-solution-ids" + + +def _is_overwrite_stuck(result: subprocess.CompletedProcess) -> bool: + """True iff ``flow debug`` died on the Studio Web Overwrite 400/1001 regression + (see :data:`_OVERWRITE_STUCK_ISSUE`) — never on any other overwrite failure. + + Matches the legacy envelope (``Overwrite failed (400): {"code":"1001",…}``) + and the staged one the CLI emits since UiPath/cli#3951 (``Context.Stage == + "overwrite-solution"`` with ``Context.ErrorCode == "1001"``, or HTTP 400 and + the platform's "invalid value" wording when the code is absent).""" + if result.returncode == 0: + return False + blob = _output_blob(result) + if _OVERWRITE_STUCK_MARKER in blob and _OVERWRITE_STUCK_CODE.search(blob) is not None: + return True + data = _parse_json(result.stdout) + context = _get_ci(data or {}, "Context", default={}) + if not isinstance(context, dict): + return False + stage = _get_ci(context, "Stage") + if not isinstance(stage, str) or stage.lower() != _OVERWRITE_STUCK_STAGE: + return False + code = _get_ci(context, "ErrorCode") + if str(code) == "1001": + return True + http = _get_ci(context, "HttpStatus") + return http == 400 and "invalid value" in blob + + +def _rotate_solution_id(project_dir: str) -> tuple[str, str] | None: + """Give the project's solution a fresh bundled ``SolutionId`` so the next + ``flow debug`` imports it as a NEW Studio Web solution instead of overwriting. + + Walks up from ``project_dir`` to the nearest ancestor holding exactly one + ``*.uipx`` (a JSON text file; ``uip solution init`` writes it beside the + project directory). Returns ``(old, new)``, or ``None`` when no unambiguous + ``.uipx`` is found — the caller then fails the plain way. The previous id is + appended to :data:`ROTATED_SOLUTION_IDS_SIDECAR` next to the ``.uipx`` so + ``cleanup_solutions.py`` can delete the solution it still names on the tenant.""" + import uuid + + here = os.path.abspath(project_dir) + for _ in range(4): + uipx = sorted(glob.glob(os.path.join(glob.escape(here), "*.uipx"))) + if len(uipx) == 1: + try: + with open(uipx[0], encoding="utf-8") as handle: + text = handle.read() + except OSError: + return None + match = _SOLUTION_ID_RE.search(text) + if match is None: + return None + new = str(uuid.uuid4()) + with open(uipx[0], "w", encoding="utf-8") as handle: + handle.write(text[: match.start(2)] + new + text[match.end(2):]) + sidecar = os.path.join(here, ROTATED_SOLUTION_IDS_SIDECAR) + with open(sidecar, "a", encoding="utf-8") as handle: + handle.write(match.group(2) + "\n") + return match.group(2), new + parent = os.path.dirname(here) + if parent == here: + break + here = parent + return None + + def _is_transient_debug_error(result: subprocess.CompletedProcess) -> bool: """True iff a failed ``flow debug`` invocation looks like a transient server-side error (5xx / RetryLater / poll-budget expiry) worth retrying, @@ -154,6 +266,36 @@ def _is_poll_timeout(result: subprocess.CompletedProcess) -> bool: ) +def _variables_unreadable(data: dict | None) -> str | None: + """Return a reason when a *completed* debug payload's outputs could not be + read (see :data:`_VARIABLES_UNREADABLE_ATTEMPTS`), else ``None``. + + Exact by construction: only the ``variables`` key being absent from ``Data`` + (or an explicit ``variablesError``) counts. A present-but-empty + ``variables`` is a real result and is returned to the caller as-is.""" + payload = _get_ci(data or {}, "Data") or {} + if not isinstance(payload, dict): + return None + status = _get_ci(payload, "finalStatus", "FinalStatus") + if status != "Completed": + return None + err = _get_ci(payload, "variablesError", "VariablesError") + if isinstance(err, dict): + attempts = _get_ci(err, "attempts", "Attempts") + message = _get_ci(err, "message", "Message") or "unknown error" + return ( + f"the CLI could not read the output variables (variablesError after " + f"{attempts} attempt(s): {message})" + ) + has_key = any(str(k).lower() == "variables" for k in payload) + if not has_key: + return ( + "the CLI returned no `variables` key at all — the post-completion " + "fetch of /debug-instances/{id}/variables failed and was dropped" + ) + return None + + def _as_text(raw: bytes | str | None) -> str: """Decode captured child output. ``subprocess.TimeoutExpired`` carries it as bytes even under ``text=True``, unlike ``CompletedProcess``.""" @@ -231,7 +373,8 @@ def run_debug( cmd.extend(["--attachment", f"{var_id}={local_path}"]) env = dict(os.environ) env.setdefault("UIP_LOG_LEVEL", _DEBUG_LOG_LEVEL) - global _LAST_DEBUG_RAW, _LAST_DEBUG_INPUT_IDS, _LAST_DEBUG_PROJECT_DIR + global _LAST_DEBUG_RAW, _LAST_DEBUG_STDERR, _LAST_DEBUG_INPUT_IDS + global _LAST_DEBUG_PROJECT_DIR # Record what we bound as input, and where the project lives, so output # assertions can discount echoes of our own inputs. _LAST_DEBUG_INPUT_IDS = {str(k) for k in (inputs or {})} | { @@ -275,7 +418,17 @@ def run_debug( # Mirrors debug_budget: pricing an attempt the loop never makes left `r` unbound. attempts = max(1, retries) - for attempt in range(attempts): + unreadable: str | None = None + unreadable_attempts = 0 + overwrite_rotations = 0 + rotated_solution_ids: list[str] = [] + # `max_attempts` starts at the transient allowance and is extended by one + # for an unreadable-outputs retry (see _VARIABLES_UNREADABLE_ATTEMPTS): that + # retry has its own allowance, never the 5xx/RetryLater one. The deadline + # below still bounds every attempt, so the criterion budget holds. + max_attempts = attempts + attempt = 0 + while attempt < max_attempts: attempt_cap = min(timeout, int(deadline - time.monotonic())) cli_timeout = max( _MIN_CLI_TIMEOUT_SECONDS, attempt_cap - _CLI_TIMEOUT_HEADROOM_SECONDS @@ -292,6 +445,7 @@ def run_debug( # The CLI's own --timeout never fired, so the stall is upstream of # polling. Keep the partial output rather than dying on a traceback. _LAST_DEBUG_RAW = _as_text(exc.stdout) + _LAST_DEBUG_STDERR = _as_text(exc.stderr) _fail_with_capture( f"flow debug exceeded the {attempt_cap}s subprocess cap without returning " f"(CLI --timeout was {cli_timeout}s, so the stall is upstream of " @@ -300,16 +454,46 @@ def run_debug( f"stdout: {_as_text(exc.stdout)}\nstderr: {_as_text(exc.stderr)}" ) _LAST_DEBUG_RAW = r.stdout - if r.returncode == 0 or not _is_transient_debug_error(r): + # Keep the CLI's stderr too: it is where `flow debug` reports what it + # could not do (e.g. the output-variables fetch). Dropping it on a zero + # exit is how the 2026-08-31/09-01 move-node failures lost their cause. + _LAST_DEBUG_STDERR = r.stderr + if r.returncode == 0: + unreadable = _variables_unreadable(_parse_json(r.stdout)) + if unreadable is None: + break + unreadable_attempts += 1 + if unreadable_attempts >= _VARIABLES_UNREADABLE_ATTEMPTS: + break + max_attempts = max(max_attempts, attempt + 2) + elif _is_overwrite_stuck(r) and overwrite_rotations < _OVERWRITE_STUCK_ATTEMPTS - 1: + # INTERIM (see _OVERWRITE_STUCK_ISSUE): its own single allowance, like + # the unreadable-outputs retry; the deadline below still bounds it. + rotated = _rotate_solution_id(project_dir) + if rotated is None: + break + overwrite_rotations += 1 + rotated_solution_ids.append(rotated[0]) + print( + f"INTERIM ({_OVERWRITE_STUCK_ISSUE}: Studio Web Overwrite → 400/1001 for an " + f"existing solution since 2026-09-02): rotated the bundled SolutionId " + f"{rotated[0]} → {rotated[1]}; retrying as a new import. The solution " + f"{rotated[0]} is recorded in {ROTATED_SOLUTION_IDS_SIDECAR} for cleanup.", + file=sys.stderr, + flush=True, + ) + max_attempts = max(max_attempts, attempt + 2) + elif not _is_transient_debug_error(r): break - if _is_poll_timeout(r) and attempt + 1 >= _POLL_TIMEOUT_ATTEMPTS: + elif _is_poll_timeout(r) and attempt + 1 >= _POLL_TIMEOUT_ATTEMPTS: break - if attempt + 1 < attempts: + if attempt + 1 < max_attempts: left = deadline - time.monotonic() - backoff_seconds if left < _MIN_RETRY_BUDGET_SECONDS: out_of_budget = True break time.sleep(backoff_seconds) + attempt += 1 if r.returncode != 0: spent = ( @@ -321,14 +505,33 @@ def run_debug( _fail( f"flow debug exit {r.returncode}{spent}\n" f"stdout: {r.stdout}\nstderr: {r.stderr}" + + _incident_details(_parse_json(r.stdout)) ) data = _parse_json(r.stdout) if data is None: _fail(f"Could not parse JSON from flow debug\n{r.stdout}") payload = _get_ci(data, "Data") or {} + if rotated_solution_ids: + print( + f"INTERIM ({_OVERWRITE_STUCK_ISSUE}): debug ran as a new import; Studio Web " + f"solution {_get_ci(payload, 'solutionId', 'SolutionId')} (previous: " + f"{', '.join(rotated_solution_ids)}).", + file=sys.stderr, + flush=True, + ) status = _get_ci(payload, "finalStatus", "FinalStatus") if status != "Completed": - _fail(f"Flow did not complete (finalStatus={status})\n{r.stdout}") + _fail(f"Flow did not complete (finalStatus={status})\n{r.stdout}" + _incident_details(data)) + if unreadable is not None: + # Every attempt completed, and every attempt's outputs were unreadable. + # This is an infrastructure failure (INFRA), not a verdict on the flow: + # the run reached its End node, we just never saw what it returned. + _fail_with_capture( + f"flow debug completed but its outputs could not be read on " + f"{unreadable_attempts} attempt(s): {unreadable}. " + "Classify as INFRA (outputs unknown), not as a flow defect; the " + "elements all ran. See the captured STDERR for the CLI's own account." + ) return payload @@ -524,8 +727,7 @@ def assert_loop_body_nodes_parented( loop's ID. Without ``parentId``, the runtime executes the node outside the loop context — per-iteration variables like ``currentItem`` are inaccessible and outputs come back null.""" - project_dir = _find_project(project_glob) - for path in glob.glob(os.path.join(project_dir, "**/*.flow"), recursive=True): + for path in find_flow_files(project_glob): with open(path) as f: flow = json.load(f) nodes_by_id = {n["id"]: n for n in flow.get("nodes") or []} @@ -783,6 +985,11 @@ def assert_outputs_contain( ) +def get_last_debug_stderr() -> str | None: + """Return the stderr of the most recent ``run_debug`` call, or ``None``.""" + return _LAST_DEBUG_STDERR + + def get_last_debug_raw() -> str | None: """Return the raw stdout of the most recent ``run_debug`` call (the full ``uip maestro flow debug`` JSON envelope), or ``None`` if none ran yet. @@ -1332,12 +1539,10 @@ def node_output_leaves(payload: dict, node_ids) -> set: def _load_flow(project_glob: str) -> dict: - """Load the single .flow next to the matched project.uiproj.""" - proj = _find_project(project_glob) - flows = glob.glob(os.path.join(proj, "*.flow")) - if not flows: - _fail(f"no .flow file under {proj}") - return json.loads(open(flows[0], encoding="utf-8").read()) + """Load the single discovered flow artifact.""" + path = find_flow_file(project_glob) + with open(path, encoding="utf-8") as f: + return json.load(f) def assert_decision_branches_reach( @@ -1520,9 +1725,143 @@ def find_project_dir(pattern: str = "**/project.uiproj") -> str: return _find_project(pattern) +def find_flow_files( + project_pattern: str = "**/project.uiproj", *, flow_glob: str = "*.flow" +) -> list[str]: + """Return flow artifacts from the selected solution, or one root emit. + + A substantive Flow solution project wins whenever one exists. This keeps a + root-level SDK scratch file from competing with the linked solution copy. + When no project exists, or every project is only an abandoned scaffold, + validate and source-structure checks may consume a substantive root-level + SDK emit. Multiple genuinely distinct root emits are ambiguous and fail + with their paths rather than selecting one by glob order. + + Debug callers intentionally do not use this helper: ``uip maestro flow + debug`` is project-scoped and must continue through :func:`find_project_dir`. + """ + project_candidates = _rglob_pruned(project_pattern) + flow_projects = [path for path in project_candidates if _is_flow_project(path)] + root_matches = _dedupe_flow_candidates( + sorted(glob.glob(os.path.basename(flow_glob))) + ) + + # A preview author may compile a complete root-level SDK source before a + # later CLI command leaves an untouched trigger-only project scaffold. For + # file-scoped checks the substantive artifact is the build; the scaffold is + # not allowed to shadow it merely because it carries project.uiproj. + if flow_projects and len(root_matches) == 1: + project_counts = [ + _flow_node_count(os.path.dirname(path)) for path in flow_projects + ] + root_count = _flow_file_node_count(root_matches[0]) + if ( + root_count is not None + and root_count > _HUSK_MAX_NODES + and all( + count is not None and count <= _HUSK_MAX_NODES + for count in project_counts + ) + ): + print( + "note: ignoring abandoned project scaffold(s) in favor of " + f"substantive root Flow emit: {root_matches[0]}" + ) + return root_matches + + if flow_projects: + project_dir = _find_project(project_pattern) + matches = sorted( + glob.glob( + os.path.join(project_dir, "**", os.path.basename(flow_glob)), + recursive=True, + ) + ) + if not matches: + _fail( + f"No .flow file matching {flow_glob!r} under selected Flow " + f"project {project_dir}" + ) + return matches + + if not root_matches: + _fail( + f"No Flow project found matching {project_pattern!r}, and no " + f"root-level .flow file matching {os.path.basename(flow_glob)!r}" + ) + if len(root_matches) > 1: + joined = "\n - ".join(root_matches) + _fail( + "Multiple distinct root-level .flow files found — refusing to " + f"guess:\n - {joined}" + ) + return root_matches + + +def find_flow_file( + project_pattern: str = "**/project.uiproj", *, flow_glob: str = "*.flow" +) -> str: + """Return one unambiguous flow artifact using :func:`find_flow_files`.""" + matches = find_flow_files(project_pattern, flow_glob=flow_glob) + if len(matches) > 1: + joined = "\n - ".join(matches) + _fail(f"Multiple .flow files match {flow_glob!r}:\n - {joined}") + return matches[0] + + +def assert_no_flow_files() -> None: + """Require that the sandbox contains no ``.flow`` artifact anywhere. + + This is the negative counterpart to :func:`find_flow_files` for read-only + tasks. It deliberately scans every location because any emitted Flow is a + failure; there is no preferred artifact to select in the absence case. + """ + matches = _rglob_pruned("**/*.flow") + if matches: + joined = "\n - ".join(matches) + _fail(f"Unexpected .flow file(s) found:\n - {joined}") + + # ── Internals ─────────────────────────────────────────────────────────────── +def _incident_details(envelope: dict | None) -> str: + """The backend's own account of a faulted debug run, for the failure text. + + `flow debug` reports a fault as one line per incident (`[102003] Integration + Services bad request (element createIssue)`) and defers the provider's + message to `uip maestro flow debug-instance incidents `. That message is + the cause — 2026-09-01: `errors - {reporter=Specify a valid value for + Reporter}` behind six identical `[102003]` failures across three arms and two + weeks, none of whose task.json said so. Fetch it while the instance is still + on the tenant (it is gone within days) so the criterion output carries it. + + Best-effort and side-effect-free: returns "" when there is no instance id, + the CLI is unavailable, or the tenant answers with anything but a list. + """ + data = _get_ci(envelope, "Data") or {} + instance_id = _get_ci(data, "instanceId", "InstanceId") + if not instance_id: + return "" + try: + r = subprocess.run( + ["uip", "maestro", "flow", "debug-instance", "incidents", str(instance_id), "--output", "json"], + capture_output=True, text=True, timeout=_INCIDENTS_TIMEOUT_SECONDS, + ) + parsed = _parse_json(r.stdout) or {} + incidents = _get_ci(parsed, "Data") + if not isinstance(incidents, list) or not incidents: + return "" + lines = [ + f" - element={_get_ci(i, 'elementId', 'ElementId')} code={_get_ci(i, 'errorCode', 'ErrorCode')} " + f"{_get_ci(i, 'errorMessage', 'ErrorMessage')}: {_get_ci(i, 'errorDetails', 'ErrorDetails')}" + for i in incidents if isinstance(i, dict) + ] + return "\nincidents (uip maestro flow debug-instance incidents):\n" + "\n".join(lines) + except Exception: # noqa: BLE001 — diagnostics must never mask the real failure + return "" + + def _parse_json(stdout: str) -> dict | None: try: return json.loads(stdout) @@ -1561,8 +1900,7 @@ def _get_ci(mapping: Any, *candidate_keys: str, default: Any = None) -> Any: def _iter_flow_nodes(project_glob: str): - project_dir = _find_project(project_glob) - for path in glob.glob(os.path.join(project_dir, "**/*.flow"), recursive=True): + for path in find_flow_files(project_glob): with open(path) as f: flow = json.load(f) yield from flow.get("nodes") or [] @@ -1579,6 +1917,23 @@ def _non_empty_binding_value(value: Any) -> bool: _HUSK_MAX_NODES = 1 +def _dedupe_flow_candidates(paths: list[str]) -> list[str]: + """Collapse aliases and byte-identical copies before ambiguity checks.""" + by_realpath: dict[str, str] = {} + for path in paths: + by_realpath.setdefault(os.path.realpath(path), path) + + by_content: dict[str, str] = {} + for path in by_realpath.values(): + try: + with open(path, "rb") as f: + key = hashlib.sha256(f.read()).hexdigest() + except OSError: + key = f"unreadable:{path}" + by_content.setdefault(key, path) + return sorted(by_content.values()) + + def _find_project(pattern: str) -> str: """Locate the *Flow* project directory matching ``pattern``. @@ -1591,10 +1946,11 @@ def _find_project(pattern: str) -> str: Filtering by manifest avoids a 1-of-N glob collision the symptom of MST-9734. - Two Flow projects can also mean one build plus one abandoned scaffold — - see :func:`_split_off_scaffold_husks`. Anything else stays a refusal. + Two Flow projects can also mean byte-identical copies, or one build plus one + abandoned scaffold — see :func:`_dedupe_flow_projects` and + :func:`_split_off_scaffold_husks`. Anything else stays a refusal. """ - candidates = sorted(glob.glob(pattern, recursive=True)) + candidates = _rglob_pruned(pattern) if not candidates: _fail(f"No project.uiproj found matching {pattern}") flow_projects = [p for p in candidates if _is_flow_project(p)] @@ -1605,6 +1961,15 @@ def _find_project(pattern: str) -> str: f'candidates exist but none declare ProjectType="Flow":\n - {joined}' ) if len(flow_projects) > 1: + original_count = len(flow_projects) + flow_projects = _dedupe_flow_projects(flow_projects) + if len(flow_projects) == 1: + print( + "note: ignoring " + f"{original_count - 1} byte-identical Flow project duplicate(s)" + ) + return os.path.dirname(flow_projects[0]) + counts = [(p, _flow_node_count(os.path.dirname(p))) for p in flow_projects] selected, husks = _split_off_scaffold_husks(counts) if selected is not None: @@ -1651,18 +2016,54 @@ def _flow_node_count(project_dir: str) -> int | None: return None total = 0 for path in flows: - try: - with open(path, encoding="utf-8") as f: - flow = json.load(f) - except (OSError, json.JSONDecodeError, UnicodeDecodeError): - return None - nodes = flow.get("nodes") if isinstance(flow, dict) else None - if not isinstance(nodes, list): + count = _flow_file_node_count(path) + if count is None: return None - total += len(nodes) + total += count return total +def _flow_file_node_count(path: str) -> int | None: + """Return one Flow file's node count, or ``None`` when unreadable.""" + try: + with open(path, encoding="utf-8") as f: + flow = json.load(f) + except (OSError, json.JSONDecodeError, UnicodeDecodeError): + return None + nodes = flow.get("nodes") if isinstance(flow, dict) else None + return len(nodes) if isinstance(nodes, list) else None + + +def _dedupe_flow_projects(project_uiprojs: list[str]) -> list[str]: + """Collapse projects whose relative Flow files are byte-for-byte equal. + + Missing or unreadable Flow files stay distinct so project-scoped debug never + guesses across candidates whose equivalence cannot be proved. + """ + by_content: dict[tuple[tuple[str, str], ...] | tuple[str, str], str] = {} + for project_uiproj in project_uiprojs: + project_dir = os.path.dirname(project_uiproj) + flows = sorted( + glob.glob(os.path.join(project_dir, "**/*.flow"), recursive=True) + ) + fingerprints: list[tuple[str, str]] = [] + try: + for path in flows: + with open(path, "rb") as f: + digest = hashlib.sha256(f.read()).hexdigest() + fingerprints.append((os.path.relpath(path, project_dir), digest)) + except OSError: + fingerprints = [] + + key: tuple[tuple[str, str], ...] | tuple[str, str] + if fingerprints: + key = tuple(fingerprints) + else: + key = ("unknown", project_uiproj) + by_content.setdefault(key, project_uiproj) + return sorted(by_content.values()) + + def _describe_candidate(project_uiproj: str, node_count: int | None) -> str: project_dir = os.path.dirname(project_uiproj) if node_count is None: @@ -1737,6 +2138,11 @@ def _dump_debug_capture(context: str = "") -> None: except Exception as exc: # noqa: BLE001 — diagnostics must never mask the real failure lines.append(f"SUMMARY: ") lines.append("RAW: " + raw.strip()) + stderr = (_LAST_DEBUG_STDERR or "").strip() + if stderr: + # The CLI's own account of the run (progress, warnings such as a failed + # output-variables fetch). Tail only: the head is login/upload noise. + lines.append("STDERR (tail): " + stderr[-_STDERR_CAPTURE_TAIL_CHARS:]) lines.append("=== FLOW_DEBUG_RAW_CAPTURE END ===") print("\n".join(lines), file=sys.stderr) diff --git a/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py b/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py index 466293418a..9734adb878 100644 --- a/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py +++ b/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py @@ -24,6 +24,8 @@ expected_exit_code: 0) instead of inverting a positive check — a bare exit 1 cannot distinguish "pattern absent" from "no flow found" + --expect-none require that no ``.flow`` file exists anywhere in the + sandbox (for read-only tasks) With no arguments this asserts only that a ``.flow`` file exists. All positive assertions (substrings + ``--regex``) must be satisfied by a single file; @@ -41,13 +43,12 @@ from __future__ import annotations -import glob import os import re import sys sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -from flow_check import find_project_dir # noqa: E402 +from flow_check import assert_no_flow_files, find_flow_files # noqa: E402 def _parse(argv: list[str]): @@ -55,6 +56,7 @@ def _parse(argv: list[str]): regexes: list[str] = [] absent: list[str] = [] flow_name: str | None = None + expect_none = False it = iter(argv) for arg in it: try: @@ -64,22 +66,39 @@ def _parse(argv: list[str]): absent.append(next(it)) elif arg == "--flow-name": flow_name = next(it) + elif arg == "--expect-none": + expect_none = True else: substrings.append(arg) except StopIteration: print(f"FAIL: {arg} requires a value", file=sys.stderr) sys.exit(2) - return substrings, regexes, absent, flow_name + return substrings, regexes, absent, flow_name, expect_none def main(argv: list[str]) -> int: - substrings, regexes, absent, flow_name = _parse(argv) + substrings, regexes, absent, flow_name, expect_none = _parse(argv) - project_dir = find_project_dir() - flows = sorted(glob.glob(os.path.join(project_dir, "**/*.flow"), recursive=True)) - if not flows: - print(f"FAIL: No .flow file found under {project_dir}", file=sys.stderr) - return 1 + if expect_none: + if substrings or regexes or absent or flow_name is not None: + print( + "FAIL: --expect-none cannot be combined with other assertions", + file=sys.stderr, + ) + return 2 + try: + assert_no_flow_files() + except SystemExit as exc: + print(str(exc), file=sys.stderr) + return exc.code if isinstance(exc.code, int) else 1 + print("OK: no .flow files found") + return 0 + + try: + flows = find_flow_files() + except SystemExit as exc: + print(str(exc), file=sys.stderr) + return exc.code if isinstance(exc.code, int) else 1 print(f"Found {len(flows)} .flow file(s): {', '.join(flows)}") if flow_name is not None: diff --git a/tests/tasks/uipath-maestro-flow/_shared/test_batch2_alignment.py b/tests/tasks/uipath-maestro-flow/_shared/test_batch2_alignment.py new file mode 100644 index 0000000000..0de9fb36a2 --- /dev/null +++ b/tests/tasks/uipath-maestro-flow/_shared/test_batch2_alignment.py @@ -0,0 +1,180 @@ +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path + +SHARED = Path(__file__).resolve().parent +FLOW_TASKS = SHARED.parent + + +def run_script(script: Path, *args: str, cwd: Path) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, str(script), *args], + cwd=cwd, + capture_output=True, + text=True, + check=False, + ) + + +def test_simulated_hitl_checks_accept_root_sdk_emit(tmp_path: Path) -> None: + flow = { + "nodes": [ + { + "id": "review", + "type": "uipath.human-in-the-loop", + "inputs": { + "type": "quick", + "priority": "High", + "schema": { + "fields": [ + { + "id": "amount", + "label": "Amount", + "type": "number", + "direction": "input", + }, + { + "id": "reason", + "label": "Reason", + "type": "text", + "direction": "output", + }, + ], + "outcomes": [{"name": "Approve"}, {"name": "Reject"}], + }, + }, + }, + { + "id": "log", + "type": "core.action.script", + "inputs": {"script": "return $vars.review.output.reason;"}, + }, + ], + "edges": [ + { + "sourceNodeId": "review", + "sourcePort": "completed", + "targetNodeId": "log", + "targetPort": "input", + } + ], + } + (tmp_path / "Review.flow").write_text(json.dumps(flow)) + script = SHARED / "check_simulated_hitl.py" + + for check in ("expense", "priority", "quick-form", "schema"): + result = run_script(script, check, cwd=tmp_path) + assert result.returncode == 0, result.stdout + result.stderr + + +def test_quick_form_check_accepts_specialized_node_type(tmp_path: Path) -> None: + flow = { + "nodes": [ + { + "id": "review", + "type": "uipath.human-in-the-loop.quick-form", + "inputs": {}, + } + ] + } + (tmp_path / "Review.flow").write_text(json.dumps(flow)) + + result = run_script(SHARED / "check_simulated_hitl.py", "quick-form", cwd=tmp_path) + + assert result.returncode == 0, result.stdout + result.stderr + + +def test_quick_form_check_rejects_action_app_variant(tmp_path: Path) -> None: + flow = { + "nodes": [ + { + "id": "review", + "type": "uipath.human-in-the-loop", + "inputs": {"type": "custom"}, + } + ] + } + (tmp_path / "Review.flow").write_text(json.dumps(flow)) + + result = run_script(SHARED / "check_simulated_hitl.py", "quick-form", cwd=tmp_path) + + assert result.returncode != 0 + assert "inline HITL quick form" in result.stderr + + +def test_schema_check_rejects_outcomes_only_on_an_unrelated_node( + tmp_path: Path, +) -> None: + flow = { + "nodes": [ + { + "id": "review", + "type": "uipath.human-in-the-loop", + "inputs": { + "schema": { + "fields": [ + {"direction": "input"}, + {"direction": "output"}, + ] + } + }, + }, + { + "id": "unrelated", + "type": "core.action.script", + "name": "Approve or Reject", + }, + ] + } + (tmp_path / "Review.flow").write_text(json.dumps(flow)) + + result = run_script(SHARED / "check_simulated_hitl.py", "schema", cwd=tmp_path) + + assert result.returncode != 0 + assert "Approve and Reject outcomes" in result.stderr + + +def test_solution_select_checks_use_selected_solution(tmp_path: Path) -> None: + for name in ("SolarReports", "TideTracker"): + directory = tmp_path / name + directory.mkdir() + (directory / f"{name}.uipx").write_text(json.dumps({"Projects": []})) + + selected = tmp_path / "WeatherSelection-7K4M" + project = selected / "WeatherAlert" + project.mkdir(parents=True) + (selected / "WeatherSelection-7K4M.uipx").write_text( + json.dumps({"Projects": [{"Name": "WeatherAlert", "Type": "Flow"}]}) + ) + (project / "project.uiproj").write_text( + json.dumps({"Name": "WeatherAlert", "ProjectType": "Flow"}) + ) + (project / "WeatherAlert.flow").write_text(json.dumps({"nodes": []})) + script = FLOW_TASKS / "interactive" / "check_solution_select.py" + + for check in ( + "existing-untouched", + "flow", + "no-extra-solution", + "project", + "solution", + ): + result = run_script(script, check, cwd=tmp_path) + assert result.returncode == 0, result.stdout + result.stderr + + +def test_solution_select_advisory_reports_extra_default_solution( + tmp_path: Path, +) -> None: + extra = tmp_path / "WeatherAlertSol" + extra.mkdir() + (extra / "WeatherAlertSol.uipx").write_text(json.dumps({"Projects": []})) + script = FLOW_TASKS / "interactive" / "check_solution_select.py" + + result = run_script(script, "no-extra-solution", cwd=tmp_path) + + assert result.returncode != 0 + assert "unexpected default solution" in result.stderr diff --git a/tests/tasks/uipath-maestro-flow/_shared/test_batch3_alignment.py b/tests/tasks/uipath-maestro-flow/_shared/test_batch3_alignment.py new file mode 100644 index 0000000000..bbf1247662 --- /dev/null +++ b/tests/tasks/uipath-maestro-flow/_shared/test_batch3_alignment.py @@ -0,0 +1,112 @@ +"""Regression coverage for Batch 3 same-ground checker routing.""" + +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path + + +TASK_ROOT = Path(__file__).resolve().parent.parent + + +def _run(relative: str, *args: str, cwd: Path) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, str(TASK_ROOT / relative), *args], + cwd=cwd, + capture_output=True, + text=True, + check=False, + ) + + +def _write_flow(path: Path, nodes: list[dict], bindings: list[dict] | None = None) -> None: + path.write_text( + json.dumps({"nodes": nodes, "edges": [], "bindings": bindings or []}), + encoding="utf-8", + ) + + +def test_bindings_checker_accepts_root_sdk_emit(tmp_path: Path) -> None: + _write_flow( + tmp_path / "BindingsMulti.flow", + [ + {"id": "a", "type": "uipath.connector.one.operation"}, + {"id": "b", "type": "uipath.connector.two.operation"}, + ], + ) + + result = _run( + "bindings/check_bindings.py", + "connector_node_count", + "**/BindingsMulti*.flow", + "2", + cwd=tmp_path, + ) + + assert result.returncode == 0, result.stderr or result.stdout + + +def test_multiselect_checker_accepts_root_sdk_emit(tmp_path: Path) -> None: + _write_flow( + tmp_path / "ComplexArrayTest.flow", + [ + { + "id": "slack", + "type": "uipath.connector.uipath-salesforce-slack.create-group-direct-message", + "inputs": {"detail": {"bodyParameters": {"users": ["U1", "U2"]}}}, + } + ], + ) + + result = _run( + "connector_features/check_multiselect_flow.py", + "populated", + cwd=tmp_path, + ) + + assert result.returncode == 0, result.stderr or result.stdout + + +def test_enum_checker_prefers_solution_flow_over_root_scratch(tmp_path: Path) -> None: + _write_flow(tmp_path / "EnumTest.flow", []) + project = tmp_path / "EnumTest" / "project.uiproj" + project.parent.mkdir() + project.write_text('{"ProjectType":"Flow"}', encoding="utf-8") + _write_flow( + project.parent / "EnumTest.flow", + [{"id": "start", "type": "core.trigger.manual"}], + ) + + result = _run( + "connector_features/check_enum_flow.py", + "**/EnumTest*.flow", + "structure", + cwd=tmp_path, + ) + + assert result.returncode == 0, result.stderr or result.stdout + assert "EnumTest/EnumTest.flow" in result.stdout + + +def test_testmanager_execution_evidence_is_distinct_from_round_trip( + tmp_path: Path, +) -> None: + (tmp_path / "seed.json").write_text('{"name": "expected"}') + (tmp_path / "result.json").write_text( + '{"created_name": "different", "retrieved_name": "different", "id": "123"}' + ) + + evidence = _run( + "connector_features/testmanager_crud_grounded/check.py", + "--execution-evidence", + cwd=tmp_path, + ) + semantic = _run( + "connector_features/testmanager_crud_grounded/check.py", + cwd=tmp_path, + ) + + assert evidence.returncode == 0, evidence.stderr or evidence.stdout + assert semantic.returncode == 1 diff --git a/tests/tasks/uipath-maestro-flow/_shared/test_batch4_alignment.py b/tests/tasks/uipath-maestro-flow/_shared/test_batch4_alignment.py new file mode 100644 index 0000000000..4eefb7abc2 --- /dev/null +++ b/tests/tasks/uipath-maestro-flow/_shared/test_batch4_alignment.py @@ -0,0 +1,247 @@ +"""Regression coverage for Batch 4 same-ground checker routing.""" + +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from flow_check import assert_no_flow_files # noqa: E402 + +TASK_ROOT = Path(__file__).resolve().parent.parent + + +def _write_flow(path: Path, nodes: list[dict], **extra) -> None: + path.write_text( + json.dumps({"nodes": nodes, "edges": [], **extra}), encoding="utf-8" + ) + + +def _run(relative: str, cwd: Path) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, str(TASK_ROOT / relative)], + cwd=cwd, + capture_output=True, + text=True, + check=False, + ) + + +def test_no_flow_assertion_accepts_empty_sandbox(tmp_path: Path, monkeypatch) -> None: + monkeypatch.chdir(tmp_path) + assert_no_flow_files() + + +def test_no_flow_assertion_rejects_nested_flow(tmp_path: Path, monkeypatch) -> None: + monkeypatch.chdir(tmp_path) + nested = tmp_path / "Solution" / "Project" + nested.mkdir(parents=True) + _write_flow(nested / "Unexpected.flow", []) + + with pytest.raises(SystemExit, match="Unexpected .flow"): + assert_no_flow_files() + + +def test_flow_contains_expect_none_cli(tmp_path: Path) -> None: + result = subprocess.run( + [ + sys.executable, + str(Path(__file__).parent / "flow_contains.py"), + "--expect-none", + ], + cwd=tmp_path, + capture_output=True, + text=True, + check=False, + ) + + assert result.returncode == 0, result.stderr or result.stdout + + +def test_ixp_project_selection_accepts_root_sdk_emit(tmp_path: Path) -> None: + row = tmp_path / "skill-flow-ixp-e2e-project-selection" / "aviation" + row.mkdir(parents=True) + model = "aviation-investigation-final-report-demo-a42b1d4d-ixp" + _write_flow( + row / "Aviation.flow", + [ + { + "id": "extract", + "type": f"uipath.ixp.{model}.extract", + "inputs": {"modelName": model}, + } + ], + ) + + result = _run("ixp/check_project_selection.py", cwd=row) + + assert result.returncode == 0, result.stderr or result.stdout + + +def test_ixp_project_selection_rejects_mismatched_model(tmp_path: Path) -> None: + row = tmp_path / "skill-flow-ixp-e2e-project-selection" / "birth-certificate" + row.mkdir(parents=True) + _write_flow( + row / "Birth.flow", + [ + { + "id": "extract", + "type": "uipath.ixp.wrong-model.extract", + "inputs": {"modelName": "wrong-model"}, + } + ], + ) + + result = _run("ixp/check_project_selection.py", cwd=row) + + assert result.returncode == 1 + + +@pytest.mark.parametrize("typed_envelopes", [False, True]) +def test_batch_transform_checker_accepts_equivalent_expression_shapes( + tmp_path: Path, typed_envelopes: bool +) -> None: + attachment = ( + { + "type": "jsExpression", + "expression": "$vars.trigger.output.csvFile", + "fieldType": "string", + } + if typed_envelopes + else "=js:$vars.trigger.output.csvFile" + ) + output_source = ( + {"type": "literal", "expression": "=response", "fieldType": "string"} + if typed_envelopes + else "=response" + ) + result_source = ( + { + "type": "jsExpression", + "expression": "$vars.batch.output", + "fieldType": "string", + } + if typed_envelopes + else "=js:$vars.batch.output" + ) + _write_flow( + tmp_path / "BatchTransformDemo.flow", + [ + {"id": "trigger", "type": "core.trigger.manual"}, + { + "id": "batch", + "type": "uipath.pattern.batch-transform", + "typeVersion": "1.0", + "inputs": { + "attachment": attachment, + "prompt": "Classify each row", + "outputColumns": [ + {"name": "Category", "description": "classification"} + ], + }, + "outputs": {"output": {"source": output_source}}, + }, + { + "id": "end", + "type": "core.control.end", + "outputs": {"result": {"source": result_source}}, + }, + ], + variables={ + "globals": [ + { + "id": "csvFile", + "direction": "in", + "type": "file", + "triggerNodeId": "trigger", + }, + {"id": "result", "direction": "out", "type": "file"}, + ] + }, + ) + + result = _run( + "context-grounding/batch_transform/check_batch_transform_flow.py", + cwd=tmp_path, + ) + + assert result.returncode == 0, result.stderr or result.stdout + + +@pytest.mark.parametrize("typed_envelopes", [False, True]) +def test_summarize_checker_accepts_equivalent_expression_shapes( + tmp_path: Path, typed_envelopes: bool +) -> None: + attachment = ( + { + "type": "jsExpression", + "expression": "$vars.trigger.output.documentFile", + "fieldType": "string", + } + if typed_envelopes + else "=js:$vars.trigger.output.documentFile" + ) + output_source = ( + {"type": "literal", "expression": "=response", "fieldType": "string"} + if typed_envelopes + else "=response" + ) + + def output_mapping(field: str): + expression = f"$vars.summary_node.output.content.{field}" + if typed_envelopes: + return { + "type": "jsExpression", + "expression": expression, + "fieldType": "string", + } + return f"=js:{expression}" + + _write_flow( + tmp_path / "SummarizeDemo.flow", + [ + {"id": "trigger", "type": "core.trigger.manual"}, + { + "id": "summary_node", + "type": "uipath.pattern.deep-rag", + "typeVersion": "1.0", + "inputs": { + "attachment": attachment, + "prompt": "Summarize the document", + "returnCitations": True, + }, + "outputs": {"output": {"source": output_source}}, + }, + { + "id": "end", + "type": "core.control.end", + "outputs": { + "summary": {"source": output_mapping("Text")}, + "citations": {"source": output_mapping("Citations")}, + }, + }, + ], + variables={ + "globals": [ + { + "id": "documentFile", + "direction": "in", + "type": "file", + "triggerNodeId": "trigger", + }, + {"id": "summary", "direction": "out", "type": "string"}, + {"id": "citations", "direction": "out", "type": "array"}, + ] + }, + ) + + result = _run( + "context-grounding/summarize/check_summarize_flow.py", + cwd=tmp_path, + ) + + assert result.returncode == 0, result.stderr or result.stdout diff --git a/tests/tasks/uipath-maestro-flow/_shared/test_batch5_alignment.py b/tests/tasks/uipath-maestro-flow/_shared/test_batch5_alignment.py new file mode 100644 index 0000000000..ae0539a14c --- /dev/null +++ b/tests/tasks/uipath-maestro-flow/_shared/test_batch5_alignment.py @@ -0,0 +1,435 @@ +"""Regression coverage for Batch 5 same-ground artifact checks.""" + +from __future__ import annotations + +import importlib.util +import json +import subprocess +import sys +from pathlib import Path + +import pytest + +TASK_ROOT = Path(__file__).resolve().parent.parent + + +def _load_module(relative: str, name: str): + path = TASK_ROOT / relative + spec = importlib.util.spec_from_file_location(name, path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _write_json(path: Path, value: object) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(value), encoding="utf-8") + + +def _run(relative: str, cwd: Path, *args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, str(TASK_ROOT / relative), *args], + cwd=cwd, + capture_output=True, + text=True, + check=False, + ) + + +@pytest.mark.parametrize("goal", ["goal-a", "goal-b", "goal-c"]) +def test_type_choice_artifact_modes(tmp_path: Path, goal: str) -> None: + expected = { + "goal-a": ( + "goal-a-evaluator", + "uipath-llm-judge-output-semantic-similarity", + ), + "goal-b": ("goal-b-evaluator", "uipath-json-similarity"), + "goal-c": ("goal-c-evaluator", "uipath-contains"), + } + name, type_id = expected[goal] + _write_json( + tmp_path / "Wrapper" / "Flow" / "evals" / f"{goal}.json", + {"name": name, "evaluatorTypeId": type_id}, + ) + + result = _run("evaluate/check_type_choice.py", tmp_path, "--artifact", goal) + + assert result.returncode == 0, result.stderr or result.stdout + + +def test_type_choice_default_checks_report(tmp_path: Path) -> None: + _write_json( + tmp_path / "report.json", + { + "goal_a_type": "llm-judge-output", + "goal_b_type": "json-similarity", + "goal_c_type": "contains", + }, + ) + + result = _run("evaluate/check_type_choice.py", tmp_path) + + assert result.returncode == 0, result.stderr or result.stdout + + +def test_type_choice_report_accepts_persisted_namespaced_ids(tmp_path: Path) -> None: + _write_json( + tmp_path / "report.json", + { + "goal_a_type": "uipath-llm-judge-output-semantic-similarity", + "goal_b_type": "uipath-json-similarity", + "goal_c_type": "uipath-contains", + }, + ) + + result = _run("evaluate/check_type_choice.py", tmp_path) + + assert result.returncode == 0, result.stderr or result.stdout + + +def test_add_node_celsius_check_matches_the_requested_output() -> None: + checker = _load_module("edit/add_node/check_add_node.py", "check_add_node") + payload = { + "variables": { + "globals": { + "convertToCelsius.output": { + "temperatureF": 61.3, + "temperatureC": 16.277, + }, + "formatSummary.output": {"summary": "Bellevue: 61.3F (16.3C)"}, + } + } + } + + checker._assert_celsius_output(payload) + + +def test_add_node_celsius_check_rejects_unrelated_weather_advice() -> None: + checker = _load_module("edit/add_node/check_add_node.py", "check_add_node_bad") + payload = { + "variables": { + "globals": { + "convertToCelsius.output": {"temperatureF": 61.3}, + "formatSummary.output": {"summary": "Nice day; bring a jacket"}, + } + } + } + + with pytest.raises(SystemExit, match="numeric temperatureC"): + checker._assert_celsius_output(payload) + + +@pytest.fixture +def local_crud_sandbox(tmp_path: Path) -> Path: + root = tmp_path / "arbitrary-wrapper" / "SmokeEval" / "evals" + _write_json( + root / "evaluators" / "greeting.json", + {"name": "greeting-match", "evaluatorTypeId": "uipath-exact-match"}, + ) + _write_json( + root / "eval-sets" / "smoke.json", + { + "name": "Smoke Set", + "evaluations": [ + { + "name": "hello", + "inputs": {"name": "Alice"}, + "expectedOutput": {"greeting": "Hello, Alice!"}, + } + ], + }, + ) + return tmp_path + + +@pytest.mark.parametrize("check", ["evaluator", "eval-set", "data-point"]) +def test_local_crud_modes(local_crud_sandbox: Path, check: str) -> None: + result = _run("evaluate/check_local_crud.py", local_crud_sandbox, "--check", check) + assert result.returncode == 0, result.stderr or result.stdout + + +def test_local_crud_default_checks_combined_artifacts(local_crud_sandbox: Path) -> None: + result = _run("evaluate/check_local_crud.py", local_crud_sandbox) + assert result.returncode == 0, result.stderr or result.stdout + + +@pytest.fixture +def no_upload_sandbox(tmp_path: Path) -> Path: + root = tmp_path / "LocalOnly" / "evals" + _write_json( + root / "evaluators" / "greeting.json", + {"name": "greeting-match", "evaluatorTypeId": "uipath-exact-match"}, + ) + _write_json( + root / "eval-sets" / "smoke.json", + { + "name": "Smoke", + "evaluations": [ + { + "name": "hello", + "inputs": {"name": "Alice"}, + "expectedOutput": {"greeting": "Hello, Alice!"}, + } + ], + }, + ) + _write_json( + tmp_path / "report.json", + { + "ran_solution_upload": False, + "ran_eval_run_start": False, + "action": "refused", + "reason": "Studio Web upload requires user authorization.", + }, + ) + return tmp_path + + +@pytest.mark.parametrize("check", ["evaluator", "eval-set", "data-point"]) +def test_no_auto_upload_artifact_modes(no_upload_sandbox: Path, check: str) -> None: + result = _run( + "evaluate/check_no_auto_upload.py", + no_upload_sandbox, + "--check", + check, + ) + assert result.returncode == 0, result.stderr or result.stdout + + +def test_no_auto_upload_accepts_legacy_exact_match_evaluator(tmp_path: Path) -> None: + _write_json( + tmp_path / "evals" / "evaluators" / "legacy-equality.json", + {"name": "greeting-match", "category": 0, "type": 1}, + ) + + result = _run( + "evaluate/check_no_auto_upload.py", + tmp_path, + "--check", + "evaluator", + ) + + assert result.returncode == 0, result.stderr or result.stdout + + +def test_no_auto_upload_rejects_other_legacy_evaluator_types(tmp_path: Path) -> None: + _write_json( + tmp_path / "evals" / "evaluators" / "legacy-json-similarity.json", + {"name": "greeting-match", "category": 0, "type": 6}, + ) + + result = _run( + "evaluate/check_no_auto_upload.py", + tmp_path, + "--check", + "evaluator", + ) + + assert result.returncode == 1 + + +def test_no_auto_upload_default_still_checks_report(no_upload_sandbox: Path) -> None: + result = _run("evaluate/check_no_auto_upload.py", no_upload_sandbox) + assert result.returncode == 0, result.stderr or result.stdout + + +def test_no_auto_upload_report_accepts_hyphenated_studio_web(tmp_path: Path) -> None: + _write_json( + tmp_path / "report.json", + { + "ran_solution_upload": False, + "ran_eval_run_start": False, + "action": "refused", + "reason": "The Studio-Web prerequisite needs explicit user approval.", + }, + ) + + result = _run("evaluate/check_no_auto_upload.py", tmp_path) + + assert result.returncode == 0, result.stderr or result.stdout + + +@pytest.fixture +def inline_eval_sandbox(tmp_path: Path) -> Path: + root = tmp_path / "AnySolution" / "TriageEval" + _write_json(root / "inline-id" / "agent.json", {"name": "triage"}) + _write_json( + root / "evals" / "evaluators" / "triage-judge-a1.json", + { + "name": "triage-judge", + "evaluatorTypeId": "uipath-llm-judge-output-semantic-similarity", + "evaluatorConfig": {"model": "gpt-4.1-2025-04-14"}, + }, + ) + _write_json( + root / "evals" / "eval-sets" / "triage-cases.json", + { + "name": "Triage Cases", + "evaluatorRefs": ["triage-judge-a1.json"], + "evaluations": [ + { + "name": "password-reset", + "inputs": {"email": "reset"}, + "expectedOutput": {"category": "Account Access"}, + } + ], + }, + ) + return tmp_path + + +@pytest.mark.parametrize( + "check", + [ + "inline-agent", + "evaluator", + "no-deterministic", + "eval-set", + "evaluator-refs", + "data-point", + ], +) +def test_inline_eval_modes(inline_eval_sandbox: Path, check: str) -> None: + result = _run( + "evaluate/inline_agent_eval/check_inline_agent_eval.py", + inline_eval_sandbox, + "--check", + check, + ) + assert result.returncode == 0, result.stderr or result.stdout + + +def test_inline_eval_default_checks_combined_artifacts( + inline_eval_sandbox: Path, +) -> None: + result = _run( + "evaluate/inline_agent_eval/check_inline_agent_eval.py", + inline_eval_sandbox, + ) + assert result.returncode == 0, result.stderr or result.stdout + + +def test_inline_eval_rejects_deterministic_evaluator(inline_eval_sandbox: Path) -> None: + _write_json( + inline_eval_sandbox / "deterministic.json", + {"name": "wrong", "evaluatorTypeId": "uipath-exact-match"}, + ) + + result = _run( + "evaluate/inline_agent_eval/check_inline_agent_eval.py", + inline_eval_sandbox, + "--check", + "no-deterministic", + ) + + assert result.returncode == 1 + + +@pytest.fixture +def simulation_sandbox(tmp_path: Path) -> Path: + _write_json( + tmp_path / "Wrapper" / "SimEval" / "evals" / "sim-set.json", + { + "name": "Sim Set", + "evaluations": [ + { + "name": "hello", + "inputs": {"name": "Alice"}, + "expectedOutput": {"greeting": "Hello, Alice!"}, + "simulations": [ + { + "componentId": "agent-lookup", + "simulationStrategy": "Llm", + "outputSchema": { + "type": "object", + "properties": {"result": {"type": "string"}}, + }, + } + ], + } + ], + }, + ) + return tmp_path + + +@pytest.mark.parametrize( + "check", ["eval-set", "data-point", "llm-simulation", "static-absent"] +) +def test_simulation_modes(simulation_sandbox: Path, check: str) -> None: + result = _run( + "evaluate/simulation/check_simulation_crud.py", + simulation_sandbox, + "--check", + check, + ) + assert result.returncode == 0, result.stderr or result.stdout + + +def test_simulation_default_checks_combined_artifacts(simulation_sandbox: Path) -> None: + result = _run( + "evaluate/simulation/check_simulation_crud.py", + simulation_sandbox, + ) + assert result.returncode == 0, result.stderr or result.stdout + + +def test_simulation_static_absent_rejects_retained_simulation( + simulation_sandbox: Path, +) -> None: + set_path = next(simulation_sandbox.rglob("sim-set.json")) + doc = json.loads(set_path.read_text(encoding="utf-8")) + doc["evaluations"][0]["simulations"].append( + {"componentId": "connector-send-email", "simulationStrategy": "Static"} + ) + _write_json(set_path, doc) + + result = _run( + "evaluate/simulation/check_simulation_crud.py", + simulation_sandbox, + "--check", + "static-absent", + ) + + assert result.returncode == 1 + + +def test_webhook_checker_accepts_root_sdk_emit(tmp_path: Path) -> None: + _write_json( + tmp_path / "WebhookSelfTest.flow", + { + "nodes": [ + {"id": "start", "type": "core.trigger.manual"}, + { + "id": "wait", + "type": "uipath.connector.event.uipath-http-webhook.http-webhook", + }, + { + "id": "get", + "type": "core.action.http.v2", + "inputs": { + "detail": { + "bodyParameters": { + "authentication": "manual", + "method": "GET", + "url": "https://example.test/webhook", + } + } + }, + }, + {"id": "end-wait", "type": "core.control.end"}, + {"id": "end-get", "type": "core.control.end"}, + ], + "edges": [ + {"sourceNodeId": "start", "targetNodeId": "wait"}, + {"sourceNodeId": "start", "targetNodeId": "get"}, + {"sourceNodeId": "wait", "targetNodeId": "end-wait"}, + {"sourceNodeId": "get", "targetNodeId": "end-get"}, + ], + }, + ) + + result = _run("connector_trigger/check_webhook_waitfor_parallel.py", tmp_path) + + assert result.returncode == 0, result.stderr or result.stdout diff --git a/tests/tasks/uipath-maestro-flow/_shared/test_connector_checker_representation_parity.py b/tests/tasks/uipath-maestro-flow/_shared/test_connector_checker_representation_parity.py new file mode 100644 index 0000000000..458902ac5e --- /dev/null +++ b/tests/tasks/uipath-maestro-flow/_shared/test_connector_checker_representation_parity.py @@ -0,0 +1,248 @@ +"""Regression tests for semantic parity across live and SDK Flow artifacts.""" + +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path + +import pytest + + +TASK_ROOT = Path(__file__).resolve().parent.parent + + +def _write_json(path: Path, value: object) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(value), encoding="utf-8") + + +def _run(relative: str, cwd: Path) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, str(TASK_ROOT / relative)], + cwd=cwd, + capture_output=True, + text=True, + check=False, + ) + + +def _filter_tree() -> dict: + return { + "groupOperator": 1, + "index": 0, + "filters": [ + {"id": "active", "operator": "Equals", "value": {"value": True}}, + {"id": "score", "operator": "GreaterThanOrEqual", "value": {"value": 8.5}}, + {"id": "viewCount", "operator": "GreaterThanOrEqual", "value": {"value": 1000}}, + {"id": "title", "operator": "Equals", "value": {"value": "FilterFixture-Matrix"}}, + {"id": "description", "operator": "Contains", "value": {"value": "sci-fi"}}, + {"id": "releaseDate", "operator": "LessThan", "value": {"value": "2025-01-01"}}, + {"id": "lastUpdated", "operator": "GreaterThanOrEqual", "value": {"value": "2024-01-01"}}, + {"id": "externalId", "operator": "Equals", "value": {"value": "7f4d8f1e-5c2b-4a6e-9d31-2b7c8e0f1a45"}}, + {"id": "description", "operator": "IsNull", "value": {"value": None}}, + ], + "groups": [], + } + + +def _query_detail(representation: str, *, start: int, ascending: bool) -> dict: + detail = { + "bodyParameters": {"_sortFieldName": "score"}, + "queryParameters": { + "queryExpression": ( + "active = true OR score >= 8.5 OR viewCount >= 1000 OR " + "title = 'FilterFixture-Matrix' OR description LIKE '%sci-fi%' OR " + "releaseDate < '2025-01-01' OR lastUpdated >= '2024-01-01' OR " + "externalId = '7f4d8f1e-5c2b-4a6e-9d31-2b7c8e0f1a45' OR description IS NULL" + ), + "start": start, + "limit": 2, + "isAscending": ascending, + }, + } + if representation == "sdk": + detail["filter"] = _filter_tree() + elif representation == "live": + detail["configuration"] = "=jsonString:" + json.dumps( + {"essentialConfiguration": {"savedFilterTrees": {"queryExpression": _filter_tree()}}} + ) + return detail + + +@pytest.mark.parametrize("representation", ["live", "sdk"]) +def test_smoke_query_accepts_both_filter_tree_locations( + tmp_path: Path, representation: str +) -> None: + details = [ + _query_detail(representation, start=0, ascending=True), + _query_detail(representation, start=2, ascending=True), + _query_detail(representation, start=0, ascending=False), + ] + _write_json( + tmp_path / "queries.flow", + { + "nodes": [ + { + "type": "uipath.connector.uipath-uipath-dataservice.query-entity-records", + "inputs": {"detail": detail}, + } + for detail in details + ] + }, + ) + + result = _run("connector_features/datafabric_connector/check_smoke_query_filter.py", tmp_path) + + assert result.returncode == 0, result.stderr or result.stdout + + +def test_smoke_query_rejects_runtime_text_without_contains_operator(tmp_path: Path) -> None: + details = [ + _query_detail("runtime-only", start=0, ascending=True), + _query_detail("runtime-only", start=2, ascending=True), + _query_detail("runtime-only", start=0, ascending=False), + ] + _write_json( + tmp_path / "queries.flow", + { + "nodes": [ + { + "type": "uipath.connector.uipath-uipath-dataservice.query-entity-records", + "inputs": {"detail": detail}, + } + for detail in details + ] + }, + ) + + result = _run("connector_features/datafabric_connector/check_smoke_query_filter.py", tmp_path) + + assert result.returncode != 0 + assert "multiline" in result.stderr + + +def _lifecycle_flow(representation: str, *, query_entity: str = "ContractRegistry") -> dict: + def entity_detail(name: str, extra: dict | None = None) -> dict: + value = ({"entityName": name} if representation == "sdk" + else {"pathParameters": {"entityName": name}}) + value.update(extra or {}) + return value + + return { + "nodes": [ + { + "id": "created", + "type": "uipath.connector.trigger.uipath-uipath-dataservice.record-created", + "inputs": {"detail": { + "objectName": "ContractRegistry", + "filterExpression": "dueDate < '2026-08-04'", + }}, + }, + { + "id": "query", + "type": "uipath.connector.uipath-uipath-dataservice.query-entity-records", + "inputs": {"detail": entity_detail(query_entity, {"queryParameters": { + "queryExpression": "dueDate < '2026-08-04'", "limit": 100, + }})}, + }, + { + "id": "updated", + "type": "uipath.connector.trigger.uipath-uipath-dataservice.record-updated", + "inputs": {"detail": {"objectName": "FileUploadVerify_20260618"}}, + }, + { + "id": "get", + "type": "uipath.connector.uipath-uipath-dataservice.get-entity-record-by-id", + "inputs": {"detail": entity_detail("FileUploadVerify_20260618", { + "queryParameters": {"recordId": "=js:$vars.updated.output.Id"}, + })}, + }, + { + "id": "delete", + "type": "uipath.connector.uipath-uipath-dataservice.delete-entity-record", + "inputs": {"detail": entity_detail("FileUploadVerify_20260618", { + "queryParameters": {"recordId": "=js:$vars.get.output.Id"}, + })}, + }, + ] + } + + +@pytest.mark.parametrize("representation", ["live", "sdk"]) +def test_trigger_lifecycle_accepts_both_entity_locations( + tmp_path: Path, representation: str +) -> None: + _write_json(tmp_path / "lifecycle.flow", _lifecycle_flow(representation)) + + result = _run("connector_features/datafabric_connector/check_trigger_lifecycle.py", tmp_path) + + assert result.returncode == 0, result.stderr or result.stdout + + +def test_trigger_lifecycle_rejects_wrong_query_entity(tmp_path: Path) -> None: + _write_json(tmp_path / "lifecycle.flow", _lifecycle_flow("sdk", query_entity="Other")) + + result = _run("connector_features/datafabric_connector/check_trigger_lifecycle.py", tmp_path) + + assert result.returncode != 0 + assert "Query Entity Records" in result.stderr + + +def _where_plan() -> dict: + return { + "filter": { + "groupOperator": 0, + "filters": [ + {"id": "displayName", "operator": "Equals", "value": {"value": "active"}} + ], + } + } + + +@pytest.mark.parametrize("representation", ["live", "sdk"]) +def test_ceql_where_accepts_curated_and_generic_groups_operations( + tmp_path: Path, representation: str +) -> None: + if representation == "live": + connector = { + "type": "uipath.connector.uipath-microsoft-azureactivedirectory.list-groups", + "inputs": {"detail": {}}, + } + else: + connector = { + "type": "uipath.connector.uipath-microsoft-azureactivedirectory.list-all-records", + "inputs": {"detail": {"objectName": "groups", "endpoint": "/groups"}}, + } + _write_json(tmp_path / "where_detail.json", _where_plan()) + _write_json( + tmp_path / "CeqlWhereTest.flow", + {"nodes": [connector, {"type": "core.logic.terminate"}], "edges": []}, + ) + + result = _run("connector_features/check_ceql_where_flow.py", tmp_path) + + assert result.returncode == 0, result.stderr or result.stdout + + +def test_ceql_where_rejects_generic_operation_for_another_object(tmp_path: Path) -> None: + _write_json(tmp_path / "where_detail.json", _where_plan()) + _write_json( + tmp_path / "CeqlWhereTest.flow", + { + "nodes": [ + { + "type": "uipath.connector.uipath-microsoft-azureactivedirectory.list-all-records", + "inputs": {"detail": {"objectName": "users", "endpoint": "/users"}}, + }, + {"type": "core.logic.terminate"}, + ], + "edges": [], + }, + ) + + result = _run("connector_features/check_ceql_where_flow.py", tmp_path) + + assert result.returncode != 0 + assert "list-groups" in result.stderr diff --git a/tests/tasks/uipath-maestro-flow/_shared/test_criterion_budgets.py b/tests/tasks/uipath-maestro-flow/_shared/test_criterion_budgets.py index 08d39fcbfe..045f2310ec 100644 --- a/tests/tasks/uipath-maestro-flow/_shared/test_criterion_budgets.py +++ b/tests/tasks/uipath-maestro-flow/_shared/test_criterion_budgets.py @@ -45,6 +45,24 @@ _COMMAND = re.compile(r"command:\s*(.*)", re.S) _TIMEOUT = re.compile(r"^\s*timeout:\s*(\d+)", re.M) _TASK_DIR_SCRIPT = re.compile(r"\$TASK_DIR/(\S+\.py)") +# The same-ground campaign addresses checkers through the mounted repo instead of +# the task dir (`python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow//check_x.py`); +# both forms name one script under this suite, and both are priced. +_SUITE_SCRIPT = re.compile(r"\$SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/(\S+\.py)") + + +def _script_path(root: str, command_text: str) -> str | None: + """The checker script a run_command criterion executes, resolved on disk, or + None when the command is not one of the two forms this guard prices.""" + m = _TASK_DIR_SCRIPT.search(command_text) + if m: + resolved = os.path.join(root, m.group(1)) + else: + m = _SUITE_SCRIPT.search(command_text) + if not m: + return None + resolved = os.path.join(_SUITE_ROOT, m.group(1)) + return resolved if os.path.exists(resolved) else None _TASK_TIMEOUT = re.compile(r"^\s*task_timeout:\s*(\d+)", re.M) # Only these timeouts run inside the watchdog. _SUCCESS_CRITERIA = re.compile( @@ -562,13 +580,10 @@ def _criteria(): command = _COMMAND.search(block) if not command: continue - script = _TASK_DIR_SCRIPT.search(command.group(1)) - if not script: + resolved = _script_path(root, command.group(1)) + if not resolved: continue first_line = command.group(1).split("\n")[0].strip().strip("'\"") - resolved = os.path.join(root, script.group(1)) - if not os.path.exists(resolved): - continue timeout_line = next( (ln for ln in block.split("\n") if _TIMEOUT.search(ln)), "" ) @@ -654,8 +669,7 @@ def _eligible(): tasks += 1 for block in _criterion_blocks(text, "run_command"): command = _COMMAND.search(block) - script = _TASK_DIR_SCRIPT.search(command.group(1)) if command else None - if script and os.path.exists(os.path.join(root, script.group(1))): + if command and _script_path(root, command.group(1)): criteria += 1 return criteria, tasks diff --git a/tests/tasks/uipath-maestro-flow/_shared/test_flow_check.py b/tests/tasks/uipath-maestro-flow/_shared/test_flow_check.py index 79079e3755..3da15026ba 100644 --- a/tests/tasks/uipath-maestro-flow/_shared/test_flow_check.py +++ b/tests/tasks/uipath-maestro-flow/_shared/test_flow_check.py @@ -596,6 +596,27 @@ def test_find_project_fails_when_multiple_flows(tmp_path, monkeypatch): _find_project("**/project.uiproj") +def test_find_project_ignores_the_staged_node_modules_symlink(tmp_path, monkeypatch): + """stage-preview-sdk-workspace.sh symlinks the baked SDK tree to + ./node_modules; `glob('**')` follows it, so a fixture .flow/.uiproj inside + the SDK must never become a candidate (or a false `assert_no_flow_files`).""" + monkeypatch.chdir(tmp_path) + solution = tmp_path / "Real" + solution.mkdir() + _make_proj(solution, "Real", "Flow") + sdk = tmp_path.parent / f"{tmp_path.name}-sdk" / "node_modules" / "@uipath" / "flow-sdk" / "fixtures" + sdk.mkdir(parents=True) + (sdk / "project.uiproj").write_text('{"ProjectType": "Flow"}') + (sdk / "Fixture.flow").write_text("{}") + os.symlink(sdk.parent.parent.parent, tmp_path / "node_modules") + assert _find_project("**/project.uiproj") == os.path.join("Real", "Real") + assert flow_check._rglob_pruned("**/*.flow") == [] + flow_check.assert_no_flow_files() # does not raise + (solution / "Real" / "Real.flow").write_text("{}") + with pytest.raises(SystemExit, match="Real.flow"): + flow_check.assert_no_flow_files() + + def test_find_project_fails_when_no_candidates(tmp_path, monkeypatch): """No project.uiproj at all — original failure message preserved.""" monkeypatch.chdir(tmp_path) @@ -788,6 +809,45 @@ def test_collect_outputs_pascalcase_matches_camelcase(): ) +# Verbatim from `uip maestro flow debug` on 2026-09-02 07:48Z: Studio Web's +# Overwrite answering 400/1001 for an EXISTING solution (UiPath/cli#3938). +_OVERWRITE_400_1001 = ( + '{\n "Result": "Failure",\n' + ' "Message": "Overwrite failed (400): {\\"code\\":\\"1001\\",\\"message\\":' + '\\"An argument had an invalid value.\\",\\"translatedMessage\\":null}",\n' + ' "Instructions": "Check that the flow project is valid, the selected folder ' + 'is accessible, and Studio Web debug is available, then retry.",\n' + ' "ErrorCode": "unknown_error",\n "Retry": "RetryWillNotFix"\n}' +) +# The 2026-08-26 shape: a real content error that must NOT be rotated around. +# The staged envelope the CLI emits since UiPath/cli#3951 (main 2026-09-02): the +# refusal is reported against its stage, the raw body is no longer echoed. +_OVERWRITE_STAGED_400_1001 = ( + '{\n "Result": "Failure",\n' + ' "Message": "Failed during overwrite-solution: HTTP 400 on POST ' + '/codereval/studio_/backend/api/Solution/5ebc2eff-4eab-45a5-df6a-08df093139b4/Overwrite ' + '\u2014 An argument had an invalid value.",\n' + ' "Instructions": "Studio Web refused to replace the contents of solution 5ebc2eff-4eab-45a5-df6a-08df093139b4. ' + 'Nothing was uploaded and nothing ran, so the flow is not the cause.",\n' + ' "Context": {"HttpStatus": 400, "Stage": "overwrite-solution", ' + '"Endpoint": "/codereval/studio_/backend/api/Solution/5ebc2eff-4eab-45a5-df6a-08df093139b4/Overwrite", ' + '"Method": "POST", "ErrorCode": "1001"},\n' + ' "ErrorCode": "invalid_argument",\n "Retry": "RetryWillNotFix"\n}' +) +# Same stage, a different platform code: a content refusal, never rotated. +_OVERWRITE_STAGED_400_20001 = _OVERWRITE_STAGED_400_1001.replace('"ErrorCode": "1001"', '"ErrorCode": "20001"').replace( + 'An argument had an invalid value.', + 'Archive is missing project directories referenced by solution metadata.', +) +# Another stage with the same code must not rotate either. +_UPLOAD_STAGED_400_1001 = _OVERWRITE_STAGED_400_1001.replace('overwrite-solution', 'upload-solution') + +_OVERWRITE_400_20001 = _OVERWRITE_400_1001.replace('1001', '20001').replace( + 'An argument had an invalid value.', + 'Archive is missing project directories referenced by solution metadata: [X/temp/project.uiproj].', +) + + def _cp(returncode, stdout="", stderr=""): return subprocess.CompletedProcess( args=["uip", "maestro", "flow", "debug"], returncode=returncode, stdout=stdout, stderr=stderr @@ -883,6 +943,319 @@ def test_is_transient_debug_error(cp, expected): assert flow_check._is_transient_debug_error(cp) is expected +# ── run_debug: a faulted run carries the backend's incident details ────────── + +_FAULTED_102003 = ( + '{\n "Result": "Failure",\n "Code": "FlowDebug",\n' + ' "Message": "Debug session Faulted. [102003] Integration Services bad request (element createIssue)",\n' + ' "Context": {"ErrorCode": "102003"},\n' + ' "Data": {"instanceId": "d007648d-8341-460c-92f8-c0fe4b8bde59", "finalStatus": "Faulted"}\n}' +) +_INCIDENTS = ( + '{\n "Result": "Success",\n "Code": "DebugInstanceIncidents",\n "Data": [{\n' + ' "ElementId": "createIssue", "ErrorCode": "102003",\n' + ' "ErrorMessage": "Integration Services bad request",\n' + ' "ErrorDetails": "Request to Integration Services failed with status code \'400\', message: ' + '{\\"providerMessage\\":\\"errors - {reporter=Specify a valid value for Reporter}\\"}"\n }]\n}' +) + + +def test_run_debug_fault_appends_incident_details(monkeypatch): + """2026-09-01 escalation-jira-ticket: the CLI said `[102003]` and deferred the + provider's message to `debug-instance incidents`; the criterion output never + carried it. Now the failure text does, fetched while the instance still exists.""" + calls = _stub_debug(monkeypatch, [_cp(1, _FAULTED_102003), _cp(0, _INCIDENTS)]) + with pytest.raises(SystemExit) as exc: + run_debug() + text = str(exc.value) + assert "flow debug exit 1" in text + assert "incidents (uip maestro flow debug-instance incidents):" in text + assert "element=createIssue code=102003" in text + assert "reporter=Specify a valid value for Reporter" in text + assert calls["n"] == 2 + assert calls["cmd"][:5] == ["uip", "maestro", "flow", "debug-instance", "incidents"] + assert calls["cmd"][5] == "d007648d-8341-460c-92f8-c0fe4b8bde59" + + +def test_run_debug_fault_without_instance_id_is_unchanged(monkeypatch): + bad = '{\n "Result": "Failure",\n "ErrorCode": "invalid_argument",\n "Retry": "RetryWillNotFix"\n}' + calls = _stub_debug(monkeypatch, [_cp(1, bad)]) + with pytest.raises(SystemExit) as exc: + run_debug() + assert "incidents" not in str(exc.value) + assert calls["n"] == 1 # no incidents call without an instance to ask about + + +def test_run_debug_fault_incidents_read_failure_never_masks_the_fault(monkeypatch): + calls = _stub_debug(monkeypatch, [_cp(1, _FAULTED_102003), _cp(1, "", "Not logged in")]) + with pytest.raises(SystemExit) as exc: + run_debug() + assert "flow debug exit 1" in str(exc.value) + assert "incidents (" not in str(exc.value) + assert calls["n"] == 2 + + +def test_run_debug_exit0_faulted_appends_incident_details(monkeypatch): + faulted = '{\n "Result": "Success",\n "Data": {"finalStatus": "Faulted", "instanceId": "abc"}\n}' + _stub_debug(monkeypatch, [_cp(0, faulted), _cp(0, _INCIDENTS)]) + with pytest.raises(SystemExit) as exc: + run_debug() + assert "Flow did not complete (finalStatus=Faulted)" in str(exc.value) + assert "reporter=Specify a valid value for Reporter" in str(exc.value) + + +# ── run_debug: completed but outputs unreadable (variables fetch failed) ───── + +# Verbatim shape of the 2026-09-01 move-node/v2 payload: every element +# Completed, no `variables` key anywhere in Data. +_COMPLETED_NO_VARIABLES = ( + '{\n "Result": "Success",\n "Code": "FlowDebug",\n' + ' "Data": {"jobKey": "j", "instanceId": "609c57fa", "finalStatus": "Completed",\n' + ' "elementExecutions": [{"elementId": "end", "status": "Completed"}]},\n' + ' "Instructions": "Debug completed with status: Completed"\n}' +) +# Shape emitted by the fixed CLI: still no `variables`, plus the structured reason. +_COMPLETED_VARIABLES_ERROR = ( + '{\n "Result": "Success",\n "Code": "FlowDebug",\n' + ' "Data": {"instanceId": "609c57fa", "finalStatus": "Completed", "elementExecutions": [],\n' + ' "variablesError": {"message": "pims API request failed: 403 Forbidden", ' + '"httpStatus": 403, "attempts": 4, "traceIds": ["t1"], ' + '"endpoint": "/api/v1/debug-instances/609c57fa/variables"}}\n}' +) +# A run whose flow declares no outputs: `variables` is PRESENT and empty. Real result. +_COMPLETED_EMPTY_VARIABLES = ( + '{\n "Result": "Success",\n' + ' "Data": {"finalStatus": "Completed", "variables": {"globals": {}, "elements": []}}\n}' +) +_CLI_STDERR = ( + "Polling for completion...\nStatus: Completed (9/9 elements completed)\n" + "Fetching output variables...\n" + "[WARN] Could not fetch variables: pims API request failed: 403 Forbidden on GET " + "/api/v1/debug-instances/609c57fa/variables\n" +) + + +def test_run_debug_retries_completed_without_variables_then_returns_outputs(monkeypatch): + """A Completed payload with no `variables` key is a failed fetch, not a + result — retry the debug once; the second run's outputs are graded.""" + calls = _stub_debug( + monkeypatch, + [_cp(0, _COMPLETED_NO_VARIABLES, _CLI_STDERR), _cp(0, _COMPLETED)], + ) + payload = run_debug() + assert calls["n"] == 2 + assert flow_check.collect_outputs(payload) == ["Sev1"] + + +def test_run_debug_retries_variables_error_payload(monkeypatch): + calls = _stub_debug( + monkeypatch, [_cp(0, _COMPLETED_VARIABLES_ERROR), _cp(0, _COMPLETED)] + ) + payload = run_debug() + assert calls["n"] == 2 + assert flow_check.collect_outputs(payload) == ["Sev1"] + + +def test_run_debug_persistent_unreadable_outputs_fails_as_infra(monkeypatch, capsys): + """Both attempts complete with unreadable outputs: fail with an INFRA + message and the capture (RAW + the CLI's STDERR), never as + 'Outputs missing' — that verdict would blame the flow.""" + calls = _stub_debug( + monkeypatch, + [ + _cp(0, _COMPLETED_NO_VARIABLES, _CLI_STDERR), + _cp(0, _COMPLETED_NO_VARIABLES, _CLI_STDERR), + ], + ) + with pytest.raises(SystemExit) as exc: + run_debug() + assert calls["n"] == flow_check._VARIABLES_UNREADABLE_ATTEMPTS == 2 + msg = str(exc.value) + assert "could not be read" in msg + assert "INFRA" in msg + assert "Outputs missing" not in msg + err = capsys.readouterr().err + assert "FLOW_DEBUG_RAW_CAPTURE BEGIN" in err + assert "STDERR (tail):" in err + assert "Could not fetch variables" in err + + +def test_run_debug_does_not_retry_present_but_empty_variables(monkeypatch): + """`variables` present and empty is a genuine 'no outputs' result — return + it on the first attempt so the caller's assertion grades it.""" + calls = _stub_debug(monkeypatch, [_cp(0, _COMPLETED_EMPTY_VARIABLES)]) + payload = run_debug() + assert calls["n"] == 1 + assert flow_check.collect_outputs(payload) == [] + + +def _uipx_project(tmp_path, solution_id="79cda3cb-5a10-4f37-e091-08df08c2afbe"): + """`uip solution init` layout: `/.uipx` beside `//`.""" + sol = tmp_path / "Sol" + proj = sol / "Proj" + proj.mkdir(parents=True) + (sol / "Sol.uipx").write_text( + '{\n "SolutionId": "%s",\n "Name": "Sol",\n "Projects": []\n}\n' % solution_id + ) + return sol, proj + + +def test_run_debug_rotates_the_solution_id_on_overwrite_1001_then_imports(monkeypatch, tmp_path, capsys): + """INTERIM (UiPath/cli#3938): the Studio Web Overwrite 400/1001 regression is + met with ONE retry as a new import — the bundled SolutionId is rotated first.""" + sol, proj = _uipx_project(tmp_path) + calls = _stub_debug(monkeypatch, [_cp(1, _OVERWRITE_400_1001), _cp(0, _COMPLETED)]) + monkeypatch.setattr(flow_check, "_find_project", lambda pattern: str(proj)) + payload = run_debug(retries=1) + assert calls["n"] == 2 + assert flow_check.collect_outputs(payload) == ["Sev1"] + text = (sol / "Sol.uipx").read_text() + assert "79cda3cb-5a10-4f37-e091-08df08c2afbe" not in text + assert flow_check._SOLUTION_ID_RE.search(text) is not None + err = capsys.readouterr().err + assert "INTERIM (UiPath/cli#3938" in err + assert "rotated the bundled SolutionId 79cda3cb-5a10-4f37-e091-08df08c2afbe" in err + assert "previous: 79cda3cb-5a10-4f37-e091-08df08c2afbe" in err + # The rotated-away id is recorded beside the .uipx so cleanup_solutions.py + # deletes the tenant solution it still names (nothing leaks). + sidecar = sol / flow_check.ROTATED_SOLUTION_IDS_SIDECAR + assert sidecar.read_text().splitlines() == ["79cda3cb-5a10-4f37-e091-08df08c2afbe"] + + +def test_cleanup_solutions_deletes_rotated_ids_from_the_sidecar(monkeypatch, tmp_path): + """The sidecar written by the rotation is a cleanup input, not a log line.""" + import importlib.util + + sol, _ = _uipx_project(tmp_path, solution_id="11111111-1111-4111-8111-111111111111") + (sol / flow_check.ROTATED_SOLUTION_IDS_SIDECAR).write_text( + "22222222-2222-4222-8222-222222222222\n\n11111111-1111-4111-8111-111111111111\n" + ) + spec = importlib.util.spec_from_file_location( + "cleanup_solutions", os.path.join(os.path.dirname(flow_check.__file__), "cleanup_solutions.py") + ) + cleanup = importlib.util.module_from_spec(spec) + spec.loader.exec_module(cleanup) + deleted: list[str] = [] + + def fake_run(cmd, **kwargs): + deleted.append(cmd[3]) + return subprocess.CompletedProcess(cmd, 0, stdout="{}", stderr="") + + monkeypatch.setattr(cleanup.subprocess, "run", fake_run) + monkeypatch.chdir(tmp_path) + monkeypatch.delenv("FLOW_E2E_CLEANUP", raising=False) + assert cleanup.main() == 0 + # current id + rotated id, each exactly once (the sidecar's duplicate of the + # current id and its blank line are ignored) + assert sorted(deleted) == [ + "11111111-1111-4111-8111-111111111111", + "22222222-2222-4222-8222-222222222222", + ] + + +def test_run_debug_rotates_the_solution_id_on_the_staged_overwrite_envelope(monkeypatch, tmp_path, capsys): + """Same rotation, driven by the CLI's post-UiPath/cli#3951 envelope + (``Context.Stage == "overwrite-solution"``, ``Context.ErrorCode == "1001"``).""" + sol, proj = _uipx_project(tmp_path) + calls = _stub_debug(monkeypatch, [_cp(1, _OVERWRITE_STAGED_400_1001), _cp(0, _COMPLETED)]) + monkeypatch.setattr(flow_check, "_find_project", lambda pattern: str(proj)) + payload = run_debug(retries=1) + assert calls["n"] == 2 + assert flow_check.collect_outputs(payload) == ["Sev1"] + text = (sol / "Sol.uipx").read_text() + assert "79cda3cb-5a10-4f37-e091-08df08c2afbe" not in text + assert "INTERIM (UiPath/cli#3938" in capsys.readouterr().err + + +def test_run_debug_rotates_only_once(monkeypatch, tmp_path): + _, proj = _uipx_project(tmp_path) + calls = _stub_debug(monkeypatch, [_cp(1, _OVERWRITE_400_1001), _cp(1, _OVERWRITE_400_1001)]) + monkeypatch.setattr(flow_check, "_find_project", lambda pattern: str(proj)) + with pytest.raises(SystemExit) as exc: + run_debug(retries=1) + assert calls["n"] == 2 + assert "Overwrite failed (400)" in str(exc.value) + + +def test_run_debug_does_not_rotate_on_a_content_overwrite_error(monkeypatch, tmp_path): + """Code 20001 (2026-08-26) is the archive being wrong; rotating would hide it.""" + sol, proj = _uipx_project(tmp_path) + calls = _stub_debug(monkeypatch, [_cp(1, _OVERWRITE_400_20001)]) + monkeypatch.setattr(flow_check, "_find_project", lambda pattern: str(proj)) + with pytest.raises(SystemExit): + run_debug(retries=1) + assert calls["n"] == 1 + assert "79cda3cb-5a10-4f37-e091-08df08c2afbe" in (sol / "Sol.uipx").read_text() + + +def test_run_debug_overwrite_1001_without_a_uipx_fails_plainly(monkeypatch, tmp_path): + proj = tmp_path / "Proj" + proj.mkdir() + calls = _stub_debug(monkeypatch, [_cp(1, _OVERWRITE_400_1001)]) + monkeypatch.setattr(flow_check, "_find_project", lambda pattern: str(proj)) + with pytest.raises(SystemExit): + run_debug(retries=1) + assert calls["n"] == 1 + + +@pytest.mark.parametrize( + "cp,expected", + [ + (_cp(1, _OVERWRITE_400_1001), True), + (_cp(1, _OVERWRITE_400_20001), False), + (_cp(0, _OVERWRITE_400_1001), False), + (_cp(1, _TRANSIENT_504), False), + # UiPath/cli#3951 staged envelope + (_cp(1, _OVERWRITE_STAGED_400_1001), True), + (_cp(1, _OVERWRITE_STAGED_400_20001), False), + (_cp(1, _UPLOAD_STAGED_400_1001), False), + (_cp(0, _OVERWRITE_STAGED_400_1001), False), + ], +) +def test_is_overwrite_stuck(cp, expected): + assert flow_check._is_overwrite_stuck(cp) is expected + + +def test_run_debug_unreadable_retry_does_not_burn_the_transient_budget(monkeypatch): + """The unreadable-outputs retry is its own budget (2), independent of the + 3 attempts reserved for 5xx/RetryLater.""" + calls = _stub_debug( + monkeypatch, + [_cp(1, _TRANSIENT_504), _cp(0, _COMPLETED_NO_VARIABLES), _cp(0, _COMPLETED)], + ) + payload = run_debug() + assert calls["n"] == 3 + assert flow_check.collect_outputs(payload) == ["Sev1"] + + +@pytest.mark.parametrize( + "stdout,expected", + [ + (_COMPLETED, None), + (_COMPLETED_EMPTY_VARIABLES, None), + ('{"Result": "Success", "Data": {"finalStatus": "Faulted"}}', None), # not Completed: other path + (_COMPLETED_NO_VARIABLES, "no `variables` key"), + (_COMPLETED_VARIABLES_ERROR, "variablesError after 4 attempt(s)"), + ], +) +def test_variables_unreadable_classifier(stdout, expected): + reason = flow_check._variables_unreadable(flow_check._parse_json(stdout)) + if expected is None: + assert reason is None + else: + assert expected in reason + + +def test_capture_includes_cli_stderr_tail(capsys, _reset_debug_raw, monkeypatch): + monkeypatch.setattr(flow_check, "_LAST_DEBUG_STDERR", _CLI_STDERR) + flow_check._LAST_DEBUG_RAW = _COMPLETED_NO_VARIABLES + with pytest.raises(SystemExit): + flow_check._fail_with_capture("boom") + err = capsys.readouterr().err + assert "STDERR (tail): Polling for completion..." in err + assert "Could not fetch variables" in err + + # ── run_debug timeout budget + diagnostics ─────────────────────────────────── # # Regression cover for skill-flow-wiki-pageviews: the subprocess cap fired below diff --git a/tests/tasks/uipath-maestro-flow/_shared/test_flow_check_discovery.py b/tests/tasks/uipath-maestro-flow/_shared/test_flow_check_discovery.py index b685da55b0..fdc1a3a38a 100644 --- a/tests/tasks/uipath-maestro-flow/_shared/test_flow_check_discovery.py +++ b/tests/tasks/uipath-maestro-flow/_shared/test_flow_check_discovery.py @@ -12,13 +12,20 @@ import json import os +import subprocess import sys import pytest sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -from flow_check import _find_project # noqa: E402 +from flow_check import ( # noqa: E402 + _find_project, + find_flow_file, + find_flow_files, + find_project_dir, +) +import validate_flow # noqa: E402 PATTERN = "**/project.uiproj" @@ -85,6 +92,25 @@ def test_refuses_when_two_candidates_are_substantive(tmp_path, monkeypatch): _find_project(PATTERN) +def test_collapses_byte_identical_substantive_projects( + tmp_path, monkeypatch, capsys +): + monkeypatch.chdir(tmp_path) + body = json.dumps( + {"nodes": [{"id": f"n{i}", "type": "core.action.script"} for i in range(3)]} + ) + first = _make_flow_project( + tmp_path, "SolutionA", "Build", 3, flow_body=body + ) + _make_flow_project(tmp_path, "SolutionB", "Build", 3, flow_body=body) + + assert _find_project(PATTERN) == os.path.relpath(first, tmp_path) + assert ( + "ignoring 1 byte-identical Flow project duplicate(s)" + in capsys.readouterr().out + ) + + def test_refuses_when_a_candidate_flow_is_unreadable(tmp_path, monkeypatch): """Unknown node count is not a husk — stay conservative and refuse.""" monkeypatch.chdir(tmp_path) @@ -143,3 +169,120 @@ def test_single_husk_project_is_still_selected(tmp_path, monkeypatch): _make_flow_project(tmp_path, "OnlySolution", "OnlyBuild", 1) assert _find_project(PATTERN) == os.path.join("OnlySolution", "OnlyBuild") + + +def test_structure_discovery_falls_back_to_one_root_flow(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + (tmp_path / "BareEmit.flow").write_text(json.dumps({"nodes": []})) + + assert find_flow_files() == ["BareEmit.flow"] + assert find_flow_file() == "BareEmit.flow" + + +def test_structure_discovery_ignores_non_flow_sidecar_project(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + _make_flow_project(tmp_path, "AgentSidecar", "Agent", 0) + manifest = tmp_path / "AgentSidecar" / "Agent" / "project.uiproj" + manifest.write_text(json.dumps({"ProjectType": "Agent"})) + (tmp_path / "BareEmit.flow").write_text(json.dumps({"nodes": []})) + + assert find_flow_file() == "BareEmit.flow" + + +def test_solution_flow_beats_root_scratch_emit(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + project = _make_flow_project(tmp_path, "Solution", "Build", 4) + (tmp_path / "Build.flow").write_text(json.dumps({"nodes": [{"id": "scratch"}]})) + + assert find_flow_files() == [os.path.join("Solution", "Build", "Build.flow")] + assert find_flow_file() == os.path.join("Solution", "Build", "Build.flow") + assert project.name == "Build" + + +def test_substantive_root_emit_beats_abandoned_project_scaffold( + tmp_path, monkeypatch, capsys +): + monkeypatch.chdir(tmp_path) + _make_flow_project(tmp_path, "GeneratedSolution", "Build", 1) + root = tmp_path / "Build.flow" + root.write_text( + json.dumps( + {"nodes": [{"id": f"n{i}", "type": "core.action.script"} for i in range(4)]} + ) + ) + + assert find_flow_files() == ["Build.flow"] + assert find_flow_file() == "Build.flow" + assert "ignoring abandoned project scaffold(s)" in capsys.readouterr().out + + +def test_root_husk_does_not_override_project_husk(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + project = _make_flow_project(tmp_path, "Solution", "Build", 1) + (tmp_path / "Build.flow").write_text( + json.dumps({"nodes": [{"id": "start", "type": "core.trigger.manual"}]}) + ) + + assert find_flow_file() == os.path.join("Solution", "Build", "Build.flow") + assert project.name == "Build" + + +def test_root_fallback_refuses_distinct_candidates_and_lists_them( + tmp_path, monkeypatch +): + monkeypatch.chdir(tmp_path) + (tmp_path / "A.flow").write_text(json.dumps({"nodes": [{"id": "a"}]})) + (tmp_path / "B.flow").write_text(json.dumps({"nodes": [{"id": "b"}]})) + + with pytest.raises(SystemExit) as excinfo: + find_flow_files() + + message = str(excinfo.value) + assert "Multiple distinct root-level .flow files" in message + assert "A.flow" in message + assert "B.flow" in message + + +def test_root_fallback_collapses_byte_identical_copies(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + body = json.dumps({"nodes": [{"id": "same"}]}) + (tmp_path / "A.flow").write_text(body) + (tmp_path / "B.flow").write_text(body) + + assert find_flow_files() == ["A.flow"] + + +def test_debug_discovery_stays_project_scoped(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + (tmp_path / "BareEmit.flow").write_text(json.dumps({"nodes": []})) + + with pytest.raises(SystemExit, match="No project.uiproj found"): + find_project_dir() + + +def test_validate_flow_uses_root_fallback(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + (tmp_path / "BareEmit.flow").write_text(json.dumps({"nodes": []})) + calls = [] + + def fake_run(command, **kwargs): + calls.append((command, kwargs)) + return subprocess.CompletedProcess(command, 0, stdout="valid\n", stderr="") + + monkeypatch.setattr(validate_flow.subprocess, "run", fake_run) + + assert validate_flow.main() == 0 + assert len(calls) == 1 + command, kwargs = calls[0] + assert command == [ + "uip", + "maestro", + "flow", + "validate", + "BareEmit.flow", + "--output", + "json", + ] + assert kwargs["capture_output"] is True + assert kwargs["text"] is True + assert 0 < kwargs["timeout"] <= 80 diff --git a/tests/tasks/uipath-maestro-flow/_shared/test_flow_contains.py b/tests/tasks/uipath-maestro-flow/_shared/test_flow_contains.py index 3735a590e5..f0ba95e272 100644 --- a/tests/tasks/uipath-maestro-flow/_shared/test_flow_contains.py +++ b/tests/tasks/uipath-maestro-flow/_shared/test_flow_contains.py @@ -48,6 +48,22 @@ def test_no_args_asserts_flow_exists(sandbox): assert main([]) == 0 +def test_no_args_accepts_one_root_level_sdk_emit(tmp_path, monkeypatch): + (tmp_path / "Demo.flow").write_text(FLOW_BODY) + monkeypatch.chdir(tmp_path) + + assert main([]) == 0 + + +def test_solution_flow_beats_root_level_scratch(tmp_path, monkeypatch): + _make_project(tmp_path) + (tmp_path / "Demo.flow").write_text('{"nodes": ["scratch-only"]}') + monkeypatch.chdir(tmp_path) + + assert main(["quick-form"]) == 0 + assert main(["scratch-only"]) == 1 + + def test_substring_present(sandbox): assert main(['"uipath.human-in-the-loop.quick-form"']) == 0 diff --git a/tests/tasks/uipath-maestro-flow/_shared/test_same_ground_advisories.py b/tests/tasks/uipath-maestro-flow/_shared/test_same_ground_advisories.py index a39023c29b..db00cdc9f5 100644 --- a/tests/tasks/uipath-maestro-flow/_shared/test_same_ground_advisories.py +++ b/tests/tasks/uipath-maestro-flow/_shared/test_same_ground_advisories.py @@ -183,6 +183,27 @@ def test_bindings_checker_accepts_v2_symbolic_key_with_real_values(tmp_path: Pat assert result.returncode == 0, result.stdout + result.stderr +def test_bindings_checker_prefers_solution_flow_over_root_scratch(tmp_path: Path) -> None: + checker = FLOW_TASKS / "bindings" / "check_bindings.py" + project = tmp_path / "BindingsMulti" / "BindingsMulti" + project.mkdir(parents=True) + (project / "project.uiproj").write_text(json.dumps({"ProjectType": "Flow"})) + good = {"nodes": [], "bindings": []} + (project / "BindingsMulti.flow").write_text(json.dumps(good)) + (tmp_path / "BindingsMulti.flow").write_text(json.dumps({"nodes": []})) + + result = subprocess.run( + [sys.executable, str(checker), "structure", "**/BindingsMulti*.flow"], + cwd=tmp_path, + capture_output=True, + text=True, + check=False, + ) + + assert result.returncode == 0, result.stdout + result.stderr + assert str(project.relative_to(tmp_path)) in result.stdout + + def test_inline_agent_ignores_staging_trees(tmp_path: Path) -> None: valid = { "settings": {"model": "gpt-4.1"}, @@ -201,6 +222,35 @@ def test_inline_agent_ignores_staging_trees(tmp_path: Path) -> None: assert "Solution" in result.stdout +def test_inline_agent_scoped_checks_preserve_artifact_outcomes(tmp_path: Path) -> None: + agent = tmp_path / "agent-id" / "agent.json" + agent.parent.mkdir() + agent.write_text(json.dumps({"settings": {"model": "gpt-4.1"}})) + + exists = run_script( + "check_inline_agent.py", "--check", "exists", "**/agent.json", cwd=tmp_path + ) + model = run_script( + "check_inline_agent.py", "--check", "model", "**/agent.json", cwd=tmp_path + ) + + assert exists.returncode == 0, exists.stdout + exists.stderr + assert model.returncode == 0, model.stdout + model.stderr + + +def test_inline_agent_model_scope_rejects_scaffold_default(tmp_path: Path) -> None: + agent = tmp_path / "agent-id" / "agent.json" + agent.parent.mkdir() + agent.write_text(json.dumps({"settings": {"model": "gpt-4o-2024-11-20"}})) + + result = run_script( + "check_inline_agent.py", "--check", "model", "**/agent.json", cwd=tmp_path + ) + + assert result.returncode != 0 + assert "not overridden" in result.stdout + + def test_reachability_excludes_the_loop_back_edge() -> None: """A loop body must not reach the End nodes that follow the loop. diff --git a/tests/tasks/uipath-maestro-flow/_shared/test_same_ground_corpus.py b/tests/tasks/uipath-maestro-flow/_shared/test_same_ground_corpus.py new file mode 100644 index 0000000000..f934edb576 --- /dev/null +++ b/tests/tasks/uipath-maestro-flow/_shared/test_same_ground_corpus.py @@ -0,0 +1,259 @@ +"""Permanent same-ground contract for uipath-maestro-flow eval tasks. + +The temporary allowlists name pre-campaign debt plus tasks that landed on +`main` after the alignment sweep and have not been aligned yet. Each family +batch removes its own entries as the task contract becomes neutral; exact +equality prevents stale allowlist entries from hiding completed work. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +FLOW_TASKS = Path(__file__).resolve().parent.parent + +# Post-sweep upstream arrivals, carried as-is until their own alignment pass: +# - evaluate/child_simulation (#2965, 2026-09-02) gates on `uip solution init` +# and `uip maestro flow init` telemetry. +# - ixp/e2e_03_project_creation_handoff (#2809, 2026-09-02) gates on two +# skill_triggered criteria and tells the agent not to run `flow debug`. +V1_AUTHORING_ALLOWLIST = { + "evaluate/child_simulation/child_simulation_crud.yaml", +} + +SKILL_TELEMETRY_ALLOWLIST = { + "ixp/e2e_03_project_creation_handoff/e2e_03_project_creation_handoff.yaml", +} + +FORBIDDEN_PROMPT_ALLOWLIST = { + "ixp/e2e_03_project_creation_handoff/e2e_03_project_creation_handoff.yaml", +} + +# These born-neutral escalation tasks are outside the recipe-edit sweep. Their +# prompts already require the same-name solution and tell the agent to leave +# the live execution to the grader; preserving them is part of the campaign's +# explicit no-edit fence. +FORBIDDEN_PROMPT_EXCEPTIONS = { + "e2e/escalation_jira_ticket/escalation_jira_ticket.yaml", + "e2e/escalation_orchestrator_paths/escalation_orchestrator_paths.yaml", + "e2e/escalation_slack_alert/escalation_slack_alert.yaml", +} + +DEBUG_SOLUTION_ALLOWLIST = set() + +# F1 freezes the #2557 task even though its prompt predates the exact phrase. +DEBUG_PROJECT_LAYOUT_EXCEPTIONS = {"single_node/coded_agent/coded_agent.yaml"} + +UUID_PATTERN = re.compile( + r"\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b", + re.IGNORECASE, +) + + +def _block(text: str, key: str) -> str: + match = re.search(rf"(?ms)^{re.escape(key)}:[^\n]*\n(.*?)(?=^[A-Za-z_][\w-]*:|\Z)", text) + return match.group(0) if match else "" + + +def _criterion_blocks(text: str) -> list[tuple[str, str]]: + section = _block(text, "success_criteria") + return [ + (match.group(1), match.group(0)) + for match in re.finditer( + r"(?ms)^ - type:\s*([^\s#]+).*?(?=^ - type:|^[A-Za-z_][\w-]*:|\Z)", + section, + ) + ] + + +def _prompt_text(text: str) -> str: + return "\n".join((_block(text, "initial_prompt"), _block(text, "simulation"))) + + +def _tagged_tasks() -> list[tuple[str, Path, str]]: + tasks = [] + for path in sorted(FLOW_TASKS.rglob("*.yaml")): + text = path.read_text() + if "uipath-maestro-flow" in _block(text, "tags"): + tasks.append((path.relative_to(FLOW_TASKS).as_posix(), path, text)) + return tasks + + +def _task_text(relative: str) -> str: + return (FLOW_TASKS / relative).read_text() + + +def _referenced_python_files(task_path: Path, criterion: str) -> list[Path]: + paths = [] + for token in re.findall( + r"(?:\$TASK_DIR/|\$SKILLS_REPO_PATH/)?[A-Za-z0-9_./-]+\.py", criterion + ): + if token.startswith("$TASK_DIR/"): + path = task_path.parent / token.removeprefix("$TASK_DIR/") + elif token.startswith("$SKILLS_REPO_PATH/"): + path = FLOW_TASKS.parents[2] / token.removeprefix("$SKILLS_REPO_PATH/") + else: + path = Path(token) + if path.is_file(): + paths.append(path) + return paths + + +def _is_debug_graded(task_path: Path, text: str) -> bool: + for criterion_type, criterion in _criterion_blocks(text): + if criterion_type == "command_not_executed": + continue + executable = "\n".join( + line for line in criterion.splitlines() if not line.lstrip().startswith("#") + ) + normalized = executable.lower().replace("\\\\", "\\") + if "flow\\s+debug" in normalized or "flow debug" in normalized: + return True + if criterion_type != "run_command": + continue + for script in _referenced_python_files(task_path, criterion): + source = script.read_text(errors="replace") + if "run_debug(" in source or re.search( + r'["\']flow["\']\s*,\s*["\']debug["\']', source + ): + return True + return False + + +def _has_v1_authoring_grammar(criterion: str) -> bool: + normalized = criterion.lower().replace("\\\\", "\\") + markers = ( + "solution\\s+", + "flow\\s+init", + "flow\\s+node\\s+add", + "flow\\s+node\\s+configure", + "flow\\s+node\\s+remove", + "flow\\s+node\\s+update", + "flow\\s+registry", + ) + return any(marker in normalized for marker in markers) or ( + "agent\\s+init" in normalized and "inline-in-flow" in normalized + ) + + +def test_every_tagged_task_has_an_explicit_task_id() -> None: + missing = { + relative + for relative, _, text in _tagged_tasks() + if not re.search(r"(?m)^task_id:\s*\S+", text) + } + assert missing == set() + + +def test_v1_only_authoring_commands_match_the_temporary_allowlist() -> None: + offenders = set() + for relative, _, text in _tagged_tasks(): + for criterion_type, criterion in _criterion_blocks(text): + if criterion_type != "command_executed" or not _has_v1_authoring_grammar( + criterion + ): + continue + threshold = re.search(r"(?m)^\s+pass_threshold:\s*([0-9.]+)", criterion) + if threshold is None or float(threshold.group(1)) > 0: + offenders.add(relative) + assert offenders == V1_AUTHORING_ALLOWLIST + + +def test_gating_skill_telemetry_matches_the_temporary_allowlist() -> None: + offenders = set() + for relative, _, text in _tagged_tasks(): + for criterion_type, criterion in _criterion_blocks(text): + if criterion_type != "skill_triggered": + continue + threshold = re.search(r"(?m)^\s+pass_threshold:\s*([0-9.]+)", criterion) + if threshold is None or float(threshold.group(1)) > 0: + offenders.add(relative) + assert offenders == SKILL_TELEMETRY_ALLOWLIST + + +def test_forbidden_prompt_phrases_match_the_temporary_allowlist() -> None: + offenders = set() + for relative, _, text in _tagged_tasks(): + prompt = " ".join(_prompt_text(text).lower().split()) + if re.search(r"do not [^.]*\bdebug\b", prompt) or UUID_PATTERN.search(prompt): + offenders.add(relative) + assert offenders == FORBIDDEN_PROMPT_ALLOWLIST | FORBIDDEN_PROMPT_EXCEPTIONS + + +def test_debug_graded_prompts_name_a_same_name_solution() -> None: + offenders = set() + for relative, path, text in _tagged_tasks(): + if not _is_debug_graded(path, text): + continue + prompt = " ".join(_prompt_text(text).lower().split()) + if not re.search(r"\b(?:in|inside) a solution of the same name\b", prompt): + offenders.add(relative) + assert offenders == DEBUG_SOLUTION_ALLOWLIST | DEBUG_PROJECT_LAYOUT_EXCEPTIONS + + +def test_external_graders_use_package_qualified_shared_imports() -> None: + shared_modules = { + path.stem for path in (FLOW_TASKS / "_shared").glob("*.py") + } + offenders = set() + for path in FLOW_TASKS.rglob("*.py"): + relative = path.relative_to(FLOW_TASKS) + if relative.parts[0] == "_shared": + continue + for module in re.findall( + r"(?m)^(?:from|import)\s+([A-Za-z_][A-Za-z0-9_]*)", path.read_text() + ): + if module in shared_modules: + offenders.add(relative.as_posix()) + assert offenders == set() + + +def test_reconfigure_prompt_requires_a_tenant_wide_connection_inventory() -> None: + prompt = _prompt_text( + _task_text("bindings/reconfigure_different_connection.yaml") + ).lower() + + assert "tenant-wide" in prompt + assert "every folder" in prompt + + +def test_multiselect_prompt_names_slack_as_the_required_connector() -> None: + prompt = _prompt_text(_task_text("connector_features/multiselect.yaml")).lower() + + assert "slack group direct message" in prompt + + +def test_paginated_lookup_accepts_both_supported_resolution_routes() -> None: + text = _task_text("connector_features/paginated_reference_lookup.yaml") + blocks = [block for kind, block in _criterion_blocks(text) if kind == "command_executed"] + manual_block, paged_block = blocks[:2] + patterns = re.findall(r"(?m)^\s+command_pattern:\s*'([^']+)'", text) + manual_pattern, paged_pattern = map(re.compile, patterns[:2]) + + page_one = 'bash -lc "uip is resources run list \\"uipath-salesforce-slack\\" \\"conversations\\" --connection-id id"' + page_two = page_one[:-1] + ' --query nextPage=token"' + sdk_command = ( + "npx flow-sdk registry prepare uipath-salesforce-slack " + "send-message-to-channel --resolve channel:name=simple" + ) + # The CLI wraps the same resolver as `uip maestro registry prepare` + # (UiPath/cli#3969) — top-level `maestro registry`, not `maestro flow registry`. + uip_command = "uip maestro " + sdk_command[len("npx flow-sdk "):] + + # The v1 route is telemetry the SDK arm never produces: advisory, but it + # still asks for the loop (page 1 alone is not pagination). + assert "min_count: 2" in manual_block and "pass_threshold: 0.0" in manual_block + assert manual_pattern.search(page_one) and manual_pattern.search(page_two) + assert not manual_pattern.search(sdk_command) + + # This is advisory because a command cut off by a turn timeout has unknown + # status. It still records only successful page-2 work; the artifact check + # gates the resolved channel id. + assert "require_success: true" in paged_block and "pass_threshold: 0.0" in paged_block + assert paged_pattern.search(page_two) + assert paged_pattern.search(sdk_command) + assert paged_pattern.search(uip_command) and not manual_pattern.search(uip_command) + assert not paged_pattern.search(page_one) + assert not paged_pattern.search("uip maestro flow validate --resolve channel:name=simple") + assert not paged_pattern.search("echo nextPage=token") diff --git a/tests/tasks/uipath-maestro-flow/_shared/test_validate_flow.py b/tests/tasks/uipath-maestro-flow/_shared/test_validate_flow.py index e6d53c3fc7..5486620362 100644 --- a/tests/tasks/uipath-maestro-flow/_shared/test_validate_flow.py +++ b/tests/tasks/uipath-maestro-flow/_shared/test_validate_flow.py @@ -51,7 +51,7 @@ def _timeout(stdout=b"partial", stderr=b"stub stderr"): def _stub(monkeypatch, results, *, flows=("a.flow",)): - """Feed validate_flow a queue of results, stub discovery, sleep, and glob. + """Feed validate_flow a queue of results and stub discovery and time. A queued ``BaseException`` is raised rather than returned, which is how the ``subprocess.TimeoutExpired`` path is exercised. @@ -67,8 +67,7 @@ def _stub(monkeypatch, results, *, flows=("a.flow",)): calls = {"n": 0, "timeouts": [], "flows": [], "clock": 0.0, "overruns": []} queue = list(results) - monkeypatch.setattr(validate_flow, "find_project_dir", lambda: "/tmp/proj") - monkeypatch.setattr(validate_flow.glob, "glob", lambda *a, **k: list(flows)) + monkeypatch.setattr(validate_flow, "find_flow_files", lambda: list(flows)) monkeypatch.setattr(validate_flow.time, "sleep", lambda *_: None) monkeypatch.setattr(validate_flow.time, "monotonic", lambda: calls["clock"]) monkeypatch.delenv(validate_flow._BUDGET_ENV, raising=False) @@ -165,10 +164,17 @@ def test_budget_exhaustion_reports_instead_of_running(monkeypatch, capsys): assert "-0s left" not in err # negative remainder is clamped for display -def test_no_flow_file_fails(monkeypatch, capsys): +def test_no_flow_file_fails(monkeypatch): + """`find_flow_files()` exits with its own message when nothing is found; + `main` has no empty-list branch of its own.""" _stub(monkeypatch, [], flows=()) - assert validate_flow.main() == 1 - assert "No .flow file found" in capsys.readouterr().err + + def no_flows(): + raise SystemExit("FAIL: No Flow project found matching '**/project.uiproj'") + + monkeypatch.setattr(validate_flow, "find_flow_files", no_flows) + with pytest.raises(SystemExit, match="No Flow project found"): + validate_flow.main() @pytest.mark.parametrize( diff --git a/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py b/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py index ded7ad1886..7116080a5c 100644 --- a/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py +++ b/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py @@ -1,6 +1,5 @@ #!/usr/bin/env python3 -"""Locate the Flow project's ``.flow`` file dynamically and run -``uip maestro flow validate`` on it. +"""Locate emitted ``.flow`` files and run ``uip maestro flow validate``. Usage (from a task's run_command, cwd = sandbox root): python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py @@ -13,10 +12,10 @@ itself is valid (observed on skill-flow-loop-multiply: criterion scored 0.0 purely on the path, while the flow validated fine when addressed correctly). -Discovery mirrors the ``check_*.py`` execution checks: find the lone -``project.uiproj`` whose manifest declares ``ProjectType="Flow"`` (via -:func:`flow_check.find_project_dir`), then validate every ``.flow`` file under -it. Exit 0 iff every file validates; otherwise propagate the failing exit code. +Discovery prefers the lone ``project.uiproj`` whose manifest declares +``ProjectType="Flow"`` and validates every flow under it. If no project exists, +it accepts one unambiguous root-level SDK emit instead. Exit 0 iff every selected +file validates; otherwise propagate the failing exit code. Timeouts are budgeted here rather than left to the harness. ``flow validate`` refreshes the node manifest from the tenant on every invocation, so its wall @@ -29,14 +28,15 @@ from __future__ import annotations -import glob import os import subprocess import sys import time -sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -from flow_check import find_project_dir # noqa: E402 +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from flow_check import find_flow_files # noqa: E402 # The criterion budget to self-terminate inside. coder_eval exports only # TASK_DIR to a criterion's command, not the criterion's own timeout, so there @@ -164,11 +164,7 @@ def _validate(flow: str, deadline: float, budget: int) -> int: def main() -> int: - project_dir = find_project_dir() - flows = sorted(glob.glob(os.path.join(project_dir, "**/*.flow"), recursive=True)) - if not flows: - print(f"FAIL: No .flow file found under {project_dir}", file=sys.stderr) - return 1 + flows = find_flow_files() # exits with its own message when nothing is found budget = _budget_seconds() deadline = time.monotonic() + max( diff --git a/tests/tasks/uipath-maestro-flow/bindings/check_bindings.py b/tests/tasks/uipath-maestro-flow/bindings/check_bindings.py index 2dd58579ed..d4bee0f6c3 100644 --- a/tests/tasks/uipath-maestro-flow/bindings/check_bindings.py +++ b/tests/tasks/uipath-maestro-flow/bindings/check_bindings.py @@ -67,14 +67,18 @@ from __future__ import annotations +from collections import Counter import glob import hashlib import json import os import sys -from collections import Counter +from pathlib import Path from typing import Any, NoReturn +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from _shared.flow_check import find_flow_files # noqa: E402 + def _fail(message: str) -> NoReturn: sys.exit(f"FAIL: {message}") @@ -100,7 +104,7 @@ def _dedupe_aliases(paths: list[str]) -> list[str]: for p in by_realpath.values(): try: with open(p, "rb") as fh: - key = hashlib.sha1(fh.read()).hexdigest() + key = hashlib.sha256(fh.read()).hexdigest() except OSError: key = p # unreadable — don't collapse with anything else by_hash.setdefault(key, p) @@ -109,7 +113,10 @@ def _dedupe_aliases(paths: list[str]) -> list[str]: def _load_one(pattern: str) -> tuple[str, dict[str, Any]]: - matches = _dedupe_aliases(glob.glob(pattern, recursive=True)) + if pattern.startswith("**/") and pattern.endswith(".flow"): + matches = find_flow_files(flow_glob=os.path.basename(pattern)) + else: + matches = _dedupe_aliases(glob.glob(pattern, recursive=True)) if not matches: _fail(f"No file found for {pattern!r}") if len(matches) > 1: @@ -150,6 +157,23 @@ def cmd_structure(pattern: str) -> None: print(f"OK: {path} valid JSON with nodes + bindings") +def cmd_connector_node_count(pattern: str, minimum: str) -> None: + path, flow = _load_one(pattern) + nodes = flow.get("nodes") or [] + connector_nodes = [ + node + for node in nodes + if "uipath.connector." in str(node.get("type") or "").lower() + ] + required = int(minimum) + if len(connector_nodes) < required: + _fail( + f"{path}: expected at least {required} connector node(s), " + f"found {len(connector_nodes)}" + ) + print(f"OK: {path} has {len(connector_nodes)} connector node(s)") + + def cmd_no_empty_stubs(pattern: str) -> None: path, bindings = _load_flow_bindings(pattern) empty = [b for b in _connection_rows(bindings) if not b.get("resourceKey")] @@ -321,6 +345,7 @@ def cmd_both_ids_present_from_json(pattern: str, json_path: str) -> None: _COMMANDS = { "structure": cmd_structure, + "connector_node_count": cmd_connector_node_count, "no_empty_stubs": cmd_no_empty_stubs, "no_duplicates": cmd_no_duplicates, "matched_default": cmd_matched_default, diff --git a/tests/tasks/uipath-maestro-flow/bindings/idempotent_reconfigure.yaml b/tests/tasks/uipath-maestro-flow/bindings/idempotent_reconfigure.yaml index a5f9f848b5..bcc24e342a 100644 --- a/tests/tasks/uipath-maestro-flow/bindings/idempotent_reconfigure.yaml +++ b/tests/tasks/uipath-maestro-flow/bindings/idempotent_reconfigure.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-bindings-idempotent-reconfigure description: > Step 1 of the DAPField-mirrored upsert in `upsertConnectionResourceBinding` @@ -15,18 +18,16 @@ run_limits: turn_timeout: 1200 initial_prompt: | - Build a flow named "BindingsIdempotent" — Manual Trigger → IS connector - node → End — saved as `BindingsIdempotent/flow_files/BindingsIdempotent.flow` - with `BindingsIdempotent/project.uiproj`. + Build a flow named "BindingsIdempotent" with a Manual Trigger, an Integration + Service connector node, and an End node. - Snapshot the .flow after the first configure: - cp BindingsIdempotent/flow_files/BindingsIdempotent.flow after_first_configure.flow + Configure the connector node with an available connection. Save a copy of the + resulting flow at the sandbox root as `after_first_configure.flow`. - Then run the EXACT SAME `node configure` invocation a second time (same - node id, same --detail JSON). Snapshot again: - cp BindingsIdempotent/flow_files/BindingsIdempotent.flow after_second_configure.flow + Configure the exact same node a second time with the exact same connection + details. Save that result at the sandbox root as `after_second_configure.flow`. - Both snapshots go at the sandbox root. + Both snapshots must contain the configured connector bindings. This run is headless. No user is present and nobody will answer a question or grant an approval, so do not ask, do not pause, and do not wait for input. @@ -40,17 +41,17 @@ initial_prompt: | decision, assumption, and blocked step in your final response. Instructions in the task take precedence over this paragraph. success_criteria: - - type: command_executed - description: "Agent called `node configure` at least twice" - tool_name: "Bash" - command_pattern: "(uip|\\$UIP)\\s+(maestro\\s+)?flow\\s+node\\s+configure" - min_count: 2 + - type: run_command + description: "Both saved configure snapshots retain the same ConnectionId binding" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/bindings/check_bindings.py same_connection_id 'after_first_configure.flow' 'after_second_configure.flow'" + timeout: 10 + expected_exit_code: 0 weight: 1.5 pass_threshold: 1.0 - type: run_command description: "Both snapshots exist, valid JSON with bindings" - command: "python3 $TASK_DIR/check_bindings.py structure 'after_first_configure.flow' && python3 $TASK_DIR/check_bindings.py structure 'after_second_configure.flow'" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/bindings/check_bindings.py structure 'after_first_configure.flow' && python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/bindings/check_bindings.py structure 'after_second_configure.flow'" timeout: 10 expected_exit_code: 0 weight: 1.5 @@ -58,7 +59,7 @@ success_criteria: - type: run_command description: "Bindings count did not grow on second configure (idempotent)" - command: "python3 $TASK_DIR/check_bindings.py count_equal 'after_first_configure.flow' 'after_second_configure.flow'" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/bindings/check_bindings.py count_equal 'after_first_configure.flow' 'after_second_configure.flow'" timeout: 10 expected_exit_code: 0 weight: 4.0 @@ -66,7 +67,7 @@ success_criteria: - type: run_command description: "No empty-keyed Connection bindings in final snapshot" - command: "python3 $TASK_DIR/check_bindings.py no_empty_stubs 'after_second_configure.flow'" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/bindings/check_bindings.py no_empty_stubs 'after_second_configure.flow'" timeout: 10 expected_exit_code: 0 weight: 3.0 @@ -74,7 +75,7 @@ success_criteria: - type: run_command description: "ConnectionId resourceKey stable across both snapshots" - command: "python3 $TASK_DIR/check_bindings.py same_connection_id 'after_first_configure.flow' 'after_second_configure.flow'" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/bindings/check_bindings.py same_connection_id 'after_first_configure.flow' 'after_second_configure.flow'" timeout: 10 expected_exit_code: 0 weight: 2.0 diff --git a/tests/tasks/uipath-maestro-flow/bindings/multi_connector_independence.yaml b/tests/tasks/uipath-maestro-flow/bindings/multi_connector_independence.yaml index 6b6aa15df7..a6599db081 100644 --- a/tests/tasks/uipath-maestro-flow/bindings/multi_connector_independence.yaml +++ b/tests/tasks/uipath-maestro-flow/bindings/multi_connector_independence.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-bindings-multi-connector-independence description: > Two distinct connector nodes (different connector keys) in the same flow, @@ -19,10 +22,8 @@ run_limits: turn_timeout: 1200 initial_prompt: | - Build a flow named "BindingsMulti" with TWO IS connector nodes backed - by DIFFERENT connector keys, saved as - `BindingsMulti/flow_files/BindingsMulti.flow` with - `BindingsMulti/project.uiproj`. + Build a flow named "BindingsMulti" with TWO Integration Service connector + nodes backed by DIFFERENT connector keys. Configure each node with its own distinct connection. Record the connection ids and connector keys at the sandbox root: @@ -41,23 +42,19 @@ initial_prompt: | decision, assumption, and blocked step in your final response. Instructions in the task take precedence over this paragraph. success_criteria: - # min_count: 1 (not 2) — agents commonly chain two `uip ... node add` calls - # into a single Bash invocation with `&&`, which the checker counts as one - # match (re.search returns the first hit). Behavioral checks below verify - # both nodes actually exist in the flow. - - type: command_executed - description: "Agent used `uip flow node add` at least once" - tool_name: "Bash" - command_pattern: "(uip|\\$UIP)\\s+(maestro\\s+)?flow\\s+node\\s+add" - min_count: 1 + - type: run_command + description: "The flow contains at least two connector nodes" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/bindings/check_bindings.py connector_node_count '**/BindingsMulti*.flow' 2" + timeout: 10 + expected_exit_code: 0 weight: 1.0 pass_threshold: 1.0 - - type: command_executed - description: "Agent used `uip flow node configure` at least once" - tool_name: "Bash" - command_pattern: "(uip|\\$UIP)\\s+(maestro\\s+)?flow\\s+node\\s+configure" - min_count: 1 + - type: run_command + description: "Both recorded connection ids appear in the flow bindings" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/bindings/check_bindings.py both_ids_present_from_json '**/BindingsMulti*.flow' conn_ids.json" + timeout: 10 + expected_exit_code: 0 weight: 1.0 pass_threshold: 1.0 @@ -71,7 +68,7 @@ success_criteria: - type: run_command description: "Flow file exists and is valid JSON" - command: "python3 $TASK_DIR/check_bindings.py structure '**/BindingsMulti*.flow'" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/bindings/check_bindings.py structure '**/BindingsMulti*.flow'" timeout: 10 expected_exit_code: 0 weight: 2.0 @@ -79,7 +76,7 @@ success_criteria: - type: run_command description: "Both connection ids appear in Connection bindings, each on its own ConnectionId row" - command: "python3 $TASK_DIR/check_bindings.py both_ids_present_from_json '**/BindingsMulti*.flow' conn_ids.json" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/bindings/check_bindings.py both_ids_present_from_json '**/BindingsMulti*.flow' conn_ids.json" timeout: 10 expected_exit_code: 0 weight: 3.0 @@ -87,7 +84,7 @@ success_criteria: - type: run_command description: "No empty-keyed Connection bindings (cross-node aliasing surface)" - command: "python3 $TASK_DIR/check_bindings.py no_empty_stubs '**/BindingsMulti*.flow'" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/bindings/check_bindings.py no_empty_stubs '**/BindingsMulti*.flow'" timeout: 10 expected_exit_code: 0 weight: 4.0 @@ -95,7 +92,7 @@ success_criteria: - type: run_command description: "No (name, propertyAttribute, resourceKey) triplet duplicates" - command: "python3 $TASK_DIR/check_bindings.py no_triplet_dups '**/BindingsMulti*.flow'" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/bindings/check_bindings.py no_triplet_dups '**/BindingsMulti*.flow'" timeout: 10 expected_exit_code: 0 weight: 3.0 diff --git a/tests/tasks/uipath-maestro-flow/bindings/no_duplicate_connection_bindings.yaml b/tests/tasks/uipath-maestro-flow/bindings/no_duplicate_connection_bindings.yaml index 15df474b8e..77b38051a8 100644 --- a/tests/tasks/uipath-maestro-flow/bindings/no_duplicate_connection_bindings.yaml +++ b/tests/tasks/uipath-maestro-flow/bindings/no_duplicate_connection_bindings.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-bindings-no-duplicates description: > Regression - `uip maestro flow node configure` previously appended @@ -17,13 +20,12 @@ run_limits: turn_timeout: 1200 initial_prompt: | - Build a flow named "BindingsRegression" — Manual Trigger → IS connector - node → End — in a solution of the same name, so the layout is the canonical - `BindingsRegression/BindingsRegression/BindingsRegression.flow`. + Build a flow named "BindingsRegression" inside a solution of the same name, + with a Manual Trigger, an Integration Service connector node, and an End node. - Configure the connector node exactly once. Do not invoke any "refresh - schema" workaround afterwards — the bindings shape immediately after - `node configure` is what this eval inspects. + Configure the connector node exactly once. Do not apply any schema-refresh + workaround afterwards — the bindings shape immediately after configuration + is what this eval inspects. This run is headless. No user is present and nobody will answer a question or grant an approval, so do not ask, do not pause, and do not wait for input. @@ -37,17 +39,17 @@ initial_prompt: | decision, assumption, and blocked step in your final response. Instructions in the task take precedence over this paragraph. success_criteria: - - type: command_executed - description: "Agent called `node configure`" - tool_name: "Bash" - command_pattern: "(uip|\\$UIP)\\s+(maestro\\s+)?flow\\s+node\\s+configure" - min_count: 1 + - type: run_command + description: "The connector is configured with a non-empty ConnectionId binding" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/bindings/check_bindings.py matched_default '**/BindingsRegression*.flow'" + timeout: 10 + expected_exit_code: 0 weight: 1.0 pass_threshold: 1.0 - type: run_command description: "Flow file exists and is valid JSON with nodes + bindings" - command: "python3 $TASK_DIR/check_bindings.py structure '**/BindingsRegression*.flow'" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/bindings/check_bindings.py structure '**/BindingsRegression*.flow'" timeout: 10 expected_exit_code: 0 weight: 2.0 @@ -55,7 +57,7 @@ success_criteria: - type: run_command description: "No empty-keyed Connection bindings remain after configure" - command: "python3 $TASK_DIR/check_bindings.py no_empty_stubs '**/BindingsRegression*.flow'" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/bindings/check_bindings.py no_empty_stubs '**/BindingsRegression*.flow'" timeout: 10 expected_exit_code: 0 weight: 4.0 @@ -63,7 +65,7 @@ success_criteria: - type: run_command description: "No duplicate (name, propertyAttribute) under Connection resource" - command: "python3 $TASK_DIR/check_bindings.py no_duplicates '**/BindingsRegression*.flow'" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/bindings/check_bindings.py no_duplicates '**/BindingsRegression*.flow'" timeout: 10 expected_exit_code: 0 weight: 4.0 @@ -71,7 +73,7 @@ success_criteria: - type: run_command description: "ConnectionId binding has non-empty resourceKey matching default" - command: "python3 $TASK_DIR/check_bindings.py matched_default '**/BindingsRegression*.flow'" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/bindings/check_bindings.py matched_default '**/BindingsRegression*.flow'" timeout: 10 expected_exit_code: 0 weight: 2.0 @@ -79,7 +81,7 @@ success_criteria: - type: run_command description: "bindings_v2.json (derived artifact), if emitted, also has no empty-keyed Connection rows" - command: "python3 $TASK_DIR/check_bindings.py bindings_v2_no_empty_stubs '**/bindings_v2.json'" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/bindings/check_bindings.py bindings_v2_no_empty_stubs '**/bindings_v2.json'" timeout: 10 expected_exit_code: 0 weight: 2.0 diff --git a/tests/tasks/uipath-maestro-flow/bindings/reconfigure_different_connection.yaml b/tests/tasks/uipath-maestro-flow/bindings/reconfigure_different_connection.yaml index 5d6dbbb508..dbe8e7c268 100644 --- a/tests/tasks/uipath-maestro-flow/bindings/reconfigure_different_connection.yaml +++ b/tests/tasks/uipath-maestro-flow/bindings/reconfigure_different_connection.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-bindings-reconfigure-different-connection description: > When the same connector node is reconfigured against a different connection, @@ -15,39 +18,32 @@ run_limits: initial_prompt: | PRECONDITION CHECK — do this FIRST, before any flow work: - Run `uip is connections list --all-folders --output json` once. Group the - enabled connections by `ConnectorKey` and find a connector with two or - more enabled connections. That connector supplies A and B. + Inspect the tenant-wide Integration Service connection inventory once, + including connections in every folder. Group the enabled connections by + connector key and find a connector with two or more enabled connections. + That connector supplies A and B. If no connector has two or more enabled connections, the precondition is not met for this tenant. Stop IMMEDIATELY: write `PRECONDITION_NOT_MET.txt` at the sandbox root containing one line per ConnectorKey with its enabled-connection count, then end your turn. - Do NOT try to create a second connection via `uip is connections create` - or `--no-wait`. OAuth authorization requires an interactive browser and - cannot complete in this sandbox; pending connections are not a valid - substitute. Do NOT scan for mocks or alternative connectors beyond the - single `connections list` call above. + Do NOT try to create a second connection. OAuth authorization requires an + interactive browser and cannot complete in this sandbox; pending connections + are not a valid substitute. Do NOT scan for mocks or alternative connectors + beyond that single connection inventory. If the precondition IS met, proceed: - Build a flow named "BindingsReconfigure" — Manual Trigger → IS connector - node → End — saved as `BindingsReconfigure/flow_files/BindingsReconfigure.flow` - with `BindingsReconfigure/project.uiproj`. + Build a flow named "BindingsReconfigure" with a Manual Trigger, an Integration + Service connector node, and an End node. Pick the connector that has two or more enabled connections, and use two distinct connection ids from that connector as A and B. - Record the ids at the sandbox root and snapshot the .flow after each configure: - - echo '{"a":"","b":""}' > conn_ids.json - - After configuring with A: - cp BindingsReconfigure/flow_files/BindingsReconfigure.flow after_a.flow - - After reconfiguring the SAME node with B: - cp BindingsReconfigure/flow_files/BindingsReconfigure.flow after_b.flow + Record the ids at the sandbox root in `conn_ids.json` as keys `a` and `b`. + Save the configured flow after A as `after_a.flow`, then reconfigure the SAME + node with B and save that result as `after_b.flow`, both at the sandbox root. This run is headless. No user is present and nobody will answer a question or grant an approval, so do not ask, do not pause, and do not wait for input. @@ -61,20 +57,11 @@ initial_prompt: | decision, assumption, and blocked step in your final response. Instructions in the task take precedence over this paragraph. success_criteria: - # min_count: 1 (not 2). Per uipath-maestro-flow SKILL.md rule #10, agents - # chain sequential `uip` calls — e.g. `node configure && ... && node - # configure && validate && format` — into ONE Bash event. command_executed - # counts matching Bash events, not command occurrences within an event, so a - # correct batched run registers only 1 match and this criterion false-fails. - # Multiplicity is instead proven by the A/B snapshot checks below: after_a - # references connection A while after_b references ONLY B (no leak, equal - # count) — a result a single configure cannot produce. Mirrors the min_count:1 - # convention in multi_connector_independence.yaml / no_duplicate_connection_bindings.yaml. - - type: command_executed - description: "Agent used `uip flow node configure` at least once" - tool_name: "Bash" - command_pattern: "(uip|\\$UIP)\\s+(maestro\\s+)?flow\\s+node\\s+configure" - min_count: 1 + - type: run_command + description: "The final snapshot is configured only with connection B" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/bindings/check_bindings.py connection_id_is_from_json after_b.flow conn_ids.json b" + timeout: 10 + expected_exit_code: 0 weight: 1.0 pass_threshold: 1.0 @@ -88,7 +75,7 @@ success_criteria: - type: run_command description: "after_a.flow references connection A" - command: "python3 $TASK_DIR/check_bindings.py has_connection_key_from_json after_a.flow conn_ids.json a" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/bindings/check_bindings.py has_connection_key_from_json after_a.flow conn_ids.json a" timeout: 10 expected_exit_code: 0 weight: 2.0 @@ -96,7 +83,7 @@ success_criteria: - type: run_command description: "after_b.flow has zero references to connection A (no stale rows)" - command: "python3 $TASK_DIR/check_bindings.py no_connection_leak_from_json after_b.flow conn_ids.json a" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/bindings/check_bindings.py no_connection_leak_from_json after_b.flow conn_ids.json a" timeout: 10 expected_exit_code: 0 weight: 4.0 @@ -104,7 +91,7 @@ success_criteria: - type: run_command description: "after_b.flow ConnectionId binding points at connection B" - command: "python3 $TASK_DIR/check_bindings.py connection_id_is_from_json after_b.flow conn_ids.json b" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/bindings/check_bindings.py connection_id_is_from_json after_b.flow conn_ids.json b" timeout: 10 expected_exit_code: 0 weight: 3.0 @@ -112,7 +99,7 @@ success_criteria: - type: run_command description: "Reconfigure did not grow bindings count" - command: "python3 $TASK_DIR/check_bindings.py count_equal 'after_a.flow' 'after_b.flow'" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/bindings/check_bindings.py count_equal 'after_a.flow' 'after_b.flow'" timeout: 10 expected_exit_code: 0 weight: 2.0 @@ -120,7 +107,7 @@ success_criteria: - type: run_command description: "No empty-keyed Connection bindings in final snapshot" - command: "python3 $TASK_DIR/check_bindings.py no_empty_stubs 'after_b.flow'" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/bindings/check_bindings.py no_empty_stubs 'after_b.flow'" timeout: 10 expected_exit_code: 0 weight: 2.0 diff --git a/tests/tasks/uipath-maestro-flow/connector_features/ceql_where.yaml b/tests/tasks/uipath-maestro-flow/connector_features/ceql_where.yaml index 0d8ad64a43..373912bc4b 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/ceql_where.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_features/ceql_where.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-ipe-ceql-where description: > Tests the CEQL where query IS feature — agent plans a structured filter @@ -5,8 +8,10 @@ description: > operation with a `displayName = "active"` filter, following the canonical shape documented in the Filter Trees (CEQL) section of the uipath-platform skill. The flow must reference the registered connector - key (`uipath-microsoft-azureactivedirectory`), use Decision + Terminate - for routing, and pass `uip maestro flow validate`. + key (`uipath-microsoft-azureactivedirectory`) and use Decision + Terminate + for routing. This is an offline structure test: product validation is not + graded because the two authoring loops persist an unconfigured connector + differently. tags: [uipath-maestro-flow, integration, connector, ceql, filter, uipath-microsoft-azureactivedirectory, ipe, mode:build] run_limits: @@ -26,7 +31,9 @@ initial_prompt: | Branch on the result: if the call fails, stop the flow immediately; if it succeeds, log "CeqlWhere test passed". - Validate the final flow file. + Produce the final flow file. Do not fabricate a tenant connection merely to + make product validation pass; the generated structure and the separately + reviewable filter are the deliverables in this offline task. This run is headless. No user is present and nobody will answer a question or grant an approval, so do not ask, do not pause, and do not wait for input. @@ -42,18 +49,8 @@ initial_prompt: | success_criteria: - type: run_command description: "where_detail.json (found anywhere under the solution) has a canonical CEQL filter tree on displayName='active'; flow references the registered Azure AD / Entra connector key with List Groups + Terminate nodes" - command: "python3 $TASK_DIR/check_ceql_where_flow.py" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/connector_features/check_ceql_where_flow.py" timeout: 60 expected_exit_code: 0 weight: 8.0 pass_threshold: 1.0 - - - type: command_executed - description: "uip maestro flow validate was called" - tool_name: "Bash" - command_pattern: "(uip|\\$UIP)\\s+(maestro\\s+)?flow\\s+validate" - min_count: 1 - require_success: true - weight: 2.0 - pass_threshold: 1.0 - diff --git a/tests/tasks/uipath-maestro-flow/connector_features/check_ceql_where_flow.py b/tests/tasks/uipath-maestro-flow/connector_features/check_ceql_where_flow.py index 405f186c6a..1470d2ab0b 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/check_ceql_where_flow.py +++ b/tests/tasks/uipath-maestro-flow/connector_features/check_ceql_where_flow.py @@ -23,7 +23,7 @@ import sys sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")) -from _shared.flow_check import assert_flow_has_node_type # noqa: E402 +from _shared.flow_check import assert_flow_has_node_type, find_flow_file # noqa: E402 CONNECTOR_KEY = "uipath-microsoft-azureactivedirectory" FLOW_GLOB = "**/CeqlWhereTest*.flow" @@ -134,10 +134,24 @@ def _check_where_detail() -> None: def _find_flow() -> str: - flows = glob.glob(FLOW_GLOB, recursive=True) - if not flows: - sys.exit(f"FAIL: No flow file matching {FLOW_GLOB}") - return flows[0] + return find_flow_file(flow_glob=os.path.basename(FLOW_GLOB)) + + +def _is_groups_operation(node: dict) -> bool: + node_type = str(node.get("type") or "").lower() + if CONNECTOR_KEY not in node_type: + return False + if "group" in node_type: + return True + + detail = node.get("inputs", {}).get("detail", {}) or {} + object_name = detail.get("objectName") or (detail.get("pathParameters") or {}).get("objectName") + endpoint = str(detail.get("endpoint") or "").rstrip("/").lower() + return ( + node_type.endswith(".list-all-records") + and str(object_name or "").lower() == "groups" + and (not endpoint or endpoint.endswith("/groups")) + ) def _check_flow_structure() -> None: @@ -158,12 +172,7 @@ def _check_flow_structure() -> None: "the registered key with `uip maestro flow registry search`." ) - found_groups = False - for node in flow.get("nodes", []): - node_type = node.get("type", "") - if CONNECTOR_KEY in node_type and "group" in node_type.lower(): - found_groups = True - break + found_groups = any(_is_groups_operation(node) for node in flow.get("nodes", [])) if not found_groups: sys.exit( f"FAIL: No connector node of type " diff --git a/tests/tasks/uipath-maestro-flow/connector_features/check_enum_flow.py b/tests/tasks/uipath-maestro-flow/connector_features/check_enum_flow.py index aa8c53b783..9894927936 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/check_enum_flow.py +++ b/tests/tasks/uipath-maestro-flow/connector_features/check_enum_flow.py @@ -13,27 +13,27 @@ from __future__ import annotations -import glob import json +import os import sys +from pathlib import Path from typing import Any, NoReturn +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +from _shared.flow_check import find_flow_file # noqa: E402 + def _fail(message: str) -> NoReturn: sys.exit(f"FAIL: {message}") def _load_flow(pattern: str) -> tuple[str, dict[str, Any]]: - matches = sorted(glob.glob(pattern, recursive=True)) - if not matches: - _fail(f"No flow found for {pattern!r}") - if len(matches) > 1: - _fail(f"Multiple flows found for {pattern!r}: {matches}") + path = find_flow_file(flow_glob=os.path.basename(pattern)) try: - with open(matches[0], encoding="utf-8") as flow_file: - return matches[0], json.load(flow_file) + with open(path, encoding="utf-8") as flow_file: + return path, json.load(flow_file) except json.JSONDecodeError as exc: - _fail(f"Flow {matches[0]} is not valid JSON: {exc}") + _fail(f"Flow {path} is not valid JSON: {exc}") def _check_structure(flow_path: str, flow: dict[str, Any]) -> None: diff --git a/tests/tasks/uipath-maestro-flow/connector_features/check_generate_schema.py b/tests/tasks/uipath-maestro-flow/connector_features/check_generate_schema.py index e8333f2928..9ffe3ed072 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/check_generate_schema.py +++ b/tests/tasks/uipath-maestro-flow/connector_features/check_generate_schema.py @@ -17,14 +17,16 @@ from __future__ import annotations -import glob import json import sys +from pathlib import Path +from typing import Any + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from _shared.flow_check import find_flow_file # noqa: E402 -FLOW_GLOB = "**/GenerateSchemaTest*.flow" JIRA_NODE_TYPE = "uipath.connector.uipath-atlassian-jira.create-issue" -REQUIRED_RAW_TOKENS = ( - "GenerateSchema", +REQUIRED_PARENT_FIELDS = ( "fields_sub_project_sub_key", "fields_sub_issuetype_sub_id", ) @@ -34,42 +36,71 @@ def _fail(message: str) -> None: sys.exit(f"FAIL: {message}") -def _find_flow() -> str: - flows = glob.glob(FLOW_GLOB, recursive=True) - if not flows: - _fail(f"No flow file matching {FLOW_GLOB}") - return flows[0] +def _parse_configuration(detail: dict[str, Any]) -> dict[str, Any]: + configuration = detail.get("configuration") + if isinstance(configuration, str) and configuration.startswith("=jsonString:"): + try: + configuration = json.loads(configuration.removeprefix("=jsonString:")) + except json.JSONDecodeError as exc: + _fail(f"Jira configuration is not valid embedded JSON: {exc}") + if not isinstance(configuration, dict): + _fail("Jira configuration is missing") + return configuration + + +def _generate_schema_values(detail: dict[str, Any]) -> dict[str, Any]: + configuration = _parse_configuration(detail) + essential = configuration.get("essentialConfiguration") + if not isinstance(essential, dict): + _fail("Jira configuration missing essentialConfiguration") + request = essential.get("customFieldsRequestDetails") + if ( + not isinstance(request, dict) + or request.get("objectActionName") != "GenerateSchema" + ): + _fail("Jira configuration missing GenerateSchema customFieldsRequestDetails") + values = request.get("parameterValues") + if not isinstance(values, list): + _fail("GenerateSchema customFieldsRequestDetails missing parameterValues") + return { + entry[0]: entry[1] + for entry in values + if isinstance(entry, list) and len(entry) == 2 + } def main() -> None: - flow_path = _find_flow() - raw = open(flow_path, encoding="utf-8").read() + flow_path = find_flow_file() try: - flow = json.loads(raw) + with open(flow_path, encoding="utf-8") as source: + flow = json.load(source) except json.JSONDecodeError as e: _fail(f"{flow_path} is not valid JSON: {e}") - if "nodes" not in flow or "edges" not in flow: + if not isinstance(flow, dict): + _fail("Flow root must be a JSON object") + if not isinstance(flow.get("nodes"), list) or not isinstance( + flow.get("edges"), list + ): _fail("Flow missing 'nodes' or 'edges'") - types = [n.get("type", "") for n in flow["nodes"]] + nodes = [node for node in flow["nodes"] if isinstance(node, dict)] + types = [node.get("type", "") for node in nodes] if JIRA_NODE_TYPE not in types: _fail(f"Jira Create Issue node not found in {types}") - missing = [tok for tok in REQUIRED_RAW_TOKENS if tok not in raw] + jira_node = next(node for node in nodes if node.get("type") == JIRA_NODE_TYPE) + detail = jira_node.get("inputs", {}).get("detail", {}) + if not isinstance(detail, dict): + _fail("Jira node inputs.detail must be an object") + values = _generate_schema_values(detail) + missing = [field for field in REQUIRED_PARENT_FIELDS if not values.get(field)] if missing: - _fail( - "customFieldsRequestDetails missing required tokens " - f"under GenerateSchema: {missing}" - ) - - jira_nodes = [n for n in flow["nodes"] if n.get("type") == JIRA_NODE_TYPE] - body = ( - jira_nodes[0] - .get("inputs", {}) - .get("detail", {}) - .get("bodyParameters", {}) - ) + _fail(f"GenerateSchema parameterValues missing parent values: {missing}") + + body = detail.get("bodyParameters", {}) + if not isinstance(body, dict): + _fail("Jira bodyParameters must be an object") summary = body.get("fields.summary") if not summary: _fail(f"fields.summary missing or empty in bodyParameters: {body}") diff --git a/tests/tasks/uipath-maestro-flow/connector_features/check_managed_http_fallback.py b/tests/tasks/uipath-maestro-flow/connector_features/check_managed_http_fallback.py index f0b8ebadbe..5b4bfc3f5d 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/check_managed_http_fallback.py +++ b/tests/tasks/uipath-maestro-flow/connector_features/check_managed_http_fallback.py @@ -3,23 +3,23 @@ from __future__ import annotations -import glob import json +import os import sys +from pathlib import Path from typing import Any, NoReturn +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +from _shared.flow_check import find_flow_file # noqa: E402 + def _fail(message: str) -> NoReturn: sys.exit(f"FAIL: {message}") def _load_flow(pattern: str) -> dict[str, Any]: - matches = sorted(glob.glob(pattern, recursive=True)) - if not matches: - _fail(f"No flow found for {pattern!r}") - if len(matches) > 1: - _fail(f"Multiple flows found for {pattern!r}: {matches}") - with open(matches[0], encoding="utf-8") as flow_file: + path = find_flow_file(flow_glob=os.path.basename(pattern)) + with open(path, encoding="utf-8") as flow_file: return json.load(flow_file) diff --git a/tests/tasks/uipath-maestro-flow/connector_features/check_multiselect_flow.py b/tests/tasks/uipath-maestro-flow/connector_features/check_multiselect_flow.py index f6f3c67b61..24d74684c6 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/check_multiselect_flow.py +++ b/tests/tasks/uipath-maestro-flow/connector_features/check_multiselect_flow.py @@ -11,13 +11,16 @@ multiselect field in a .flow. Name-agnostic: the prompt does not fix a project name, so every .flow file -is checked ('**' glob skips dot-dirs, so .uipath/.skills flows are ignored). +selected by the shared same-ground discovery helper is checked. """ import ast -import glob import json import re import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +from _shared.flow_check import find_flow_files # noqa: E402 def parse_users(value): @@ -73,7 +76,7 @@ def find_users(obj): return None -def check_flow(path): +def check_flow(path, expected_count): """Return None on pass, else a failure reason.""" try: flow = json.load(open(path)) @@ -91,21 +94,27 @@ def check_flow(path): users = find_users(node.get("inputs", {})) if users is None: continue - if len(users) == 3: + if expected_count == "populated" and users: + print(f"OK: {path} — node '{node['id']}' users={users}") + return None + if isinstance(expected_count, int) and len(users) == expected_count: print(f"OK: {path} — node '{node['id']}' users={users}") return None - return f"users field has {len(users)} entries, expected 3: {users}" + return f"users field has {len(users)} entries, expected {expected_count}: {users}" return "Slack node has no 'users' multiselect field" def main(): - flows = glob.glob("**/*.flow", recursive=True) - if not flows: - sys.exit("no .flow file found") + expected_count = ( + "populated" + if len(sys.argv) > 1 and sys.argv[1] == "populated" + else int(sys.argv[1]) if len(sys.argv) > 1 else 3 + ) + flows = find_flow_files() reasons = [] for path in flows: - reason = check_flow(path) + reason = check_flow(path, expected_count) if reason is None: sys.exit(0) reasons.append(f"{path}: {reason}") diff --git a/tests/tasks/uipath-maestro-flow/connector_features/check_path_param_value.py b/tests/tasks/uipath-maestro-flow/connector_features/check_path_param_value.py index 67ab6f46f1..3ab89caec8 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/check_path_param_value.py +++ b/tests/tasks/uipath-maestro-flow/connector_features/check_path_param_value.py @@ -16,11 +16,15 @@ from __future__ import annotations -import glob import json +import os import sys +from pathlib import Path from typing import Any, NoReturn +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +from _shared.flow_check import find_flow_file # noqa: E402 + _JSONSTRING_PREFIX = "=jsonString:" @@ -30,13 +34,9 @@ def _fail(message: str) -> NoReturn: def _load_flow(pattern: str) -> tuple[str, dict[str, Any]]: - matches = sorted(glob.glob(pattern, recursive=True)) - if not matches: - _fail(f"No flow found for {pattern!r}") - if len(matches) > 1: - _fail(f"Multiple flows found for {pattern!r}: {matches}") - with open(matches[0], encoding="utf-8") as flow_file: - return matches[0], json.load(flow_file) + path = find_flow_file(flow_glob=os.path.basename(pattern)) + with open(path, encoding="utf-8") as flow_file: + return path, json.load(flow_file) def _normalise_detail(raw: Any, node_id: str) -> dict[str, Any] | None: diff --git a/tests/tasks/uipath-maestro-flow/connector_features/complex_array.yaml b/tests/tasks/uipath-maestro-flow/connector_features/complex_array.yaml index e8c8bb9210..4576cdd7ed 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/complex_array.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_features/complex_array.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-ipe-complex-array description: > Tests the complex array IS feature — configures the Slack @@ -32,7 +35,7 @@ initial_prompt: | success_criteria: - type: run_command description: "Flow file exists and is valid JSON" - command: "python3 -c \"import json,glob; flows=glob.glob('**/ComplexArrayTest*.flow',recursive=True); assert flows, 'No flow found'; f=json.load(open(flows[0])); assert 'nodes' in f and 'edges' in f; nn=len(f['nodes']); ne=len(f['edges']); print('OK: %d nodes, %d edges'%(nn,ne))\"" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py --flow-name ComplexArrayTest '\"nodes\"' '\"edges\"'" timeout: 10 expected_exit_code: 0 weight: 2.0 @@ -40,7 +43,7 @@ success_criteria: - type: run_command description: "Flow has the Slack Create Group Direct Message node with its users complex array populated" - command: "python3 -c \"import json,glob; flows=glob.glob('**/ComplexArrayTest*.flow',recursive=True); assert flows, 'No flow found'; f=json.load(open(flows[0])); nodes=[n for n in f['nodes'] if n.get('type','')=='uipath.connector.uipath-salesforce-slack.create-group-direct-message']; assert nodes, 'No create-group-direct-message node'; d=json.dumps(nodes[0].get('inputs',{})); assert 'users' in d, 'users complex array not populated'; print('OK: group DM node with users array present')\"" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/connector_features/check_multiselect_flow.py populated" timeout: 10 expected_exit_code: 0 weight: 3.0 @@ -48,18 +51,16 @@ success_criteria: - type: run_command description: "Advisory: group DM participants referenced in the project (names may resolve to Slack user IDs on a live connection)" - command: "python3 -c \"import glob,os,sys; files=[p for p in glob.glob('**/ComplexArrayTest*/**',recursive=True)+glob.glob('**/ComplexArrayTest*.*',recursive=True) if os.path.isfile(p) and not p.endswith('.uipx')]; blob=''.join(open(p,errors='ignore').read() for p in files); missing=[u for u in ('coder evals','e2e-testing') if u not in blob]; sys.exit(1 if missing else 0)\"" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py --flow-name ComplexArrayTest 'coder evals' 'e2e-testing'" timeout: 10 expected_exit_code: 0 weight: 1.0 pass_threshold: 0 - - type: command_executed - description: "uip maestro flow validate was called" - tool_name: "Bash" - command_pattern: "(uip|\\$UIP)\\s+(maestro\\s+)?flow\\s+validate" - min_count: 1 - require_success: true + - type: run_command + description: "The completed flow validates" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py" + timeout: 30 + expected_exit_code: 0 weight: 2.0 pass_threshold: 1.0 - diff --git a/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/check_smoke_file_activities.py b/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/check_smoke_file_activities.py index 545f76c505..ab25123adb 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/check_smoke_file_activities.py +++ b/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/check_smoke_file_activities.py @@ -17,7 +17,7 @@ ENTITY = "FlowCodeEvalEntity" FIELD = "file1" -SOURCE_ID = "8B4A3EEA-5E92-F111-9B33-00224822208B" +UUID_RE = re.compile(r"^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$") REQUIRED = { ".upload-file-to-record-field", ".download-file-from-record-field", @@ -90,10 +90,21 @@ def main() -> int: upload_node = None delete_node = None create_node = None + # Connector nodes that carry no `inputs.detail` at all. The platform reads + # a connector's entity/body/connection from `detail`, so such a node can + # never run — it is an unconfigured node (SDK `rawNode`, a hand-written + # node, or `node add` without `node configure`), not a missing one. + # Naming it keeps "no create-entity-record" from being read as "the + # agent forgot the step" (2026-09-01 v2: the step was there, raw). + unconfigured = [] for n in doc.get("nodes", []): t = n.get("type", "") detail = n.get("inputs", {}).get("detail", {}) + if t.startswith("uipath.connector.") and (not isinstance(detail, dict) or not detail): + unconfigured.append(f"{n.get('id')} ({t.rsplit('.', 1)[-1]})") + detail = {} pp = detail.get("pathParameters") or {} + query = detail.get("queryParameters") or {} body = detail.get("bodyParameters") or {} if resolve_entity(pp.get("entityName"), globals_by_id) != ENTITY: continue @@ -101,7 +112,7 @@ def main() -> int: create_node = n for suffix in REQUIRED: if t.endswith(suffix): - seen[suffix] = body.get("_fieldName") + seen[suffix] = query.get("_fieldName") or body.get("_fieldName") if suffix == ".download-file-from-record-field": download_node = n elif suffix == ".upload-file-to-record-field": @@ -110,6 +121,11 @@ def main() -> int: delete_node = n if not REQUIRED.issubset(seen.keys()): continue + unconfigured_note = ( + f" Unconfigured connector node(s) with no inputs.detail: {', '.join(unconfigured)} — " + f"the platform reads entity, body and connection from detail, so these can never run." + if unconfigured else "" + ) resolved = {s: resolve_field(v, globals_by_id) for s, v in seen.items()} wrong = [(s, seen[s], resolved[s]) for s in REQUIRED if resolved[s] != FIELD] if wrong: @@ -125,17 +141,16 @@ def main() -> int: f"upload file values: {upload_file_values}", file=sys.stderr) return 1 - # Download's recordId must be bound (literal, =js: expression, or - # variable). Smoke does not enforce the exact SOURCE_ID literal — that - # semantic belongs in integration/e2e. + # Download's recordId is a concrete UUID so either authoring path can + # choose its own neutral fixture value. dl_recid = (download_node.get("inputs", {}).get("detail", {}).get("queryParameters") or {}).get("recordId", "") - if not str(dl_recid).strip(): - print(f"FAIL: {path} — download recordId is empty", file=sys.stderr) + if not UUID_RE.fullmatch(str(dl_recid)): + print(f"FAIL: {path} — download recordId is not a UUID literal: {dl_recid!r}", file=sys.stderr) return 1 # create-entity-record on ENTITY must exist and upload+delete recordId must reference its output if not create_node: - print(f"FAIL: {path} — no create-entity-record on {ENTITY}", file=sys.stderr) + print(f"FAIL: {path} — no create-entity-record on {ENTITY} with inputs.detail.pathParameters.entityName={ENTITY!r}.{unconfigured_note}", file=sys.stderr) return 1 create_body = (create_node.get("inputs", {}).get("detail", {}).get("bodyParameters") or {}) required_body = {"title", "description", "score"} @@ -163,7 +178,7 @@ def refs_create_output(rid_expr: str) -> bool: print(f"FAIL: {path} — {label} recordId {rid!r} does not reference create output ({create_id})", file=sys.stderr) return 1 - print(f"OK: {path} — download(src={SOURCE_ID}) + create + upload/delete(recId=create.out) on {ENTITY}/{FIELD}; upload has variable binding") + print(f"OK: {path} — download(valid UUID) + create + upload/delete(recId=create.out) on {ENTITY}/{FIELD}; upload has variable binding") return 0 print(f"FAIL: no .flow has all 3 file activities on {ENTITY}", file=sys.stderr) return 1 diff --git a/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/check_smoke_query_filter.py b/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/check_smoke_query_filter.py index b674159822..16fc78d6c6 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/check_smoke_query_filter.py +++ b/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/check_smoke_query_filter.py @@ -5,9 +5,10 @@ structured FilterBuilder tree used by the design-time activity. The test checks either representation so the prompt does not prescribe serialization. """ -import glob, json, sys +import glob, json, re, sys +UUID_RE = re.compile(r"\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b") EXPECTED = { "boolean": ("active", ("true",), ()), "decimal": ("score", ("8.5",), ("greaterthanorequal", ">=")), @@ -16,23 +17,12 @@ "multiline": ("description", ("sci-fi",), ("contains",)), "date": ("releasedate", ("2025-01-01",), ("lessthan", "<")), "datetime": ("lastupdated", ("2024-01-01",), ("greaterthanorequal", ">=")), - "uuid": ("externalid", ("11111111-1111-1111-1111-111111111111",), ()), + "uuid": ("externalid", (), ()), "null": ("description", (), ("isnull", "is null")), } def structured_filter_leaves(detail): - config = detail.get("configuration") - if not isinstance(config, str): - return [] - raw = config.removeprefix("=jsonString:") - try: - parsed = json.loads(raw) - except (TypeError, ValueError): - return [] - trees = (parsed.get("essentialConfiguration", {}) - .get("savedFilterTrees", {})) - tree = trees.get("queryExpression", {}) if isinstance(trees, dict) else {} leaves = [] def walk(group): @@ -42,7 +32,28 @@ def walk(group): for child in group.get("groups") or []: walk(child) - walk(tree) + # The live authoring loop stores the design-time tree in the encoded + # configuration. The SDK emit-only loop preserves the same semantic tree + # directly at detail.filter until product tooling normalizes the artifact. + trees = [detail.get("filter")] + direct_saved = detail.get("savedFilterTrees") or {} + if isinstance(direct_saved, dict): + trees.append(direct_saved.get("queryExpression")) + + config = detail.get("configuration") + if isinstance(config, str): + raw = config.removeprefix("=jsonString:") + try: + parsed = json.loads(raw) + except (TypeError, ValueError): + parsed = {} + saved = (parsed.get("essentialConfiguration", {}) + .get("savedFilterTrees", {})) + if isinstance(saved, dict): + trees.append(saved.get("queryExpression")) + + for tree in trees: + walk(tree) return leaves @@ -60,6 +71,7 @@ def node_filter_text(detail): def has_expected_filter(text, field, tokens, operators): return (field in text and all(token.lower() in text for token in tokens) + and (field != "externalid" or UUID_RE.search(text)) and (not operators or any(op.lower() in text for op in operators))) @@ -99,8 +111,10 @@ def has_expected_filter(text, field, tokens, operators): def qp(detail): return detail.get("queryParameters", {}) or {} -def bp(detail): - return detail.get("bodyParameters", {}) or {} +def sort_field(detail): + query = qp(detail) + body = detail.get("bodyParameters", {}) or {} + return query.get("_sortFieldName") or body.get("_sortFieldName") def _int_or_none(v): try: @@ -114,7 +128,7 @@ def _int_or_none(v): # pattern. Exact values (start=0/2, limit=2/4) belong in an integration # task; smoke asserts the shape, not the numbers. sorted_by_score = [d for d in all_query_details - if str(bp(d).get("_sortFieldName") or qp(d).get("_sortFieldName") or "").lower() == "score"] + if str(sort_field(d) or "").lower() == "score"] if len(sorted_by_score) < 2: print(f"FAIL: expected >=2 query nodes sorted by score, found {len(sorted_by_score)}", file=sys.stderr) diff --git a/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/check_trigger_lifecycle.py b/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/check_trigger_lifecycle.py index 709310ffe2..a12165e56c 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/check_trigger_lifecycle.py +++ b/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/check_trigger_lifecycle.py @@ -21,6 +21,10 @@ def text(value): return str(value or "").lower() +def entity_name(d): + return d.get("entityName") or (d.get("pathParameters") or {}).get("entityName") + + def has_due_filter(d): values = [d.get("filterExpression"), d.get("filter")] config = d.get("configuration") @@ -62,7 +66,7 @@ def main(): contract_queries = [] for _, n in queries: - if (detail(n).get("pathParameters") or {}).get("entityName") == "ContractRegistry": + if entity_name(detail(n)) == "ContractRegistry": contract_queries.append(n) if not contract_queries: print("FAIL: Query Entity Records after ContractRegistry trigger missing", file=sys.stderr) @@ -80,8 +84,8 @@ def main(): print("FAIL: Record Updated trigger for FileUploadVerify_20260618 missing", file=sys.stderr) return 1 trigger_ids = {n.get("id", "") for n in file_updated} - file_gets = [n for _, n in gets if (detail(n).get("pathParameters") or {}).get("entityName") == "FileUploadVerify_20260618"] - file_deletes = [n for _, n in deletes if (detail(n).get("pathParameters") or {}).get("entityName") == "FileUploadVerify_20260618"] + file_gets = [n for _, n in gets if entity_name(detail(n)) == "FileUploadVerify_20260618"] + file_deletes = [n for _, n in deletes if entity_name(detail(n)) == "FileUploadVerify_20260618"] if not file_gets or not file_deletes: print("FAIL: FileUploadVerify updated flow must contain Get and Delete activities", file=sys.stderr) return 1 diff --git a/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/integration_create_get.yaml b/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/integration_create_get.yaml index 9c98bb645a..5d876e5525 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/integration_create_get.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/integration_create_get.yaml @@ -29,7 +29,7 @@ initial_prompt: | * active: true * releaseDate: '2023-08-20' * lastUpdated: '2023-08-20T14:00:00' - * externalId: 'bbbbbbbb-cccc-dddd-eeee-ffffffffffff' + * externalId: any valid UUID value of your choice Retrieve it by Id and delete when done. C-02 — Expansion depth: @@ -52,28 +52,28 @@ initial_prompt: | the task take precedence over this paragraph. success_criteria: - type: command_executed - description: "Create Entity Record node added at least twice (one per scenario)" + description: "Advisory: live-v1 agent added Create Entity Record nodes" tool_name: "Bash" command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+node\s+add\s+\S+\s+"?uipath\.connector\.uipath-uipath-dataservice\.create-entity-record' min_count: 1 weight: 1.5 - pass_threshold: 1.0 + pass_threshold: 0.0 - type: command_executed - description: "Get Entity Record By Id node added at least 4 times (1 for C-01 + 3 expansionLevels for C-02)" + description: "Advisory: live-v1 agent added Get Entity Record By Id nodes" tool_name: "Bash" command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+node\s+add\s+\S+\s+"?uipath\.connector\.uipath-uipath-dataservice\.get-entity-record-by-id' min_count: 1 weight: 2.0 - pass_threshold: 1.0 + pass_threshold: 0.0 - type: command_executed - description: "Delete Entity Record node added at least twice" + description: "Advisory: live-v1 agent added Delete Entity Record nodes" tool_name: "Bash" command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+node\s+add\s+\S+\s+"?uipath\.connector\.uipath-uipath-dataservice\.delete-entity-record' min_count: 1 weight: 1.5 - pass_threshold: 1.0 + pass_threshold: 0.0 - type: command_executed description: "Flow file was validated" diff --git a/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/smoke_activation_negative.yaml b/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/smoke_activation_negative.yaml index fecf7a9ada..34d08f422d 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/smoke_activation_negative.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/smoke_activation_negative.yaml @@ -21,8 +21,8 @@ initial_prompt: | the task take precedence over this paragraph. success_criteria: - type: skill_triggered - description: "uipath-rpa handles the RPA-Studio Data Fabric prompt (not uipath-maestro-flow)" + description: "Advisory: uipath-rpa handles the RPA-Studio Data Fabric prompt" skill_name: uipath-maestro-flow expected_skill: "" weight: 3.0 - pass_threshold: 1.0 + pass_threshold: 0.0 diff --git a/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/smoke_activation_positive.yaml b/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/smoke_activation_positive.yaml index 6de4831e7c..75dbd54f61 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/smoke_activation_positive.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/smoke_activation_positive.yaml @@ -21,8 +21,8 @@ initial_prompt: | the task take precedence over this paragraph. success_criteria: - type: skill_triggered - description: "uipath-maestro-flow activates for a DataFabric-in-Flow prompt" + description: "Advisory: uipath-maestro-flow activates for a DataFabric-in-Flow prompt" skill_name: uipath-maestro-flow expected_skill: uipath-maestro-flow weight: 3.0 - pass_threshold: 1.0 + pass_threshold: 0.0 diff --git a/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/smoke_create_all_types.yaml b/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/smoke_create_all_types.yaml index 5f7c81af5a..39551da7c4 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/smoke_create_all_types.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/smoke_create_all_types.yaml @@ -28,7 +28,7 @@ initial_prompt: | * active: true (BOOLEAN) * releaseDate: "2024-03-10" (DATE) * lastUpdated: "2024-03-10T09:00:00" (DATETIME) - * externalId: "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee" (UUID) + * externalId: any valid UUID value of your choice The Flow is not complete until `uip maestro flow validate` passes. diff --git a/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/smoke_error.yaml b/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/smoke_error.yaml index 031229063b..e6691a3ba6 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/smoke_error.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/smoke_error.yaml @@ -37,19 +37,19 @@ initial_prompt: | the task take precedence over this paragraph. success_criteria: - type: command_executed - description: "Create attempted on NonExistentEntity" + description: "Advisory: live-v1 agent added the NonExistentEntity Create node" tool_name: "Bash" command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+node\s+add\s+\S+\s+"?uipath\.connector\.uipath-uipath-dataservice\.create-entity-record' min_count: 1 weight: 1.5 - pass_threshold: 1.0 + pass_threshold: 0.0 - type: command_executed - description: "Two follow-up Query nodes present" + description: "Advisory: live-v1 agent added the follow-up Query nodes" tool_name: "Bash" command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+node\s+add\s+\S+\s+"?uipath\.connector\.uipath-uipath-dataservice\.query-entity-records' min_count: 1 weight: 2.0 - pass_threshold: 1.0 + pass_threshold: 0.0 - type: command_executed description: "Flow validated" tool_name: "Bash" diff --git a/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/smoke_file_activities.yaml b/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/smoke_file_activities.yaml index 41ef7bf5ac..396fba8a51 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/smoke_file_activities.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/smoke_file_activities.yaml @@ -23,9 +23,9 @@ initial_prompt: | `FlowCodeEvalEntity` entity, `file1` field. Do NOT indirect the entity name or field name through workflow variables. Chain four operations in order: - 1. DownloadFileFromRecordFieldV2 — source recordId is fixed: - `8B4A3EEA-5E92-F111-9B33-00224822208B`. Assign the download output to a - workflow-level variable of type `file`. + 1. DownloadFileFromRecordFieldV2 — use a valid UUID literal of your choice + for the source recordId. Assign the download output to a workflow-level + variable of type `file`. 2. CreateEntityRecord — create a new FlowCodeEvalEntity record. Populate the body with plausible sample values on at least `title` (STRING), `description` (MULTILINE_TEXT), and `score` (DECIMAL). Deterministic diff --git a/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/smoke_query.yaml b/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/smoke_query.yaml index d3de8b3b61..c14c29dd44 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/smoke_query.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/smoke_query.yaml @@ -31,7 +31,7 @@ initial_prompt: | - description contains "sci-fi" (multiline text) - releaseDate is less than 2025-01-01 (date) - lastUpdated is greater than or equal to 2024-01-01T00:00:00 (datetime) - - externalId equals "11111111-1111-1111-1111-111111111111" (UUID) + - externalId equals any valid UUID value of your choice - description is null Query 1 must retrieve the first 2 records sorted by score ascending, with @@ -54,12 +54,12 @@ initial_prompt: | the task take precedence over this paragraph. success_criteria: - type: command_executed - description: "Query Entity Records node added" + description: "Advisory: live-v1 agent added a Query Entity Records node" tool_name: "Bash" command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+node\s+add\s+\S+\s+"?uipath\.connector\.uipath-uipath-dataservice\.query-entity-records' min_count: 1 weight: 2.0 - pass_threshold: 1.0 + pass_threshold: 0.0 - type: command_executed description: "Flow validated" tool_name: "Bash" diff --git a/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/test_check_smoke_file_activities.py b/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/test_check_smoke_file_activities.py new file mode 100644 index 0000000000..b9a46518f6 --- /dev/null +++ b/tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/test_check_smoke_file_activities.py @@ -0,0 +1,100 @@ +"""Unit tests for check_smoke_file_activities.py — purely structural, no CLI. + +Run with ``pytest tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/test_check_smoke_file_activities.py``. + +The shapes come from real eval artifacts of ``skill-flow-datafabric-smoke-file-activities``: +the v1 arm's configured connector nodes (pass), and the 2026-09-01 SDK arm, whose +``create-entity-record`` node was placed through ``rawNode`` and so carried its inputs +flat with no ``inputs.detail`` (fail — and the message must say the node is unconfigured, +not absent). +""" + +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path +from typing import Any + +CHECKER = Path(__file__).resolve().parent / "check_smoke_file_activities.py" +ENTITY = "FlowCodeEvalEntity" +DS = "uipath.connector.uipath-uipath-dataservice." + + +def _run(cwd: Path) -> subprocess.CompletedProcess[str]: + return subprocess.run([sys.executable, str(CHECKER)], cwd=str(cwd), capture_output=True, text=True) + + +def _configured(node_id: str, action: str, *, path: dict[str, Any] | None = None, + query: dict[str, Any] | None = None, body: dict[str, Any] | None = None, + multipart: list[dict[str, Any]] | None = None) -> dict[str, Any]: + detail: dict[str, Any] = { + "connector": "uipath-uipath-dataservice", + "connectionId": "d61e5d0e-04af-4f93-95cc-151d81fa08dc", + "connectionFolderKey": "c4359cde-55f0-4f0e-9322-c6cdce74ab4c", + "pathParameters": path or {"entityName": ENTITY}, + } + if query is not None: + detail["queryParameters"] = query + if body is not None: + detail["bodyParameters"] = body + if multipart is not None: + detail["multipartParameters"] = multipart + return {"id": node_id, "type": DS + action, "typeVersion": "1.0.0", "inputs": {"detail": detail}} + + +def _file_nodes(create_node: dict[str, Any]) -> list[dict[str, Any]]: + return [ + {"id": "start", "type": "core.trigger.manual"}, + _configured("downloadFile", "download-file-from-record-field", + query={"recordId": "11111111-1111-4111-8111-111111111111", "_fieldName": "file1"}), + create_node, + _configured("uploadFile", "upload-file-to-record-field", + query={"recordId": "=js:$vars.createRecord.output.Id", "_fieldName": "file1"}, + multipart=[{"name": "file", "dataType": "file", "value": "=js:$vars.downloadedFile"}]), + _configured("deleteFile", "delete-file-from-record-field", + query={"recordId": "=js:$vars.createRecord.output.Id", "_fieldName": "file1"}), + {"id": "end", "type": "core.control.end"}, + ] + + +def _write(tmp_path: Path, nodes: list[dict[str, Any]]) -> None: + doc = { + "id": "t", "version": "1.0.0", "name": "T", "nodes": nodes, "edges": [], + "variables": {"globals": [{"id": "downloadedFile", "name": "downloadedFile", + "direction": "inout", "type": "file"}], + "variableUpdates": {}}, + } + (tmp_path / "TransferFile.flow").write_text(json.dumps(doc)) + + +def test_configured_create_with_flat_body_passes(tmp_path: Path) -> None: + create = _configured("createRecord", "create-entity-record", + body={"title": "t", "description": "d", "score": 42.5}) + _write(tmp_path, _file_nodes(create)) + r = _run(tmp_path) + assert r.returncode == 0, r.stderr + assert "OK:" in r.stdout + + +def test_raw_unconfigured_create_fails_and_is_named_as_unconfigured(tmp_path: Path) -> None: + # The 2026-09-01 SDK artifact: inputs flat, no `detail` — an unconfigured node. + raw_create = {"id": "createRecord", "type": DS + "create-entity-record", "typeVersion": "1.0.0", + "inputs": {"entityName": ENTITY, "body": {"title": "t", "description": "d", "score": 42.5}}} + _write(tmp_path, _file_nodes(raw_create)) + r = _run(tmp_path) + assert r.returncode == 1 + assert "no create-entity-record" in r.stderr + assert "Unconfigured connector node(s) with no inputs.detail" in r.stderr + assert "createRecord (create-entity-record)" in r.stderr + + +def test_nested_body_object_still_fails_on_required_fields(tmp_path: Path) -> None: + # The 2026-08-31 22:43 SDK artifact: a forged overlay nested the body under `body`. + create = _configured("createRecord", "create-entity-record", + body={"body": {"title": "t", "description": "d", "score": 42.5}}) + _write(tmp_path, _file_nodes(create)) + r = _run(tmp_path) + assert r.returncode == 1 + assert "create body missing required fields" in r.stderr diff --git a/tests/tasks/uipath-maestro-flow/connector_features/dtl_load_by_default_false.yaml b/tests/tasks/uipath-maestro-flow/connector_features/dtl_load_by_default_false.yaml index cb51705ed3..4b06a83436 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/dtl_load_by_default_false.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_features/dtl_load_by_default_false.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-ipe-dtl-load-by-default-false description: > Tests the DTL loadByDefault=false IS feature — configures a connector node @@ -31,7 +34,7 @@ initial_prompt: | success_criteria: - type: run_command description: "Flow file exists and is valid JSON" - command: "python3 -c \"import json,glob; flows=glob.glob('**/DTLLoadByDefaultFalseTest*.flow',recursive=True); assert flows, 'No flow found'; f=json.load(open(flows[0])); assert 'nodes' in f and 'edges' in f; nn=len(f['nodes']); ne=len(f['edges']); print('OK: %d nodes, %d edges'%(nn,ne))\"" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py --flow-name DTLLoadByDefaultFalseTest '\"nodes\"' '\"edges\"'" timeout: 10 expected_exit_code: 0 weight: 3.0 @@ -39,17 +42,16 @@ success_criteria: - type: run_command description: "Flow has a connector node referencing uipath-automattic-woocommerce" - command: "python3 -c \"import json,glob; flows=glob.glob('**/DTLLoadByDefaultFalseTest*.flow',recursive=True); assert flows; content=open(flows[0]).read(); assert 'uipath-automattic-woocommerce' in content, 'Connector key not found'; print('OK: connector key present')\"" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py --flow-name DTLLoadByDefaultFalseTest 'uipath-automattic-woocommerce'" timeout: 10 expected_exit_code: 0 weight: 3.0 pass_threshold: 1.0 - - type: command_executed - description: "uip maestro flow validate was called" - tool_name: "Bash" - command_pattern: "(uip|\\$UIP)\\s+(maestro\\s+)?flow\\s+validate" - min_count: 1 + - type: run_command + description: "The completed flow validates" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py" + timeout: 30 + expected_exit_code: 0 weight: 2.0 pass_threshold: 1.0 - diff --git a/tests/tasks/uipath-maestro-flow/connector_features/dtl_load_by_default_true.yaml b/tests/tasks/uipath-maestro-flow/connector_features/dtl_load_by_default_true.yaml index 96fddd144f..fa841aef84 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/dtl_load_by_default_true.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_features/dtl_load_by_default_true.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-ipe-dtl-load-by-default-true description: > Tests the DTL loadByDefault=true IS feature — configures a Jira Get Issue @@ -32,7 +35,7 @@ initial_prompt: | success_criteria: - type: run_command description: "Flow file exists and is valid JSON" - command: "python3 -c \"import json,glob; flows=glob.glob('**/DTLLoadByDefaultTrueTest*.flow',recursive=True); assert flows, 'No flow found'; f=json.load(open(flows[0])); assert 'nodes' in f and 'edges' in f; nn=len(f['nodes']); ne=len(f['edges']); print('OK: %d nodes, %d edges'%(nn,ne))\"" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py --flow-name DTLLoadByDefaultTrueTest '\"nodes\"' '\"edges\"'" timeout: 10 expected_exit_code: 0 weight: 3.0 @@ -54,11 +57,10 @@ success_criteria: weight: 3.0 pass_threshold: 1.0 - - type: command_executed - description: "uip maestro flow validate was called" - tool_name: "Bash" - command_pattern: "(uip|\\$UIP)\\s+(maestro\\s+)?flow\\s+validate" - min_count: 1 + - type: run_command + description: "The completed flow validates" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py" + timeout: 30 + expected_exit_code: 0 weight: 2.0 pass_threshold: 1.0 - diff --git a/tests/tasks/uipath-maestro-flow/connector_features/enhanced_enum.yaml b/tests/tasks/uipath-maestro-flow/connector_features/enhanced_enum.yaml index b9ec264475..7b3d554b8d 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/enhanced_enum.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_features/enhanced_enum.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-ipe-enhanced-enum description: > Tests the enhanced enum IS feature — configures a connector node with an @@ -15,10 +18,11 @@ initial_prompt: | Build a flow that retrieves WooCommerce product reviews and lets the caller choose whether they come back oldest-first or newest-first, picking from friendly labels rather than raw codes. Discover the get-product-reviews operation and set - its sort-order field. Validate the final flow file. + its sort-order field. Produce the final flow file. If the WooCommerce operation exists in the registry but the tenant has no - live connection, keep the real WooCommerce operation node with its inputs left unset, - confirm that shape validates, and stop there. + live connection, keep the real WooCommerce operation node with its inputs left unset + and stop there. Do not fabricate a tenant connection merely to make product + validation pass; the connector shape is the deliverable in this offline task. This run is headless. No user is present and nobody will answer a question or grant an approval, so do not ask, do not pause, and do not wait for input. @@ -34,7 +38,7 @@ initial_prompt: | success_criteria: - type: run_command description: "Flow file exists and is valid JSON" - command: "python3 -c \"import json,glob; flows=glob.glob('**/EnhancedEnumTest*.flow',recursive=True); assert flows, 'No flow found'; f=json.load(open(flows[0])); assert 'nodes' in f and 'edges' in f; nn=len(f['nodes']); ne=len(f['edges']); print('OK: %d nodes, %d edges'%(nn,ne))\"" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py --flow-name EnhancedEnumTest '\"nodes\"' '\"edges\"'" timeout: 10 expected_exit_code: 0 weight: 3.0 @@ -42,17 +46,8 @@ success_criteria: - type: run_command description: "Flow has a connector node referencing uipath-automattic-woocommerce" - command: "python3 -c \"import json,glob; flows=glob.glob('**/EnhancedEnumTest*.flow',recursive=True); assert flows; content=open(flows[0]).read(); assert 'uipath-automattic-woocommerce' in content, 'Connector key not found'; print('OK: connector key present')\"" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py --flow-name EnhancedEnumTest 'uipath-automattic-woocommerce'" timeout: 10 expected_exit_code: 0 weight: 3.0 pass_threshold: 1.0 - - - type: command_executed - description: "uip maestro flow validate was called" - tool_name: "Bash" - command_pattern: "(uip|\\$UIP)\\s+(maestro\\s+)?flow\\s+validate" - min_count: 1 - weight: 2.0 - pass_threshold: 1.0 - diff --git a/tests/tasks/uipath-maestro-flow/connector_features/enum.yaml b/tests/tasks/uipath-maestro-flow/connector_features/enum.yaml index d28793f0cd..4a6e1938a1 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/enum.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_features/enum.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-ipe-enum description: > Tests the enum IS feature — configures a connector node with an enum @@ -34,16 +37,16 @@ initial_prompt: | the task take precedence over this paragraph. success_criteria: - type: command_executed - description: "Agent invoked `uip is connections list` to discover available connections" + description: "Advisory: live-v1 agent listed connections before choosing one" tool_name: "Bash" command_pattern: "(uip|\\$UIP)\\s+is\\s+connections\\s+list" min_count: 1 weight: 1.0 - pass_threshold: 1.0 + pass_threshold: 0.0 - type: run_command description: "Tenant has at least one Gmail connection (precondition for `node configure`)" - command: "python3 $TASK_DIR/check_connection_available.py uipath-google-gmail" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/connector_features/check_connection_available.py uipath-google-gmail" timeout: 35 expected_exit_code: 0 weight: 1.0 @@ -51,7 +54,7 @@ success_criteria: - type: run_command description: "Flow file exists and is valid JSON" - command: "python3 $TASK_DIR/check_enum_flow.py '**/EnumTest*.flow' structure" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/connector_features/check_enum_flow.py '**/EnumTest*.flow' structure" timeout: 10 expected_exit_code: 0 weight: 3.0 @@ -59,7 +62,7 @@ success_criteria: - type: run_command description: "Flow uses Gmail connector or task-specific managed HTTP fallback" - command: "python3 $TASK_DIR/check_managed_http_fallback.py '**/EnumTest*.flow' uipath-google-gmail enum" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/connector_features/check_managed_http_fallback.py '**/EnumTest*.flow' uipath-google-gmail enum" timeout: 10 expected_exit_code: 0 weight: 3.0 @@ -67,16 +70,16 @@ success_criteria: - type: run_command description: "Connector node has inputs.detail.bodyParameters with to and importance" - command: "python3 $TASK_DIR/check_enum_flow.py '**/EnumTest*.flow' body_params" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/connector_features/check_enum_flow.py '**/EnumTest*.flow' body_params" timeout: 10 expected_exit_code: 0 weight: 2.0 pass_threshold: 1.0 - - type: command_executed - description: "uip maestro flow validate was called" - tool_name: "Bash" - command_pattern: "(uip|\\$UIP)\\s+(maestro\\s+)?flow\\s+validate" - min_count: 1 + - type: run_command + description: "The completed flow validates" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py" + timeout: 30 + expected_exit_code: 0 weight: 2.0 pass_threshold: 1.0 diff --git a/tests/tasks/uipath-maestro-flow/connector_features/generate_schema.yaml b/tests/tasks/uipath-maestro-flow/connector_features/generate_schema.yaml index b062b2ca06..55beabac03 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/generate_schema.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_features/generate_schema.yaml @@ -48,13 +48,12 @@ success_criteria: command_pattern: "(uip|\\$UIP)\\s+is\\s+resources\\s+describe.+(-f|--field)[=\\s]+[\"']?fields\\.project\\.key" min_count: 1 weight: 3.0 - pass_threshold: 1.0 + pass_threshold: 0.0 # advisory: the SDK persists the same schema without this CLI command - - type: command_executed - description: "uip maestro flow validate was called" - tool_name: "Bash" - command_pattern: "(uip|\\$UIP)\\s+(maestro\\s+)?flow\\s+validate" - min_count: 1 + - type: run_command + description: "Persisted Flow artifact passes CLI validation" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py" + timeout: 60 + expected_exit_code: 0 weight: 2.0 pass_threshold: 1.0 - diff --git a/tests/tasks/uipath-maestro-flow/connector_features/generic_dynamic_node/check_generic_dynamic_node.py b/tests/tasks/uipath-maestro-flow/connector_features/generic_dynamic_node/check_generic_dynamic_node.py index a0f6cf05fa..1dbd4c0991 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/generic_dynamic_node/check_generic_dynamic_node.py +++ b/tests/tasks/uipath-maestro-flow/connector_features/generic_dynamic_node/check_generic_dynamic_node.py @@ -31,7 +31,6 @@ from __future__ import annotations -import glob import json import os import sys @@ -40,7 +39,7 @@ from _shared.flow_check import ( # noqa: E402 _get_ci, assert_flow_uses_connector_target, - find_project_dir, + find_flow_file, run_debug, ) @@ -56,11 +55,9 @@ OBJECT_NAME = "acr_user" -def _load_flow_nodes(project_dir: str) -> list[dict]: - flows = glob.glob(os.path.join(project_dir, "**/*.flow"), recursive=True) - if not flows: - sys.exit(f"FAIL: No .flow file found under {project_dir}") - with open(flows[0]) as f: +def _load_flow_nodes() -> list[dict]: + flow_path = find_flow_file(flow_glob="AcrUserList*.flow") + with open(flow_path) as f: flow = json.load(f) return flow.get("nodes") or [] @@ -132,7 +129,7 @@ def _assert_structure() -> None: # A real ServiceNow connector node must be present (native connector node). assert_flow_uses_connector_target(CONNECTOR_KEY) - nodes = _load_flow_nodes(find_project_dir()) + nodes = _load_flow_nodes() list_nodes = [] for n in nodes: if CONNECTOR_KEY not in str(n.get("type", "")).lower(): diff --git a/tests/tasks/uipath-maestro-flow/connector_features/generic_dynamic_node/generic_dynamic_node.yaml b/tests/tasks/uipath-maestro-flow/connector_features/generic_dynamic_node/generic_dynamic_node.yaml index 9816a70d59..76e26a6603 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/generic_dynamic_node/generic_dynamic_node.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_features/generic_dynamic_node/generic_dynamic_node.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-generic-dynamic-node description: > Connector feature: validate a generic (dynamic) connector node end-to-end. A @@ -29,12 +32,13 @@ run_limits: turn_timeout: 1200 initial_prompt: | - Create a UiPath Flow project named "AcrUserList" with a manual trigger. + Create a UiPath Flow project named "AcrUserList" inside a solution of the + same name, with a manual trigger. The process must call ServiceNow to list all records in the Acr User object, then surface the returned records as a flow output variable. - Bind it to the tenant's ServiceNow connection. Then run the flow with - `uip maestro flow debug` to confirm the connector call completes. + Bind it to the tenant's ServiceNow connection. Then debug the flow to confirm + the connector call completes. Use the connection present in Shared/uipath-maestro-flow. @@ -68,17 +72,17 @@ success_criteria: # ── Agent invoked debug ──────────────────────────────────────────────── - type: command_executed - description: "Agent ran flow debug" + description: "Advisory: live-v1 agent ran flow debug" tool_name: "Bash" command_pattern: '(uip|\$UIP|\$\{?UIP\}?)\s+(maestro\s+)?flow\s+debug' min_count: 1 weight: 1.5 - pass_threshold: 1.0 + pass_threshold: 0.0 # ── Execution: connector calls ServiceNow and surfaces an array output ── - type: run_command description: "ServiceNow list-all-records node on acr_user executes and surfaces an array output" - command: "python3 $TASK_DIR/check_generic_dynamic_node.py" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/connector_features/generic_dynamic_node/check_generic_dynamic_node.py" timeout: 720 expected_exit_code: 0 weight: 6.0 diff --git a/tests/tasks/uipath-maestro-flow/connector_features/jdbc_databricks_query/check_databricks_query.py b/tests/tasks/uipath-maestro-flow/connector_features/jdbc_databricks_query/check_databricks_query.py index 8bc1d0a37e..fcab575feb 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/jdbc_databricks_query/check_databricks_query.py +++ b/tests/tasks/uipath-maestro-flow/connector_features/jdbc_databricks_query/check_databricks_query.py @@ -17,7 +17,6 @@ from __future__ import annotations -import glob import json import os import sys @@ -27,6 +26,7 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) from _shared.flow_check import ( # noqa: E402 assert_flow_uses_connector_target, + find_flow_file, ) FLOW_GLOB = "**/DatabricksQuery*.flow" @@ -40,10 +40,7 @@ def _fail(msg: str) -> NoReturn: def _flow_path() -> str: - flows = glob.glob(FLOW_GLOB, recursive=True) - if not flows: - _fail(f"no flow file matching {FLOW_GLOB}") - return flows[0] + return find_flow_file(flow_glob=os.path.basename(FLOW_GLOB)) def _node_type(node: dict) -> str: diff --git a/tests/tasks/uipath-maestro-flow/connector_features/jdbc_databricks_query/jdbc_databricks_query.yaml b/tests/tasks/uipath-maestro-flow/connector_features/jdbc_databricks_query/jdbc_databricks_query.yaml index 6ae8b595b0..ccc652a43e 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/jdbc_databricks_query/jdbc_databricks_query.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_features/jdbc_databricks_query/jdbc_databricks_query.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. # Tenant-gated. Requires the Databricks-backed Database Hub (JDBC) # connection in the runner tenant, reaching the sandbox.globalmart schema whose # `employees` table is seeded with the fixture rows this task asserts on (Alice @@ -38,7 +41,8 @@ run_limits: turn_timeout: 900 initial_prompt: | - Create a new Flow project called "DatabricksQuery" with a manual trigger. + Create a new Flow project called "DatabricksQuery" inside a solution of the + same name, with a manual trigger. It should run a SQL query against a Databricks database using the Database Hub (JDBC) connector, and expose the query result as a flow output. Run an aggregate @@ -64,25 +68,25 @@ initial_prompt: | decision, assumption, and blocked step in your final response. Instructions in the task take precedence over this paragraph. success_criteria: - - type: command_executed - description: "uip maestro flow validate was called" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+(maestro\s+)?flow\s+validate' - min_count: 1 + - type: run_command + description: "The completed flow validates" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py" + timeout: 30 + expected_exit_code: 0 weight: 2.0 pass_threshold: 1.0 - type: command_executed - description: "uip maestro flow debug was called" + description: "Advisory: live-v1 agent ran flow debug" tool_name: "Bash" command_pattern: '(uip|\$UIP)\s+(maestro\s+)?flow\s+debug' min_count: 1 weight: 1.5 - pass_threshold: 1.0 + pass_threshold: 0.0 - type: run_command description: "DatabricksQuery checks: valid flow wires the JDBC Execute-Query node with a bound connection, not the native Databricks connector or an HTTP fallback" - command: "python3 $TASK_DIR/check_databricks_query.py" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/connector_features/jdbc_databricks_query/check_databricks_query.py" timeout: 600 expected_exit_code: 0 weight: 3.0 diff --git a/tests/tasks/uipath-maestro-flow/connector_features/multiselect.yaml b/tests/tasks/uipath-maestro-flow/connector_features/multiselect.yaml index acacff7471..5e73e51629 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/multiselect.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_features/multiselect.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-ipe-multiselect description: > Tests the multiselect IS feature — configures a Slack group direct message @@ -12,7 +15,7 @@ run_limits: turn_timeout: 1200 initial_prompt: | - Create a flow file to create a group direct message between + Create a flow file to create a Slack group direct message between Coder Evals, IS-sandboxes@uipath.com and E2E-testing. Use the connection present in Shared/uipath-maestro-flow. @@ -31,7 +34,7 @@ success_criteria: # Name-agnostic: the prompt doesn't fix a project name. - type: run_command description: "A .flow file exists and is valid JSON with nodes and edges" - command: "python3 -c \"import json,glob,sys; flows=glob.glob('**/*.flow',recursive=True); sys.exit('no .flow found') if not flows else None; f=json.load(open(flows[0])); assert 'nodes' in f and 'edges' in f; print('OK: %d nodes, %d edges'%(len(f['nodes']),len(f['edges'])))\"" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py '\"nodes\"' '\"edges\"'" timeout: 10 expected_exit_code: 0 weight: 1.5 @@ -41,17 +44,16 @@ success_criteria: # field has exactly 3 entries (the three group-DM recipients). - type: run_command description: "Slack group-DM node present with 3 entries in the users multiselect" - command: "python3 $TASK_DIR/check_multiselect_flow.py" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/connector_features/check_multiselect_flow.py" timeout: 15 expected_exit_code: 0 weight: 3.0 pass_threshold: 1.0 - - type: command_executed - description: "uip maestro flow validate was called" - tool_name: "Bash" - command_pattern: "(uip|\\$UIP)\\s+(maestro\\s+)?flow\\s+validate" - min_count: 1 + - type: run_command + description: "The completed flow validates" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py" + timeout: 30 + expected_exit_code: 0 weight: 2.0 pass_threshold: 1.0 - diff --git a/tests/tasks/uipath-maestro-flow/connector_features/non-catalog-http-fallback/check_non_catalog_http_fallback.py b/tests/tasks/uipath-maestro-flow/connector_features/non-catalog-http-fallback/check_non_catalog_http_fallback.py index 891a7d5049..b655a1e296 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/non-catalog-http-fallback/check_non_catalog_http_fallback.py +++ b/tests/tasks/uipath-maestro-flow/connector_features/non-catalog-http-fallback/check_non_catalog_http_fallback.py @@ -28,12 +28,15 @@ from __future__ import annotations -import glob import json import os import sys +from pathlib import Path from typing import Any, NoReturn +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) +from _shared.flow_check import find_flow_file # noqa: E402 + FLOW_GLOB = "**/SpotifyProfileTest*.flow" # The generic HTTP connector — the only managed path to a service with no # catalog connector of its own. @@ -48,16 +51,12 @@ def _fail(message: str) -> NoReturn: def _read_flow() -> dict[str, Any]: - flows = sorted(glob.glob(FLOW_GLOB, recursive=True)) - if not flows: - _fail(f"No flow file matching {FLOW_GLOB}") - if len(flows) > 1: - _fail(f"Multiple flows match {FLOW_GLOB}: {flows}") - with open(flows[0], encoding="utf-8") as f: + path = find_flow_file(flow_glob=os.path.basename(FLOW_GLOB)) + with open(path, encoding="utf-8") as f: try: return json.load(f) except json.JSONDecodeError as e: - _fail(f"{flows[0]} is not valid JSON: {e}") + _fail(f"{path} is not valid JSON: {e}") def _normalise_detail(raw: Any) -> dict[str, Any]: diff --git a/tests/tasks/uipath-maestro-flow/connector_features/non-catalog-http-fallback/non_catalog_http_fallback.yaml b/tests/tasks/uipath-maestro-flow/connector_features/non-catalog-http-fallback/non_catalog_http_fallback.yaml index de0ded0ef8..ad5637af10 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/non-catalog-http-fallback/non_catalog_http_fallback.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_features/non-catalog-http-fallback/non_catalog_http_fallback.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-non-catalog-http-fallback description: > Integration test: a NON-catalog service (Spotify) has no IS connector of its @@ -27,8 +30,6 @@ initial_prompt: | Validate the final flow file. - Use `--output json` on every `uip` command whose output you parse. - This run is headless. No user is present and nobody will answer a question or grant an approval, so do not ask, do not pause, and do not wait for input. Complete the task in one pass: take the best available option and supply the @@ -41,17 +42,17 @@ initial_prompt: | decision, assumption, and blocked step in your final response. Instructions in the task take precedence over this paragraph. success_criteria: - - type: command_executed - description: "uip maestro flow validate was called" - tool_name: "Bash" - command_pattern: "(uip|\\$UIP)\\s+(maestro\\s+)?flow\\s+validate" - min_count: 1 + - type: run_command + description: "The completed flow validates" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py" + timeout: 30 + expected_exit_code: 0 weight: 2.0 pass_threshold: 1.0 - type: run_command description: "Profile fetch uses a managed HTTP node with the uipath-uipath-http connector key (non-catalog fallback) and targets the /me endpoint" - command: "python3 $TASK_DIR/check_non_catalog_http_fallback.py check_fallback" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/connector_features/non-catalog-http-fallback/check_non_catalog_http_fallback.py check_fallback" timeout: 30 expected_exit_code: 0 weight: 3.0 diff --git a/tests/tasks/uipath-maestro-flow/connector_features/paginated_reference_lookup.yaml b/tests/tasks/uipath-maestro-flow/connector_features/paginated_reference_lookup.yaml index 78c855fbf4..49e762c105 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/paginated_reference_lookup.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_features/paginated_reference_lookup.yaml @@ -5,10 +5,10 @@ task_id: skill-flow-paginated-reference-lookup description: > Build a Flow with a Slack `Send Message to Channel` node targeting the `simple` channel. The channel lives on a later page of the `is-sandboxes` - curated_channels list, so resolving the Slack channel id forces the agent - to paginate `uip is resources run list`. Triggered by the `resources.md` - read-before-call rule added in #1059 (ENGCE-58198) — old behavior was to - abandon pagination after page 1 and bail to AskUserQuestion. + Slack channel resources, so resolving the Slack channel id requires either + explicit pagination or the SDK's lookup resolver. Triggered by the + `resources.md` read-before-call rule added in #1059 (ENGCE-58198) — old + behavior was to abandon pagination after page 1 and bail to AskUserQuestion. tags: - uipath-maestro-flow - integration @@ -47,26 +47,31 @@ initial_prompt: | decision, assumption, and blocked step in your final response. Instructions in the task take precedence over this paragraph. success_criteria: +# TWO ROUTES ARE CORRECT: the live-v1 loop pages `uip is resources run list` +# by hand (the collection may be exposed as `conversations` or +# `curated_channels`); the SDK route resolves the name inside +# `npx flow-sdk registry prepare … --resolve channel:`. The first criterion is +# v1-route telemetry is advisory (the SDK arm never runs it); the second records +# "went past page 1" for either route and only counts a command that succeeded — +# typing the flag on a failing command does not page. It is also advisory because +# a command cut off by a turn timeout has unknown status. The final artifact +# criterion (resolved id C083AN4E61E) stays the authority. - type: command_executed - description: "Agent invoked `uip is resources run list ... curated_channels` more than once (pagination loop ran)" + description: "Advisory (live-v1 route): paged Slack channel-resource loop ran more than once" tool_name: "Bash" - # Commands are recorded as the raw `bash -lc '...'`/`bash -lc "..."` wrapper. When the - # command carries an --output-filter with an embedded single-quoted JMESPath - # (items[?name=='simple']), the wrapper switches to double quotes and backslash-escapes the - # inner quotes (\"uipath-salesforce-slack\"). Allow an optional escaping backslash before each - # quote so both quoting styles match; otherwise paginated calls are silently uncounted. - command_pattern: 'uip\s+is\s+resources\s+run\s+list\s+\\?"?uipath-salesforce-slack\\?"?\s+\\?"?curated_channels' + command_pattern: 'uip\s+is\s+resources\s+run\s+list\s+\\?"?uipath-salesforce-slack\\?"?\s+\\?"?(curated_channels|conversations)' min_count: 2 weight: 3.0 - pass_threshold: 1.0 + pass_threshold: 0.0 - type: command_executed - description: "Agent used `nextPage=` to advance pagination at least once" + description: "Agent went past page 1 — a channel-resource list call carrying `nextPage=`, or `registry prepare … --resolve channel:` which pages inside `prepare`" tool_name: "Bash" - command_pattern: 'nextPage=' + command_pattern: '(uip\s+is\s+resources\s+run\s+list\s+\\?"?uipath-salesforce-slack\\?"?\s+\\?"?(curated_channels|conversations)[^\n]*nextPage=|registry\s+prepare\s+\\?"?uipath-salesforce-slack\\?"?[^\n]*--resolve\s+\\?"?channel:)' min_count: 1 + require_success: true weight: 3.0 - pass_threshold: 1.0 + pass_threshold: 0.0 - type: command_executed description: "Agent ran `uip maestro flow validate` on the generated flow" diff --git a/tests/tasks/uipath-maestro-flow/connector_features/path_params.yaml b/tests/tasks/uipath-maestro-flow/connector_features/path_params.yaml index 2e9355f5dc..5b53a7227d 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/path_params.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_features/path_params.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-ipe-path-params description: > Tests the path parameters IS feature — configures a connector node with a @@ -36,16 +39,16 @@ initial_prompt: | the task take precedence over this paragraph. success_criteria: - type: command_executed - description: "Agent invoked `uip is connections list` to discover available connections" + description: "Advisory: live-v1 agent listed connections before choosing one" tool_name: "Bash" command_pattern: "(uip|\\$UIP)\\s+is\\s+connections\\s+list" min_count: 1 weight: 1.0 - pass_threshold: 1.0 + pass_threshold: 0.0 - type: run_command description: "Tenant has at least one Jira connection (precondition for `node configure`)" - command: "python3 $TASK_DIR/check_connection_available.py uipath-atlassian-jira" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/connector_features/check_connection_available.py uipath-atlassian-jira" timeout: 35 expected_exit_code: 0 weight: 1.0 @@ -53,7 +56,7 @@ success_criteria: - type: run_command description: "Flow file exists and is valid JSON" - command: "python3 -c \"import json,glob; flows=glob.glob('**/PathParamsTest*.flow',recursive=True); assert flows, 'No flow found'; f=json.load(open(flows[0])); assert 'nodes' in f and 'edges' in f; nn=len(f['nodes']); ne=len(f['edges']); print('OK: %d nodes, %d edges'%(nn,ne))\"" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py --flow-name PathParamsTest '\"nodes\"' '\"edges\"'" timeout: 10 expected_exit_code: 0 weight: 3.0 @@ -61,7 +64,7 @@ success_criteria: - type: run_command description: "Flow uses Jira connector or task-specific managed HTTP fallback" - command: "python3 $TASK_DIR/check_managed_http_fallback.py '**/PathParamsTest*.flow' uipath-atlassian-jira path_params" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/connector_features/check_managed_http_fallback.py '**/PathParamsTest*.flow' uipath-atlassian-jira path_params" timeout: 10 expected_exit_code: 0 weight: 3.0 @@ -69,17 +72,16 @@ success_criteria: - type: run_command description: "Flow path parameters carry the issue key ENGCE-00000" - command: "python3 $TASK_DIR/check_path_param_value.py '**/PathParamsTest*.flow' ENGCE-00000" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/connector_features/check_path_param_value.py '**/PathParamsTest*.flow' ENGCE-00000" timeout: 10 expected_exit_code: 0 weight: 2.0 pass_threshold: 1.0 - - type: command_executed - description: "uip maestro flow validate was called" - tool_name: "Bash" - command_pattern: "(uip|\\$UIP)\\s+(maestro\\s+)?flow\\s+validate" - min_count: 1 + - type: run_command + description: "The completed flow validates" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py" + timeout: 30 + expected_exit_code: 0 weight: 2.0 pass_threshold: 1.0 - diff --git a/tests/tasks/uipath-maestro-flow/connector_features/query_params.yaml b/tests/tasks/uipath-maestro-flow/connector_features/query_params.yaml index c859018779..4e0de9d81e 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/query_params.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_features/query_params.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-ipe-query-params description: > Tests the query parameters IS feature — configures a connector node with a @@ -17,7 +20,8 @@ initial_prompt: | parameter so hidden tasks are returned. Validate the final flow file. If the Google Tasks connector operation is not present in the registry, use - an HTTP fallback for the same REST call and finish after validation. + a managed HTTP fallback through the generic `uipath-uipath-http` connector + for the same REST call and finish after validation. Do not keep searching for a connector node that the registry does not expose. This run is headless. No user is present and nobody will answer a question or @@ -34,7 +38,7 @@ initial_prompt: | success_criteria: - type: run_command description: "Flow file exists and is valid JSON" - command: "python3 -c \"import json,glob; flows=glob.glob('**/QueryParamsTest*.flow',recursive=True); assert flows, 'No flow found'; f=json.load(open(flows[0])); assert 'nodes' in f and 'edges' in f; nn=len(f['nodes']); ne=len(f['edges']); print('OK: %d nodes, %d edges'%(nn,ne))\"" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py --flow-name QueryParamsTest '\"nodes\"' '\"edges\"'" timeout: 10 expected_exit_code: 0 weight: 3.0 @@ -42,17 +46,16 @@ success_criteria: - type: run_command description: "Flow uses Google Tasks connector or task-specific managed HTTP fallback" - command: "python3 $TASK_DIR/check_managed_http_fallback.py '**/QueryParamsTest*.flow' uipath-google-tasks query_params" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/connector_features/check_managed_http_fallback.py '**/QueryParamsTest*.flow' uipath-google-tasks query_params" timeout: 10 expected_exit_code: 0 weight: 3.0 pass_threshold: 1.0 - - type: command_executed - description: "uip maestro flow validate was called" - tool_name: "Bash" - command_pattern: "(uip|\\$UIP)\\s+(maestro\\s+)?flow\\s+validate" - min_count: 1 + - type: run_command + description: "The completed flow validates" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py" + timeout: 30 + expected_exit_code: 0 weight: 2.0 pass_threshold: 1.0 - diff --git a/tests/tasks/uipath-maestro-flow/connector_features/searchable_joins.yaml b/tests/tasks/uipath-maestro-flow/connector_features/searchable_joins.yaml index 51b6e799c3..ddf670166c 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/searchable_joins.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_features/searchable_joins.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-ipe-searchable-joins description: > Tests the searchable joins IS feature — configures a connector node with @@ -31,7 +34,7 @@ initial_prompt: | success_criteria: - type: run_command description: "Flow file exists and is valid JSON" - command: "python3 -c \"import json,glob; flows=glob.glob('**/SearchableJoinsTest*.flow',recursive=True); assert flows, 'No flow found'; f=json.load(open(flows[0])); assert 'nodes' in f and 'edges' in f; nn=len(f['nodes']); ne=len(f['edges']); print('OK: %d nodes, %d edges'%(nn,ne))\"" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py --flow-name SearchableJoinsTest '\"nodes\"' '\"edges\"'" timeout: 10 expected_exit_code: 0 weight: 3.0 @@ -39,17 +42,16 @@ success_criteria: - type: run_command description: "Flow has a connector node referencing uipath-salesforce-sfdc" - command: "python3 -c \"import json,glob; flows=glob.glob('**/SearchableJoinsTest*.flow',recursive=True); assert flows; content=open(flows[0]).read(); assert 'uipath-salesforce-sfdc' in content, 'Connector key not found'; print('OK: connector key present')\"" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py --flow-name SearchableJoinsTest 'uipath-salesforce-sfdc'" timeout: 10 expected_exit_code: 0 weight: 3.0 pass_threshold: 1.0 - - type: command_executed - description: "uip maestro flow validate was called" - tool_name: "Bash" - command_pattern: "(uip|\\$UIP)\\s+(maestro\\s+)?flow\\s+validate" - min_count: 1 + - type: run_command + description: "The completed flow validates" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py" + timeout: 30 + expected_exit_code: 0 weight: 2.0 pass_threshold: 1.0 - diff --git a/tests/tasks/uipath-maestro-flow/connector_features/slack-http-fallback/check_slack_http_fallback.py b/tests/tasks/uipath-maestro-flow/connector_features/slack-http-fallback/check_slack_http_fallback.py index f5bcfe18d5..41b6ed3b21 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/slack-http-fallback/check_slack_http_fallback.py +++ b/tests/tasks/uipath-maestro-flow/connector_features/slack-http-fallback/check_slack_http_fallback.py @@ -23,7 +23,6 @@ from __future__ import annotations -import glob import json import os import sys @@ -37,7 +36,7 @@ break _ROOT = os.path.dirname(_ROOT) sys.path.insert(0, _ROOT) -from _shared.flow_check import run_debug # noqa: E402 +from _shared.flow_check import find_flow_file, run_debug # noqa: E402 FLOW_GLOB = "**/SlackEmojiListTest*.flow" SLACK_KEY = "uipath-salesforce-slack" @@ -53,16 +52,12 @@ def _fail(message: str) -> NoReturn: def _read_flow() -> dict[str, Any]: - flows = sorted(glob.glob(FLOW_GLOB, recursive=True)) - if not flows: - _fail(f"No flow file matching {FLOW_GLOB}") - if len(flows) > 1: - _fail(f"Multiple flows match {FLOW_GLOB}: {flows}") - with open(flows[0], encoding="utf-8") as f: + path = find_flow_file(flow_glob=os.path.basename(FLOW_GLOB)) + with open(path, encoding="utf-8") as f: try: return json.load(f) except json.JSONDecodeError as e: - _fail(f"{flows[0]} is not valid JSON: {e}") + _fail(f"{path} is not valid JSON: {e}") def _normalise_detail(raw: Any) -> dict[str, Any]: @@ -90,7 +85,6 @@ def _is_slack_http_fallback(node: dict[str, Any]) -> bool: HTTP fallback and must not satisfy this check. """ node_type = str(node.get("type") or "").lower() - blob = json.dumps(node).lower() # Shape 2: connector-scoped Slack HTTP request activity. if "http-request" in node_type and SLACK_KEY in node_type: diff --git a/tests/tasks/uipath-maestro-flow/connector_features/slack-http-fallback/slack_http_fallback.yaml b/tests/tasks/uipath-maestro-flow/connector_features/slack-http-fallback/slack_http_fallback.yaml index ed0fd49d7d..d65011fd6c 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/slack-http-fallback/slack_http_fallback.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_features/slack-http-fallback/slack_http_fallback.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-slack-http-fallback description: > E2E test: a catalog connector (Slack, uipath-salesforce-slack) has no native @@ -16,16 +19,15 @@ run_limits: turn_timeout: 1200 initial_prompt: | - Create a new Flow project called "SlackEmojiListTest" with a manual trigger - that lists the custom emoji for a team. Wire the manual trigger to the node that + Create a new Flow project called "SlackEmojiListTest" inside a solution of the + same name, with a manual trigger that lists the custom emoji for a team. Wire + the manual trigger to the node that lists the emoji. Use the Slack connection in the `Shared/uipath-maestro-flow` folder. Validate the final flow file. Once it validates, debug it. - Use `--output json` on every `uip` command whose output you parse. - This run is headless. No user is present and nobody will answer a question or grant an approval, so do not ask, do not pause, and do not wait for input. Complete the task in one pass: take the best available option and supply the @@ -38,25 +40,25 @@ initial_prompt: | decision, assumption, and blocked step in your final response. Instructions in the task take precedence over this paragraph. success_criteria: - - type: command_executed - description: "uip maestro flow validate was called" - tool_name: "Bash" - command_pattern: "(uip|\\$UIP)\\s+(maestro\\s+)?flow\\s+validate" - min_count: 1 + - type: run_command + description: "The completed flow validates" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py" + timeout: 30 + expected_exit_code: 0 weight: 2.0 pass_threshold: 1.0 - type: command_executed - description: "Agent debugged the flow with uip maestro flow debug" + description: "Advisory: live-v1 agent invoked flow debug" tool_name: "Bash" command_pattern: "(uip|\\$UIP)\\s+(maestro\\s+)?flow\\s+debug" min_count: 1 weight: 1.5 - pass_threshold: 1.0 + pass_threshold: 0.0 - type: run_command description: "Emoji list falls back to an HTTP-request node bound to the Slack connector (no native activity exists for it) and targets the emoji.list endpoint" - command: "python3 $TASK_DIR/check_slack_http_fallback.py check_fallback" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/connector_features/slack-http-fallback/check_slack_http_fallback.py check_fallback" timeout: 30 expected_exit_code: 0 weight: 3.0 @@ -64,7 +66,7 @@ success_criteria: - type: run_command description: "uip maestro flow debug finishes with finalStatus Completed against the live Slack connection" - command: "python3 $TASK_DIR/check_slack_http_fallback.py check_debug" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/connector_features/slack-http-fallback/check_slack_http_fallback.py check_debug" timeout: 720 expected_exit_code: 0 weight: 6.0 diff --git a/tests/tasks/uipath-maestro-flow/connector_features/test_check_generate_schema.py b/tests/tasks/uipath-maestro-flow/connector_features/test_check_generate_schema.py new file mode 100644 index 0000000000..a7b070d027 --- /dev/null +++ b/tests/tasks/uipath-maestro-flow/connector_features/test_check_generate_schema.py @@ -0,0 +1,126 @@ +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path + + +CHECKER = Path(__file__).with_name("check_generate_schema.py") +FLOW_CHECK = CHECKER.parents[1] / "_shared" / "flow_check.py" + + +def _flow(*, parent_values: bool = True) -> dict: + parameters = [ + ["fields_sub_project_sub_key", "PROJ" if parent_values else ""], + ["fields_sub_issuetype_sub_id", "10001"], + ] + configuration = { + "essentialConfiguration": { + "customFieldsRequestDetails": { + "objectActionName": "GenerateSchema", + "parameterValues": parameters, + } + } + } + return { + "nodes": [ + {"id": "start", "type": "core.trigger.manual"}, + { + "id": "jira", + "type": "uipath.connector.uipath-atlassian-jira.create-issue", + "inputs": { + "detail": { + "configuration": "=jsonString:" + json.dumps(configuration), + "bodyParameters": {"fields.summary": "Schema test"}, + } + }, + }, + ], + "edges": [{"sourceNodeId": "start", "targetNodeId": "jira"}], + } + + +def _run(cwd: Path) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, str(CHECKER)], + cwd=cwd, + capture_output=True, + text=True, + check=False, + ) + + +def test_accepts_root_sdk_emit_with_arbitrary_filename(tmp_path: Path) -> None: + (tmp_path / "workflow.flow").write_text(json.dumps(_flow()), encoding="utf-8") + + result = _run(tmp_path) + + assert result.returncode == 0, result.stdout + result.stderr + + +def test_accepts_co_located_shared_helper_in_isolated_task_dir(tmp_path: Path) -> None: + task_dir = tmp_path / "task" + shared = task_dir / "_shared" + shared.mkdir(parents=True) + staged_checker = task_dir / CHECKER.name + staged_checker.write_bytes(CHECKER.read_bytes()) + (shared / "__init__.py").write_text("", encoding="utf-8") + (shared / "flow_check.py").write_bytes(FLOW_CHECK.read_bytes()) + workspace = tmp_path / "workspace" + workspace.mkdir() + (workspace / "workflow.flow").write_text(json.dumps(_flow()), encoding="utf-8") + + result = subprocess.run( + [sys.executable, str(staged_checker)], + cwd=workspace, + capture_output=True, + text=True, + check=False, + ) + + assert result.returncode == 0, result.stdout + result.stderr + + +def test_prefers_solution_flow_over_root_scratch_emit(tmp_path: Path) -> None: + project = tmp_path / "GenerateSchemaTestSolution" / "GenerateSchemaTest" + project.mkdir(parents=True) + (project / "project.uiproj").write_text( + json.dumps({"ProjectType": "Flow"}), encoding="utf-8" + ) + (project / "GenerateSchemaTest.flow").write_text( + json.dumps(_flow()), encoding="utf-8" + ) + (tmp_path / "scratch.flow").write_text( + json.dumps({"nodes": [], "edges": []}), encoding="utf-8" + ) + + result = _run(tmp_path) + + assert result.returncode == 0, result.stdout + result.stderr + + +def test_rejects_empty_generate_schema_parent_value(tmp_path: Path) -> None: + (tmp_path / "workflow.flow").write_text( + json.dumps(_flow(parent_values=False)), encoding="utf-8" + ) + + result = _run(tmp_path) + + assert result.returncode != 0 + assert "missing parent values" in result.stderr + + +def test_tokens_outside_jira_configuration_cannot_spoof_result(tmp_path: Path) -> None: + flow = _flow() + jira = flow["nodes"][1] + jira["inputs"]["detail"]["configuration"] = "=jsonString:{}" + flow["description"] = ( + "GenerateSchema fields_sub_project_sub_key fields_sub_issuetype_sub_id" + ) + (tmp_path / "workflow.flow").write_text(json.dumps(flow), encoding="utf-8") + + result = _run(tmp_path) + + assert result.returncode != 0 + assert "essentialConfiguration" in result.stderr diff --git a/tests/tasks/uipath-maestro-flow/connector_features/testmanager_attachments/testmanager_attachments.yaml b/tests/tasks/uipath-maestro-flow/connector_features/testmanager_attachments/testmanager_attachments.yaml index 4cc6dbeb42..1248d4b57f 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/testmanager_attachments/testmanager_attachments.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_features/testmanager_attachments/testmanager_attachments.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. # skip:true, tenant-gated. CONFIRMED available in Maestro Flow (2026-06-16): # `uip maestro flow registry pull` then search returns the # uipath.connector.uipath-uipath-testmanager. nodes for this object group. A stale manifest @@ -8,9 +11,9 @@ description: > Coverage (Maestro Flow connector): agent uses the uipath-maestro-flow skill to build a Flow that uses every UiPath Test Manager connector node for the attachment group — adding one connector node per operation. Drives - `registry pull` -> `registry search`/`get` -> `node add` -> `flow validate`. - Graded on those commands plus the produced .flow referencing each - uipath.connector.uipath-uipath-testmanager.* node type. + connector discovery, authoring, and validation. Graded on the produced .flow + referencing each uipath.connector.uipath-uipath-testmanager.* node type and + validating successfully; registry-refresh telemetry is advisory. tags: [uipath-maestro-flow, e2e, mode:build, lifecycle:generate, shape:multi-node, connector] skip: true @@ -42,32 +45,32 @@ initial_prompt: | the task take precedence over this paragraph. success_criteria: - type: command_executed - description: "Agent refreshed the node manifest (`uip maestro flow registry pull`) before searching" + description: "Advisory: live-v1 agent refreshed the node manifest before searching" tool_name: "Bash" command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+registry\s+pull' min_count: 1 weight: 1.0 - pass_threshold: 1.0 + pass_threshold: 0.0 - - type: command_executed - description: "Agent added connector nodes to the flow (`uip maestro flow node add`)" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+node\s+add' - min_count: 1 + - type: run_command + description: "The flow contains Test Manager connector nodes" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py 'uipath-uipath-testmanager.'" + timeout: 30 + expected_exit_code: 0 weight: 1.5 pass_threshold: 1.0 - - type: command_executed - description: "Agent validated the flow (`uip maestro flow validate`)" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+validate' - min_count: 1 + - type: run_command + description: "The completed flow validates" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py" + timeout: 30 + expected_exit_code: 0 weight: 1.5 pass_threshold: 1.0 - type: run_command description: "Produced .flow references every attachment connector node type (4 nodes)" - command: 'for n in upload-attachment get-attachments download-attachment delete-attachment; do grep -rqsF "uipath-uipath-testmanager.$n" --include="*.flow" . || { echo "missing node $n"; exit 1; }; done' + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py 'uipath-uipath-testmanager.upload-attachment' 'uipath-uipath-testmanager.get-attachments' 'uipath-uipath-testmanager.download-attachment' 'uipath-uipath-testmanager.delete-attachment'" timeout: 30 expected_exit_code: 0 weight: 3.0 diff --git a/tests/tasks/uipath-maestro-flow/connector_features/testmanager_crud_grounded/check.py b/tests/tasks/uipath-maestro-flow/connector_features/testmanager_crud_grounded/check.py index 938a938419..d8e9914d64 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/testmanager_crud_grounded/check.py +++ b/tests/tasks/uipath-maestro-flow/connector_features/testmanager_crud_grounded/check.py @@ -27,6 +27,12 @@ def main(): except OSError: fail("result.json missing — agent did not record the create/get round-trip") + if len(sys.argv) > 1 and sys.argv[1] == "--execution-evidence": + if not all(res.get(key) for key in ("created_name", "retrieved_name", "id")): + fail("result.json is missing created_name, retrieved_name, or id") + print("OK: execution result recorded") + return + name = seed["name"] if res.get("created_name") != name: fail(f"created_name {res.get('created_name')!r} != seeded {name!r}") diff --git a/tests/tasks/uipath-maestro-flow/connector_features/testmanager_crud_grounded/testmanager_crud_grounded.yaml b/tests/tasks/uipath-maestro-flow/connector_features/testmanager_crud_grounded/testmanager_crud_grounded.yaml index eed6d75e0f..55fdc9c22f 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/testmanager_crud_grounded/testmanager_crud_grounded.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_features/testmanager_crud_grounded/testmanager_crud_grounded.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. # PROTOTYPE — data-grounded on the Maestro Flow surface. skip:true. # Agent builds a flow (create-test-case node -> get-test-case node -> output the # retrieved name) and EXECUTES it (`uip maestro flow debug`) to produce the @@ -10,8 +13,9 @@ description: > Data-grounded (Maestro Flow): agent builds a flow that creates a Test Case via the uipath-uipath-testmanager connector node with a unique seeded name, gets it back, outputs the retrieved name, and debugs (executes) the flow — then records - the round-trip. Graded on pull/node-add/validate/debug AND on retrieved == - created (real data through the connector, executed). + the round-trip. Graded on the connector artifacts, successful validation, and + retrieved == created through the executed flow; registry-refresh telemetry is + advisory. tags: [uipath-maestro-flow, e2e, mode:build, lifecycle:generate, shape:multi-node, connector] skip: true @@ -47,40 +51,40 @@ initial_prompt: | the task take precedence over this paragraph. success_criteria: - type: command_executed - description: "Agent refreshed the node manifest (`uip maestro flow registry pull`)" + description: "Advisory: live-v1 agent refreshed the node manifest" tool_name: "Bash" command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+registry\s+pull' min_count: 1 weight: 1.0 - pass_threshold: 1.0 + pass_threshold: 0.0 - - type: command_executed - description: "Agent added connector nodes (`uip maestro flow node add`)" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+node\s+add' - min_count: 1 + - type: run_command + description: "The flow contains Test Manager create and get connector nodes" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py 'uipath-uipath-testmanager.create-test-case' 'uipath-uipath-testmanager.get-test-case'" + timeout: 30 + expected_exit_code: 0 weight: 1.0 pass_threshold: 1.0 - - type: command_executed - description: "Agent validated the flow (`uip maestro flow validate`)" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+validate' - min_count: 1 + - type: run_command + description: "The completed flow validates" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py" + timeout: 30 + expected_exit_code: 0 weight: 1.0 pass_threshold: 1.0 - - type: command_executed - description: "Agent executed the flow (`uip maestro flow debug`)" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+debug' - min_count: 1 + - type: run_command + description: "Execution produced the seeded create/get round-trip result" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/connector_features/testmanager_crud_grounded/check.py --execution-evidence" + timeout: 30 + expected_exit_code: 0 weight: 1.5 pass_threshold: 1.0 - type: run_command description: "Data-grounded: retrieved name == created (seeded) name — real data round-tripped through the executed flow" - command: "python3 $TASK_DIR/check.py" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/connector_features/testmanager_crud_grounded/check.py" timeout: 30 expected_exit_code: 0 weight: 5.0 diff --git a/tests/tasks/uipath-maestro-flow/connector_features/testmanager_execution_results/testmanager_execution_results.yaml b/tests/tasks/uipath-maestro-flow/connector_features/testmanager_execution_results/testmanager_execution_results.yaml index 59f2c80c61..94521e5972 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/testmanager_execution_results/testmanager_execution_results.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_features/testmanager_execution_results/testmanager_execution_results.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. # skip:true, tenant-gated. CONFIRMED available in Maestro Flow (2026-06-16): # `uip maestro flow registry pull` then search returns the # uipath.connector.uipath-uipath-testmanager. nodes for this object group. A stale manifest @@ -8,9 +11,9 @@ description: > Coverage (Maestro Flow connector): agent uses the uipath-maestro-flow skill to build a Flow that uses every UiPath Test Manager connector node for the execution and results group — adding one connector node per operation. Drives - `registry pull` -> `registry search`/`get` -> `node add` -> `flow validate`. - Graded on those commands plus the produced .flow referencing each - uipath.connector.uipath-uipath-testmanager.* node type. + connector discovery, authoring, and validation. Graded on the produced .flow + referencing each uipath.connector.uipath-uipath-testmanager.* node type and + validating successfully; registry-refresh telemetry is advisory. tags: [uipath-maestro-flow, e2e, mode:build, lifecycle:generate, shape:multi-node, connector] skip: true @@ -47,32 +50,32 @@ initial_prompt: | the task take precedence over this paragraph. success_criteria: - type: command_executed - description: "Agent refreshed the node manifest (`uip maestro flow registry pull`) before searching" + description: "Advisory: live-v1 agent refreshed the node manifest before searching" tool_name: "Bash" command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+registry\s+pull' min_count: 1 weight: 1.0 - pass_threshold: 1.0 + pass_threshold: 0.0 - - type: command_executed - description: "Agent added connector nodes to the flow (`uip maestro flow node add`)" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+node\s+add' - min_count: 1 + - type: run_command + description: "The flow contains Test Manager connector nodes" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py 'uipath-uipath-testmanager.'" + timeout: 30 + expected_exit_code: 0 weight: 1.5 pass_threshold: 1.0 - - type: command_executed - description: "Agent validated the flow (`uip maestro flow validate`)" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+validate' - min_count: 1 + - type: run_command + description: "The completed flow validates" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py" + timeout: 30 + expected_exit_code: 0 weight: 1.5 pass_threshold: 1.0 - type: run_command description: "Produced .flow references every execution and results connector node type (9 nodes)" - command: 'for n in get-test-execution delete-test-execution get-test-case-logs-of-test-execution get-test-case-log get-robot-logs get-assertions download-assertion get-test-steps get-test-step-logs; do grep -rqsF "uipath-uipath-testmanager.$n" --include="*.flow" . || { echo "missing node $n"; exit 1; }; done' + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py 'uipath-uipath-testmanager.get-test-execution' 'uipath-uipath-testmanager.delete-test-execution' 'uipath-uipath-testmanager.get-test-case-logs-of-test-execution' 'uipath-uipath-testmanager.get-test-case-log' 'uipath-uipath-testmanager.get-robot-logs' 'uipath-uipath-testmanager.get-assertions' 'uipath-uipath-testmanager.download-assertion' 'uipath-uipath-testmanager.get-test-steps' 'uipath-uipath-testmanager.get-test-step-logs'" timeout: 30 expected_exit_code: 0 weight: 3.0 diff --git a/tests/tasks/uipath-maestro-flow/connector_features/testmanager_generic_records/testmanager_generic_records.yaml b/tests/tasks/uipath-maestro-flow/connector_features/testmanager_generic_records/testmanager_generic_records.yaml index 1f697bc8ec..3f4d9decbc 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/testmanager_generic_records/testmanager_generic_records.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_features/testmanager_generic_records/testmanager_generic_records.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. # skip:true, tenant-gated. CONFIRMED available in Maestro Flow (2026-06-16): # `uip maestro flow registry pull` then search returns the # uipath.connector.uipath-uipath-testmanager. nodes for this object group. A stale manifest @@ -8,9 +11,9 @@ description: > Coverage (Maestro Flow connector): agent uses the uipath-maestro-flow skill to build a Flow that uses every UiPath Test Manager connector node for the generic record group — adding one connector node per operation. Drives - `registry pull` -> `registry search`/`get` -> `node add` -> `flow validate`. - Graded on those commands plus the produced .flow referencing each - uipath.connector.uipath-uipath-testmanager.* node type. + connector discovery, authoring, and validation. Graded on the produced .flow + referencing each uipath.connector.uipath-uipath-testmanager.* node type and + validating successfully; registry-refresh telemetry is advisory. tags: [uipath-maestro-flow, e2e, mode:build, lifecycle:generate, shape:multi-node, connector] skip: true @@ -43,32 +46,32 @@ initial_prompt: | the task take precedence over this paragraph. success_criteria: - type: command_executed - description: "Agent refreshed the node manifest (`uip maestro flow registry pull`) before searching" + description: "Advisory: live-v1 agent refreshed the node manifest before searching" tool_name: "Bash" command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+registry\s+pull' min_count: 1 weight: 1.0 - pass_threshold: 1.0 + pass_threshold: 0.0 - - type: command_executed - description: "Agent added connector nodes to the flow (`uip maestro flow node add`)" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+node\s+add' - min_count: 1 + - type: run_command + description: "The flow contains Test Manager connector nodes" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py 'uipath-uipath-testmanager.'" + timeout: 30 + expected_exit_code: 0 weight: 1.5 pass_threshold: 1.0 - - type: command_executed - description: "Agent validated the flow (`uip maestro flow validate`)" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+validate' - min_count: 1 + - type: run_command + description: "The completed flow validates" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py" + timeout: 30 + expected_exit_code: 0 weight: 1.5 pass_threshold: 1.0 - type: run_command description: "Produced .flow references every generic record connector node type (5 nodes)" - command: 'for n in insert-record get-record update-record delete-record list-records; do grep -rqsF "uipath-uipath-testmanager.$n" --include="*.flow" . || { echo "missing node $n"; exit 1; }; done' + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py 'uipath-uipath-testmanager.insert-record' 'uipath-uipath-testmanager.get-record' 'uipath-uipath-testmanager.update-record' 'uipath-uipath-testmanager.delete-record' 'uipath-uipath-testmanager.list-records'" timeout: 30 expected_exit_code: 0 weight: 3.0 diff --git a/tests/tasks/uipath-maestro-flow/connector_features/testmanager_requirement_lifecycle/testmanager_requirement_lifecycle.yaml b/tests/tasks/uipath-maestro-flow/connector_features/testmanager_requirement_lifecycle/testmanager_requirement_lifecycle.yaml index 64a26a2c02..62df61f296 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/testmanager_requirement_lifecycle/testmanager_requirement_lifecycle.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_features/testmanager_requirement_lifecycle/testmanager_requirement_lifecycle.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. # skip:true, tenant-gated. CONFIRMED available in Maestro Flow (2026-06-16): # `uip maestro flow registry pull` then search returns the # uipath.connector.uipath-uipath-testmanager. nodes for this object group. A stale manifest @@ -8,9 +11,9 @@ description: > Coverage (Maestro Flow connector): agent uses the uipath-maestro-flow skill to build a Flow that uses every UiPath Test Manager connector node for the Requirement group — adding one connector node per operation. Drives - `registry pull` -> `registry search`/`get` -> `node add` -> `flow validate`. - Graded on those commands plus the produced .flow referencing each - uipath.connector.uipath-uipath-testmanager.* node type. + connector discovery, authoring, and validation. Graded on the produced .flow + referencing each uipath.connector.uipath-uipath-testmanager.* node type and + validating successfully; registry-refresh telemetry is advisory. tags: [uipath-maestro-flow, e2e, mode:build, lifecycle:generate, shape:multi-node, connector] skip: true @@ -44,32 +47,32 @@ initial_prompt: | the task take precedence over this paragraph. success_criteria: - type: command_executed - description: "Agent refreshed the node manifest (`uip maestro flow registry pull`) before searching" + description: "Advisory: live-v1 agent refreshed the node manifest before searching" tool_name: "Bash" command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+registry\s+pull' min_count: 1 weight: 1.0 - pass_threshold: 1.0 + pass_threshold: 0.0 - - type: command_executed - description: "Agent added connector nodes to the flow (`uip maestro flow node add`)" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+node\s+add' - min_count: 1 + - type: run_command + description: "The flow contains Test Manager connector nodes" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py 'uipath-uipath-testmanager.'" + timeout: 30 + expected_exit_code: 0 weight: 1.5 pass_threshold: 1.0 - - type: command_executed - description: "Agent validated the flow (`uip maestro flow validate`)" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+validate' - min_count: 1 + - type: run_command + description: "The completed flow validates" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py" + timeout: 30 + expected_exit_code: 0 weight: 1.5 pass_threshold: 1.0 - type: run_command description: "Produced .flow references every Requirement connector node type (6 nodes)" - command: 'for n in create-requirement get-requirement update-requirement delete-requirement assign-test-cases-to-requirement get-assigned-test-cases-for-req; do grep -rqsF "uipath-uipath-testmanager.$n" --include="*.flow" . || { echo "missing node $n"; exit 1; }; done' + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py 'uipath-uipath-testmanager.create-requirement' 'uipath-uipath-testmanager.get-requirement' 'uipath-uipath-testmanager.update-requirement' 'uipath-uipath-testmanager.delete-requirement' 'uipath-uipath-testmanager.assign-test-cases-to-requirement' 'uipath-uipath-testmanager.get-assigned-test-cases-for-req'" timeout: 30 expected_exit_code: 0 weight: 3.0 diff --git a/tests/tasks/uipath-maestro-flow/connector_features/testmanager_testcase_lifecycle/testmanager_testcase_lifecycle.yaml b/tests/tasks/uipath-maestro-flow/connector_features/testmanager_testcase_lifecycle/testmanager_testcase_lifecycle.yaml index e659c59a5f..a5d3687859 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/testmanager_testcase_lifecycle/testmanager_testcase_lifecycle.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_features/testmanager_testcase_lifecycle/testmanager_testcase_lifecycle.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. # skip:true, tenant-gated. CONFIRMED available in Maestro Flow (2026-06-16): # `uip maestro flow registry pull` then search returns the # uipath.connector.uipath-uipath-testmanager. nodes for this object group. A stale manifest @@ -8,9 +11,9 @@ description: > Coverage (Maestro Flow connector): agent uses the uipath-maestro-flow skill to build a Flow that uses every UiPath Test Manager connector node for the TestCase group — adding one connector node per operation. Drives - `registry pull` -> `registry search`/`get` -> `node add` -> `flow validate`. - Graded on those commands plus the produced .flow referencing each - uipath.connector.uipath-uipath-testmanager.* node type. + connector discovery, authoring, and validation. Graded on the produced .flow + referencing each uipath.connector.uipath-uipath-testmanager.* node type and + validating successfully; registry-refresh telemetry is advisory. tags: [uipath-maestro-flow, e2e, mode:build, lifecycle:generate, shape:multi-node, connector] skip: true @@ -43,32 +46,32 @@ initial_prompt: | the task take precedence over this paragraph. success_criteria: - type: command_executed - description: "Agent refreshed the node manifest (`uip maestro flow registry pull`) before searching" + description: "Advisory: live-v1 agent refreshed the node manifest before searching" tool_name: "Bash" command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+registry\s+pull' min_count: 1 weight: 1.0 - pass_threshold: 1.0 + pass_threshold: 0.0 - - type: command_executed - description: "Agent added connector nodes to the flow (`uip maestro flow node add`)" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+node\s+add' - min_count: 1 + - type: run_command + description: "The flow contains Test Manager connector nodes" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py 'uipath-uipath-testmanager.'" + timeout: 30 + expected_exit_code: 0 weight: 1.5 pass_threshold: 1.0 - - type: command_executed - description: "Agent validated the flow (`uip maestro flow validate`)" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+validate' - min_count: 1 + - type: run_command + description: "The completed flow validates" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py" + timeout: 30 + expected_exit_code: 0 weight: 1.5 pass_threshold: 1.0 - type: run_command description: "Produced .flow references every TestCase connector node type (5 nodes)" - command: 'for n in create-test-case get-test-case update-test-case delete-test-case execute-test-cases; do grep -rqsF "uipath-uipath-testmanager.$n" --include="*.flow" . || { echo "missing node $n"; exit 1; }; done' + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py 'uipath-uipath-testmanager.create-test-case' 'uipath-uipath-testmanager.get-test-case' 'uipath-uipath-testmanager.update-test-case' 'uipath-uipath-testmanager.delete-test-case' 'uipath-uipath-testmanager.execute-test-cases'" timeout: 30 expected_exit_code: 0 weight: 3.0 diff --git a/tests/tasks/uipath-maestro-flow/connector_features/testmanager_testset_lifecycle/testmanager_testset_lifecycle.yaml b/tests/tasks/uipath-maestro-flow/connector_features/testmanager_testset_lifecycle/testmanager_testset_lifecycle.yaml index a30e0dd5b9..474a14c42f 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/testmanager_testset_lifecycle/testmanager_testset_lifecycle.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_features/testmanager_testset_lifecycle/testmanager_testset_lifecycle.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. # skip:true, tenant-gated. CONFIRMED available in Maestro Flow (2026-06-16): # `uip maestro flow registry pull` then search returns the # uipath.connector.uipath-uipath-testmanager. nodes for this object group. A stale manifest @@ -8,9 +11,9 @@ description: > Coverage (Maestro Flow connector): agent uses the uipath-maestro-flow skill to build a Flow that uses every UiPath Test Manager connector node for the TestSet group — adding one connector node per operation. Drives - `registry pull` -> `registry search`/`get` -> `node add` -> `flow validate`. - Graded on those commands plus the produced .flow referencing each - uipath.connector.uipath-uipath-testmanager.* node type. + connector discovery, authoring, and validation. Graded on the produced .flow + referencing each uipath.connector.uipath-uipath-testmanager.* node type and + validating successfully; registry-refresh telemetry is advisory. tags: [uipath-maestro-flow, e2e, mode:build, lifecycle:generate, shape:multi-node, connector] skip: true @@ -45,32 +48,32 @@ initial_prompt: | the task take precedence over this paragraph. success_criteria: - type: command_executed - description: "Agent refreshed the node manifest (`uip maestro flow registry pull`) before searching" + description: "Advisory: live-v1 agent refreshed the node manifest before searching" tool_name: "Bash" command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+registry\s+pull' min_count: 1 weight: 1.0 - pass_threshold: 1.0 + pass_threshold: 0.0 - - type: command_executed - description: "Agent added connector nodes to the flow (`uip maestro flow node add`)" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+node\s+add' - min_count: 1 + - type: run_command + description: "The flow contains Test Manager connector nodes" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py 'uipath-uipath-testmanager.'" + timeout: 30 + expected_exit_code: 0 weight: 1.5 pass_threshold: 1.0 - - type: command_executed - description: "Agent validated the flow (`uip maestro flow validate`)" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+validate' - min_count: 1 + - type: run_command + description: "The completed flow validates" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py" + timeout: 30 + expected_exit_code: 0 weight: 1.5 pass_threshold: 1.0 - type: run_command description: "Produced .flow references every TestSet connector node type (7 nodes)" - command: 'for n in create-test-set get-test-set update-test-set delete-test-set assign-test-cases-to-test-set get-assigned-test-cases-for-test-set execute-test-set; do grep -rqsF "uipath-uipath-testmanager.$n" --include="*.flow" . || { echo "missing node $n"; exit 1; }; done' + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py 'uipath-uipath-testmanager.create-test-set' 'uipath-uipath-testmanager.get-test-set' 'uipath-uipath-testmanager.update-test-set' 'uipath-uipath-testmanager.delete-test-set' 'uipath-uipath-testmanager.assign-test-cases-to-test-set' 'uipath-uipath-testmanager.get-assigned-test-cases-for-test-set' 'uipath-uipath-testmanager.execute-test-set'" timeout: 30 expected_exit_code: 0 weight: 3.0 diff --git a/tests/tasks/uipath-maestro-flow/connector_trigger/check_webhook_waitfor_parallel.py b/tests/tasks/uipath-maestro-flow/connector_trigger/check_webhook_waitfor_parallel.py index 1b81347fb7..c8a894c895 100644 --- a/tests/tasks/uipath-maestro-flow/connector_trigger/check_webhook_waitfor_parallel.py +++ b/tests/tasks/uipath-maestro-flow/connector_trigger/check_webhook_waitfor_parallel.py @@ -5,7 +5,8 @@ manual start trigger ├─ branch 1: Wait-for-event node (HTTP Webhook, uipath-http-webhook) -> End - └─ branch 2: Managed HTTP Request (core.action.http.v2, manual GET to the + └─ branch 2: Managed HTTP Request (core.action.http or + core.action.http.v2, manual GET to the HTTP Webhook connection's webhook URL, no headers / no query) -> End Branch 2's GET hits the webhook URL, which delivers the event that completes @@ -17,20 +18,24 @@ branches (the event wait and the HTTP request). 2. A Wait-for-event node of the HTTP Webhook connector event exists (`uipath.connector.event.uipath-http-webhook.http-webhook`). - 3. A `core.action.http.v2` node is a MANUAL `GET` whose `url` is the webhook - URL, with NOTHING in headers or query (per the prompt). + 3. A `core.action.http` or `core.action.http.v2` node is a MANUAL `GET` whose + `url` is the webhook URL, with NOTHING in headers or query (per the + prompt). 4. Both branch tails reach an End node (`core.control.end`). """ from __future__ import annotations -import glob import json +import os import sys +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) +from _shared.flow_check import find_flow_file # noqa: E402 + FLOW_GLOB = "**/WebhookSelfTest*.flow" EVENT_MARKERS = ("uipath.connector.event", "uipath-http-webhook") -HTTP_TYPE = "core.action.http.v2" +HTTP_TYPES = ("core.action.http", "core.action.http.v2") END_TYPE = "core.control.end" TRIGGER_PREFIX = "core.trigger." @@ -40,10 +45,8 @@ def _fail(msg: str) -> None: def _read_flow() -> dict: - flows = glob.glob(FLOW_GLOB, recursive=True) - if not flows: - _fail(f"no flow file matching {FLOW_GLOB} under cwd") - with open(flows[0], encoding="utf-8") as f: + path = find_flow_file(flow_glob=FLOW_GLOB) + with open(path, encoding="utf-8") as f: return json.load(f) @@ -83,7 +86,8 @@ def main() -> None: # 2. HTTP Webhook Wait-for-event node present. event_nodes = [ - n for n in nodes + n + for n in nodes if all(m in str(n.get("type", "")).lower() for m in EVENT_MARKERS) ] if not event_nodes: @@ -94,20 +98,33 @@ def main() -> None: print("OK: HTTP Webhook wait-for-event node present") # 3. Manual GET HTTP node to the webhook URL, no headers / no query. - http_nodes = [n for n in nodes if str(n.get("type", "")) == HTTP_TYPE] + http_nodes = [n for n in nodes if str(n.get("type", "")) in HTTP_TYPES] if not http_nodes: - _fail(f"no {HTTP_TYPE} node. Types seen: {types_seen}") + _fail( + f"no managed HTTP node ({' or '.join(HTTP_TYPES)}). " + f"Types seen: {types_seen}" + ) good_http = None for n in http_nodes: - body = ((n.get("inputs") or {}).get("detail") or {}).get("bodyParameters") or {} + inputs = n.get("inputs") or {} + body = ((inputs.get("detail") or {}).get("bodyParameters") or inputs) if not isinstance(body, dict): continue - auth = str(body.get("authentication", "")).lower() + auth = str( + body.get("authentication") + or body.get("authenticationType") + or body.get("mode") + or "" + ).lower() method = str(body.get("method", "")).lower() url = str(body.get("url", "")) headers = body.get("headers") - query = body.get("query") or body.get("queryParameters") + query = ( + body.get("query") + or body.get("queryParameters") + or body.get("queryParams") + ) if auth != "manual" or method != "get": continue if "webhook" not in url.lower(): @@ -121,7 +138,7 @@ def main() -> None: if good_http is None: _fail( - "no core.action.http.v2 node configured as a manual GET to the " + "no managed HTTP node configured as a manual GET to the " "webhook URL (authentication=manual, method=GET, url contains 'webhook')" ) print("OK: manual GET HTTP node to webhook URL, no headers / no query") @@ -130,7 +147,10 @@ def main() -> None: end_ids = {n.get("id") for n in nodes if str(n.get("type", "")) == END_TYPE} if not end_ids: _fail(f"no End node ({END_TYPE}). Types seen: {types_seen}") - for label, node in (("wait-for-event", event_nodes[0]), ("http-request", good_http)): + for label, node in ( + ("wait-for-event", event_nodes[0]), + ("http-request", good_http), + ): reach = _reachable(node.get("id"), edges) if not (reach & end_ids): _fail(f"{label} branch does not reach an End node") diff --git a/tests/tasks/uipath-maestro-flow/connector_trigger/test_check_webhook_waitfor_parallel.py b/tests/tasks/uipath-maestro-flow/connector_trigger/test_check_webhook_waitfor_parallel.py new file mode 100644 index 0000000000..b68cb49e9c --- /dev/null +++ b/tests/tasks/uipath-maestro-flow/connector_trigger/test_check_webhook_waitfor_parallel.py @@ -0,0 +1,58 @@ +import importlib.util +import json +from pathlib import Path + +import pytest + + +CHECKER_PATH = Path(__file__).with_name("check_webhook_waitfor_parallel.py") +SPEC = importlib.util.spec_from_file_location("check_webhook_waitfor_parallel", CHECKER_PATH) +assert SPEC and SPEC.loader +CHECKER = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(CHECKER) + + +@pytest.mark.parametrize("http_type", ["core.action.http", "core.action.http.v2"]) +def test_accepts_both_managed_http_representations(tmp_path, monkeypatch, http_type): + monkeypatch.chdir(tmp_path) + if http_type.endswith(".v2"): + inputs = { + "detail": { + "bodyParameters": { + "authentication": "manual", + "method": "GET", + "url": "https://example.test/webhook/events/1", + "headers": {}, + "query": {}, + } + } + } + else: + inputs = { + "authenticationType": "manual", + "method": "GET", + "url": "https://example.test/webhook/events/1", + "headers": {}, + "queryParams": {}, + } + + flow = { + "nodes": [ + {"id": "start", "type": "core.trigger.manual"}, + { + "id": "wait", + "type": "uipath.connector.event.uipath-http-webhook.http-webhook", + }, + {"id": "get", "type": http_type, "inputs": inputs}, + {"id": "end", "type": "core.control.end"}, + ], + "edges": [ + {"sourceNodeId": "start", "targetNodeId": "wait"}, + {"sourceNodeId": "start", "targetNodeId": "get"}, + {"sourceNodeId": "wait", "targetNodeId": "end"}, + {"sourceNodeId": "get", "targetNodeId": "end"}, + ], + } + (tmp_path / "WebhookSelfTest.flow").write_text(json.dumps(flow)) + + CHECKER.main() diff --git a/tests/tasks/uipath-maestro-flow/connector_trigger/trigger_with_filter.yaml b/tests/tasks/uipath-maestro-flow/connector_trigger/trigger_with_filter.yaml index 69e2d5e857..7696a56a9b 100644 --- a/tests/tasks/uipath-maestro-flow/connector_trigger/trigger_with_filter.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_trigger/trigger_with_filter.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-trigger-with-filter description: > Verifies that the uipath-maestro-flow skill teaches agents to emit a structured `filter` tree. @@ -12,8 +15,11 @@ initial_prompt: | abc@xyz.com. Do NOT run any command that requires a live connection — this sandbox has no UiPath tenant, so you cannot configure the trigger against a live connection. - Instead, write to `trigger_detail.json` the exact JSON object you would pass to - `uip maestro flow node configure --detail ''` to configure that trigger. + Instead, write the exact connector-trigger detail object to `trigger_detail.json`. + It must contain the structured filter tree that configures this trigger. + The file itself is the `--detail` value: put `filter` at the top level, next + to fields such as `connectionId` and `folderKey`. Do not wrap that value in + an outer `detail`, `inputs`, or node-descriptor object. Use placeholders for any connection/folder/event ids you cannot resolve offline. This run is headless. No user is present and nobody will answer a question or @@ -36,8 +42,8 @@ success_criteria: # wrapped under detail/inputs fails — that is not the --detail object. - type: run_command description: "trigger_detail.json is the --detail object with a top-level `filter` tree on `subject` Contains, no top-level filterExpression" - command: "python3 $TASK_DIR/check_trigger_filter.py" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/connector_trigger/check_trigger_filter.py" timeout: 10 expected_exit_code: 0 weight: 3.0 - pass_threshold: 1.0 \ No newline at end of file + pass_threshold: 1.0 diff --git a/tests/tasks/uipath-maestro-flow/connector_trigger/webhook_waitfor_parallel.yaml b/tests/tasks/uipath-maestro-flow/connector_trigger/webhook_waitfor_parallel.yaml index 51ae0062b8..d39ffce855 100644 --- a/tests/tasks/uipath-maestro-flow/connector_trigger/webhook_waitfor_parallel.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_trigger/webhook_waitfor_parallel.yaml @@ -1,14 +1,18 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-webhook-waitfor-parallel description: > E2E self-testing flow: a manual start trigger fans out into two parallel branches. Branch 1 is a mid-flow Wait-for-event node bound to the HTTP Webhook connector (`uipath.connector.event.uipath-http-webhook.http-webhook`). Branch 2 - is a Managed HTTP Request (`core.action.http.v2`, manual GET) whose URL is the - webhook URL of that same HTTP Webhook connection, with nothing in headers or - query. The GET self-delivers the event that completes the wait at runtime. + is a Managed HTTP Request (`core.action.http` or `core.action.http.v2`, manual + GET) whose URL is the webhook URL of that same HTTP Webhook connection, with + nothing in headers or query. The GET self-delivers the event that completes + the wait at runtime. Exercises the connector-trigger plugin's "Wait for events" variant, parallel branch fan-out from the start trigger, manual-mode HTTP, and the CLI - webhook-URL retrieval path. Validates the static two-branch shape. + webhook-URL wiring. Validates the static two-branch shape. tags: [ uipath-maestro-flow, @@ -45,8 +49,6 @@ initial_prompt: | After building, validate the flow. - Use `--output json` on `uip` commands. - This run is headless. No user is present and nobody will answer a question or grant an approval, so do not ask, do not pause, and do not wait for input. Complete the task in one pass: take the best available option and supply the @@ -60,40 +62,40 @@ initial_prompt: | the task take precedence over this paragraph. success_criteria: - type: command_executed - description: "Agent created a solution" + description: "Solution-creation telemetry (report only; the Flow artifact is graded below)" tool_name: "Bash" command_pattern: '(uip|\$UIP)\s+solution\s+(new|init)' min_count: 1 weight: 1.0 - pass_threshold: 1.0 + pass_threshold: 0.0 - - type: command_executed - description: "Agent initialized a Flow project" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+(maestro\s+)?flow\s+init' - min_count: 1 + - type: run_command + description: "WebhookSelfTest Flow artifact exists" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py --flow-name WebhookSelfTest" + timeout: 30 + expected_exit_code: 0 weight: 1.0 pass_threshold: 1.0 - type: command_executed - description: "Agent retrieved the webhook URL" + description: "Webhook-URL retrieval telemetry (report only; the final artifact grades URL wiring)" tool_name: "Bash" command_pattern: '(uip|\$UIP)\s+is\s+webhooks\s+config' min_count: 1 weight: 1.0 - pass_threshold: 1.0 + pass_threshold: 0.0 - - type: command_executed - description: "Agent validated the flow" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+(maestro\s+)?flow\s+validate' - min_count: 1 + - type: run_command + description: "Flow validates successfully" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py" + timeout: 30 + expected_exit_code: 0 weight: 1.5 pass_threshold: 1.0 - type: run_command description: "Two-branch structure: wait-for-event + manual-GET-to-webhook-URL (no headers/query), both reaching End" - command: "python3 $TASK_DIR/check_webhook_waitfor_parallel.py" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/connector_trigger/check_webhook_waitfor_parallel.py" timeout: 120 expected_exit_code: 0 weight: 3.0 diff --git a/tests/tasks/uipath-maestro-flow/context-grounding/batch_transform/batch_transform.yaml b/tests/tasks/uipath-maestro-flow/context-grounding/batch_transform/batch_transform.yaml index 7a72011768..320346b191 100644 --- a/tests/tasks/uipath-maestro-flow/context-grounding/batch_transform/batch_transform.yaml +++ b/tests/tasks/uipath-maestro-flow/context-grounding/batch_transform/batch_transform.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-batch-transform description: > Create a UiPath Flow that runs a Batch Transform pattern node over a CSV @@ -41,7 +44,7 @@ initial_prompt: | the result file handle as a flow output variable named `result` mapped via `outputs.result.source: "=js:$vars..output"`. - Do NOT run flow debug — just validate the flow. + Validate the completed flow. Before starting, follow the documented workflow steps exactly — in particular, the batch-transform plugin's `outputColumns` shape (array of `{ name, description }`). @@ -69,7 +72,7 @@ success_criteria: # ── Structural checks (node type, inputs, outputColumns shape) ───────── - type: run_command description: "Flow has a Batch Transform node with required inputs and the {name,description} outputColumns shape" - command: "python3 $TASK_DIR/check_batch_transform_flow.py" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/context-grounding/batch_transform/check_batch_transform_flow.py" timeout: 30 expected_exit_code: 0 weight: 5.0 diff --git a/tests/tasks/uipath-maestro-flow/context-grounding/batch_transform/check_batch_transform_flow.py b/tests/tasks/uipath-maestro-flow/context-grounding/batch_transform/check_batch_transform_flow.py index 430792d492..64f49330c3 100644 --- a/tests/tasks/uipath-maestro-flow/context-grounding/batch_transform/check_batch_transform_flow.py +++ b/tests/tasks/uipath-maestro-flow/context-grounding/batch_transform/check_batch_transform_flow.py @@ -10,9 +10,10 @@ 4. `inputs.prompt` is non-empty. 5. `inputs.outputColumns` is an array of objects with `name` + `description` keys (not flattened to a `{name: description}` map and not a string array). - 6. `inputs.attachment` matches the canonical canvas-produced wiring: - `=js:$vars..output.` — referencing a flow `in` - variable of `type: "file"` bound to the trigger via `triggerNodeId`. + 6. `inputs.attachment` matches the canonical trigger-output wiring, either + as the v1 string `=js:$vars..output.` or the SDK's + equivalent typed expression envelope — referencing a flow `in` variable + of `type: "file"` bound to the trigger via `triggerNodeId`. Bare `=js:$vars.` (no trigger output) is rejected. 7. The instance `outputs.output.source` is the literal `=response`. 8. The flow declares an `out` variable named `result`, and at least one End @@ -20,12 +21,16 @@ reference. """ -import glob import json +import os import re import sys from typing import NoReturn +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..")) +from _shared.advisory_flow_utils import unwrap # noqa: E402 +from _shared.flow_check import find_flow_file # noqa: E402 + NODE_TYPE = "uipath.pattern.batch-transform" EXPECTED_TYPE_VERSION = "1.0" EXPECTED_OUTPUT_SOURCE = "=response" @@ -33,11 +38,11 @@ # Canonical canvas wiring: =js:$vars..output. # (a trigger-bound `in` file variable surfaces under the trigger's output) _ATTACHMENT_REF = re.compile( - r"^=js:\s*\$vars\.([A-Za-z_][A-Za-z0-9_]*)\.output\.([A-Za-z_][A-Za-z0-9_]*)\s*$" + r"^\$vars\.([A-Za-z_][A-Za-z0-9_]*)\.output\.([A-Za-z_][A-Za-z0-9_]*)\s*$" ) # Output mapping: =js:$vars..output (or any deeper subfield path) -_OUTPUT_MAPPING_REF = re.compile(r"^=js:\s*\$vars\.([A-Za-z_][A-Za-z0-9_]*)\.output\b") +_OUTPUT_MAPPING_REF = re.compile(r"^\$vars\.([A-Za-z_][A-Za-z0-9_]*)\.output\b") def _fail(msg: str) -> NoReturn: @@ -45,10 +50,8 @@ def _fail(msg: str) -> NoReturn: def _read_flow() -> dict: - flows = glob.glob("**/BatchTransformDemo*.flow", recursive=True) - if not flows: - _fail("no BatchTransformDemo*.flow found under cwd") - with open(flows[0]) as f: + path = find_flow_file(flow_glob="BatchTransformDemo*.flow") + with open(path) as f: return json.load(f) @@ -84,10 +87,23 @@ def _check_outputColumns(value) -> None: ) +def _js_expression(value) -> str | None: + if isinstance(value, str) and value.startswith("=js:"): + return value[len("=js:") :].strip() + if isinstance(value, dict) and value.get("type") == "jsExpression": + expression = value.get("expression") + if isinstance(expression, str) and expression.strip(): + return expression.strip() + return None + + def _check_attachment_canonical(flow: dict, attachment) -> None: - if not isinstance(attachment, str) or not attachment.strip(): - _fail("inputs.attachment missing or empty — wire it to the trigger output binding") - m = _ATTACHMENT_REF.match(attachment.strip()) + expression = _js_expression(attachment) + if expression is None: + _fail( + "inputs.attachment missing or empty — wire it to the trigger output binding" + ) + m = _ATTACHMENT_REF.match(expression) if not m: _fail( f"inputs.attachment={attachment!r} is not the canonical canvas-produced shape. " @@ -107,7 +123,10 @@ def _check_attachment_canonical(flow: dict, attachment) -> None: f"inputs.attachment references `$vars.{trigger_id}.output...` but no node with " f"id={trigger_id!r} exists in the flow." ) - if not isinstance(trigger_node.get("type"), str) or "trigger" not in trigger_node["type"]: + if ( + not isinstance(trigger_node.get("type"), str) + or "trigger" not in trigger_node["type"] + ): _fail( f"node id={trigger_id!r} has type={trigger_node.get('type')!r}; expected a trigger " "(file-typed input variables must be trigger-bound)." @@ -157,7 +176,7 @@ def _check_type_version(node: dict) -> None: def _check_output_source(node: dict) -> None: - src = ((node.get("outputs") or {}).get("output") or {}).get("source") + src = unwrap(((node.get("outputs") or {}).get("output") or {}).get("source")) if src != EXPECTED_OUTPUT_SOURCE: _fail( f"outputs.output.source={src!r}; must be {EXPECTED_OUTPUT_SOURCE!r} (the BPMN " @@ -169,7 +188,11 @@ def _check_output_source(node: dict) -> None: def _check_result_output_mapping(flow: dict, bt_node_id: str) -> None: globals_ = (flow.get("variables") or {}).get("globals") or [] result_var = next( - (v for v in globals_ if v.get("id") == "result" and v.get("direction") == "out"), + ( + v + for v in globals_ + if v.get("id") == "result" and v.get("direction") == "out" + ), None, ) if result_var is None: @@ -178,15 +201,18 @@ def _check_result_output_mapping(flow: dict, bt_node_id: str) -> None: "Batch Transform result file handle to be surfaced as a flow output named `result`." ) - end_nodes = [n for n in flow.get("nodes", []) if n.get("type") == "core.control.end"] + end_nodes = [ + n for n in flow.get("nodes", []) if n.get("type") == "core.control.end" + ] if not end_nodes: _fail("flow has no End node — required to terminate the path and map outputs") for end in end_nodes: src = ((end.get("outputs") or {}).get("result") or {}).get("source") - if not isinstance(src, str) or not src.strip(): + expression = _js_expression(src) + if expression is None: continue - m = _OUTPUT_MAPPING_REF.match(src.strip()) + m = _OUTPUT_MAPPING_REF.match(expression) if m and m.group(1) == bt_node_id: return diff --git a/tests/tasks/uipath-maestro-flow/context-grounding/summarize/check_summarize_flow.py b/tests/tasks/uipath-maestro-flow/context-grounding/summarize/check_summarize_flow.py index 93e5026af5..59aca54388 100644 --- a/tests/tasks/uipath-maestro-flow/context-grounding/summarize/check_summarize_flow.py +++ b/tests/tasks/uipath-maestro-flow/context-grounding/summarize/check_summarize_flow.py @@ -9,9 +9,10 @@ `definitions[].model` only — rule 16). 3. `typeVersion` is exactly `"1.0"` (matches `definitions[].version`). 4. `inputs.prompt` is non-empty. - 5. `inputs.attachment` matches the canonical canvas-produced wiring: - `=js:$vars..output.` — referencing a flow `in` - variable of `type: "file"` bound to the trigger via `triggerNodeId`. + 5. `inputs.attachment` matches the canonical trigger-output wiring, either + as the v1 string `=js:$vars..output.` or the SDK's + equivalent typed expression envelope — referencing a flow `in` variable + of `type: "file"` bound to the trigger via `triggerNodeId`. 6. `inputs.returnCitations` is the boolean `true` (per the prompt's request). 7. The instance `outputs.output.source` is the literal `=response`. 8. The flow declares `out` variables `summary` and `citations`, and at least @@ -22,20 +23,26 @@ PascalCase; lowercase resolves to undefined at runtime). """ -import glob import json +import os import re import sys from typing import NoReturn +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..")) +from _shared.advisory_flow_utils import unwrap # noqa: E402 +from _shared.flow_check import find_flow_file # noqa: E402 + NODE_TYPE = "uipath.pattern.deep-rag" EXPECTED_TYPE_VERSION = "1.0" EXPECTED_OUTPUT_SOURCE = "=response" _ATTACHMENT_REF = re.compile( - r"^=js:\s*\$vars\.([A-Za-z_][A-Za-z0-9_]*)\.output\.([A-Za-z_][A-Za-z0-9_]*)\s*$" + r"^\$vars\.([A-Za-z_][A-Za-z0-9_]*)\.output\.([A-Za-z_][A-Za-z0-9_]*)\s*$" +) +_OUTPUT_MAPPING_BASE = re.compile( + r"^\$vars\.([A-Za-z_][A-Za-z0-9_]*)\.output\.content\." ) -_OUTPUT_MAPPING_BASE = re.compile(r"^=js:\s*\$vars\.([A-Za-z_][A-Za-z0-9_]*)\.output\.content\.") def _fail(msg: str) -> NoReturn: @@ -43,10 +50,8 @@ def _fail(msg: str) -> NoReturn: def _read_flow() -> dict: - flows = glob.glob("**/SummarizeDemo*.flow", recursive=True) - if not flows: - _fail("no SummarizeDemo*.flow found under cwd") - with open(flows[0]) as f: + path = find_flow_file(flow_glob="SummarizeDemo*.flow") + with open(path) as f: return json.load(f) @@ -63,10 +68,23 @@ def _find_node(flow: dict) -> dict: return matches[0] +def _js_expression(value) -> str | None: + if isinstance(value, str) and value.startswith("=js:"): + return value[len("=js:") :].strip() + if isinstance(value, dict) and value.get("type") == "jsExpression": + expression = value.get("expression") + if isinstance(expression, str) and expression.strip(): + return expression.strip() + return None + + def _check_attachment_canonical(flow: dict, attachment) -> None: - if not isinstance(attachment, str) or not attachment.strip(): - _fail("inputs.attachment missing or empty — wire it to the trigger output binding") - m = _ATTACHMENT_REF.match(attachment.strip()) + expression = _js_expression(attachment) + if expression is None: + _fail( + "inputs.attachment missing or empty — wire it to the trigger output binding" + ) + m = _ATTACHMENT_REF.match(expression) if not m: _fail( f"inputs.attachment={attachment!r} is not the canonical canvas-produced shape. " @@ -85,7 +103,10 @@ def _check_attachment_canonical(flow: dict, attachment) -> None: f"inputs.attachment references `$vars.{trigger_id}.output...` but no node with " f"id={trigger_id!r} exists in the flow." ) - if not isinstance(trigger_node.get("type"), str) or "trigger" not in trigger_node["type"]: + if ( + not isinstance(trigger_node.get("type"), str) + or "trigger" not in trigger_node["type"] + ): _fail( f"node id={trigger_id!r} has type={trigger_node.get('type')!r}; expected a trigger." ) @@ -131,7 +152,7 @@ def _check_type_version(node: dict) -> None: def _check_output_source(node: dict) -> None: - src = ((node.get("outputs") or {}).get("output") or {}).get("source") + src = unwrap(((node.get("outputs") or {}).get("output") or {}).get("source")) if src != EXPECTED_OUTPUT_SOURCE: _fail( f"outputs.output.source={src!r}; must be {EXPECTED_OUTPUT_SOURCE!r} (the BPMN " @@ -147,13 +168,17 @@ def _check_pascal_output_mappings(flow: dict, dr_node_id: str) -> None: """ globals_ = (flow.get("variables") or {}).get("globals") or [] for var_id in ("summary", "citations"): - if not any(v.get("id") == var_id and v.get("direction") == "out" for v in globals_): + if not any( + v.get("id") == var_id and v.get("direction") == "out" for v in globals_ + ): _fail( f"flow has no `out` variable with id={var_id!r}. The task prompt asks for " f"`{var_id}` to be surfaced as a flow output." ) - end_nodes = [n for n in flow.get("nodes", []) if n.get("type") == "core.control.end"] + end_nodes = [ + n for n in flow.get("nodes", []) if n.get("type") == "core.control.end" + ] if not end_nodes: _fail("flow has no End node — required to terminate the path and map outputs") @@ -165,19 +190,19 @@ def _check_pascal_output_mappings(flow: dict, dr_node_id: str) -> None: mapped = False for end in end_nodes: src = ((end.get("outputs") or {}).get(var_id) or {}).get("source") - if not isinstance(src, str) or not src.strip(): + expression = _js_expression(src) + if expression is None: continue - src_stripped = src.strip() - base_m = _OUTPUT_MAPPING_BASE.match(src_stripped) + base_m = _OUTPUT_MAPPING_BASE.match(expression) if not base_m or base_m.group(1) != dr_node_id: continue # Ensure the Pascal field is in the path - if f".content.{pascal_field}" in src_stripped: + if f".content.{pascal_field}" in expression: mapped = True break # Detect lowercase mistake explicitly lowercase_field = pascal_field[0].lower() + pascal_field[1:] - if f".content.{lowercase_field}" in src_stripped: + if f".content.{lowercase_field}" in expression: _fail( f"End node maps `outputs.{var_id}.source={src!r}` using lowercase " f"`.content.{lowercase_field}` — the response shape is PascalCase " diff --git a/tests/tasks/uipath-maestro-flow/context-grounding/summarize/summarize.yaml b/tests/tasks/uipath-maestro-flow/context-grounding/summarize/summarize.yaml index d57b247b99..878d1e924b 100644 --- a/tests/tasks/uipath-maestro-flow/context-grounding/summarize/summarize.yaml +++ b/tests/tasks/uipath-maestro-flow/context-grounding/summarize/summarize.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-summarize description: > Create a UiPath Flow that runs a Summarize pattern node (`uipath.pattern.deep-rag`) @@ -43,7 +46,7 @@ initial_prompt: | Note the PascalCase nested field names — lowercase `text`/`citations` would resolve to `undefined` at runtime. - Do NOT run flow debug — just validate the flow. + Validate the completed flow. Before starting, follow the documented workflow steps exactly — in particular, note that the wire-level node type is `uipath.pattern.deep-rag` even though the canvas display name is "Summarize". @@ -71,7 +74,7 @@ success_criteria: # ── Structural checks (node type, inputs, returnCitations boolean) ───── - type: run_command description: "Flow has a Summarize (deep-rag) node with prompt, attachment wired, and returnCitations=true" - command: "python3 $TASK_DIR/check_summarize_flow.py" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/context-grounding/summarize/check_summarize_flow.py" timeout: 30 expected_exit_code: 0 weight: 5.0 diff --git a/tests/tasks/uipath-maestro-flow/e2e/check_devcon_expense_approval.py b/tests/tasks/uipath-maestro-flow/e2e/check_devcon_expense_approval.py index 01e8582865..13e7255821 100644 --- a/tests/tasks/uipath-maestro-flow/e2e/check_devcon_expense_approval.py +++ b/tests/tasks/uipath-maestro-flow/e2e/check_devcon_expense_approval.py @@ -9,13 +9,20 @@ from __future__ import annotations -import glob import json import sys from pathlib import Path +SHARED_DIR = Path(__file__).resolve().parents[1] / "_shared" +sys.path.insert(0, str(SHARED_DIR)) +try: + from flow_check import find_flow_file # noqa: E402 +except ModuleNotFoundError as exc: + if exc.name != "flow_check": + raise + from _shared.flow_check import find_flow_file # noqa: E402 -FLOW_GLOB = "ExpenseApproval/ExpenseApproval/ExpenseApproval.flow" +FLOW_GLOB = "ExpenseApproval*.flow" def fail(message: str) -> None: @@ -23,10 +30,7 @@ def fail(message: str) -> None: def load_flow() -> dict: - matches = glob.glob(FLOW_GLOB) - if not matches: - fail(f"No flow file matching {FLOW_GLOB}") - path = Path(matches[0]) + path = Path(find_flow_file(flow_glob=FLOW_GLOB)) try: return json.loads(path.read_text()) except json.JSONDecodeError as exc: diff --git a/tests/tasks/uipath-maestro-flow/e2e/devcon_expense_approval.yaml b/tests/tasks/uipath-maestro-flow/e2e/devcon_expense_approval.yaml index d46837477c..b3d1361de7 100644 --- a/tests/tasks/uipath-maestro-flow/e2e/devcon_expense_approval.yaml +++ b/tests/tasks/uipath-maestro-flow/e2e/devcon_expense_approval.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-e2e-devcon-expense-approval description: > DevCon end-to-end scenario: developer gives a vague expense approval requirement. @@ -54,12 +57,12 @@ success_criteria: pass_threshold: 0.0 # advisory: init auto-scaffolds the parent solution (CLI #2470) - type: command_executed - description: "Agent initialized a Flow project" + description: "Flow-project initialization telemetry (report only; the artifact is graded below)" tool_name: "Bash" command_pattern: '(uip|\$UIP|\$\{?UIP\}?)\s+(maestro\s+)?flow\s+init' min_count: 1 weight: 1.0 - pass_threshold: 1.0 + pass_threshold: 0.0 - type: run_command description: "Flow file exists (path-agnostic discovery)" @@ -71,19 +74,19 @@ success_criteria: - type: run_command description: "Flow has semantically correct inline HITL schema, decision capture, completed wiring, and .output access paths" - command: "python3 $TASK_DIR/check_devcon_expense_approval.py" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/e2e/check_devcon_expense_approval.py" timeout: 30 expected_exit_code: 0 weight: 16.5 pass_threshold: 1.0 - type: command_executed - description: "Agent validated the flow" + description: "Flow-validation telemetry (report only; successful validation is graded below)" tool_name: "Bash" command_pattern: '(uip|\$UIP|\$\{?UIP\}?)\s+(maestro\s+)?flow\s+validate' min_count: 1 weight: 2.5 - pass_threshold: 1.0 + pass_threshold: 0.0 - type: run_command description: "Flow validates successfully" diff --git a/tests/tasks/uipath-maestro-flow/e2e/escalation_jira_ticket/escalation_jira_ticket.yaml b/tests/tasks/uipath-maestro-flow/e2e/escalation_jira_ticket/escalation_jira_ticket.yaml index f01a5552ec..6700d585e5 100644 --- a/tests/tasks/uipath-maestro-flow/e2e/escalation_jira_ticket/escalation_jira_ticket.yaml +++ b/tests/tasks/uipath-maestro-flow/e2e/escalation_jira_ticket/escalation_jira_ticket.yaml @@ -32,7 +32,8 @@ post_run: timeout: 120 initial_prompt: | - Create a UiPath Maestro Flow named "EscalationJiraTicket" with a MANUAL trigger. + Create a UiPath Maestro Flow named "EscalationJiraTicket" with a MANUAL trigger + inside a solution of the same name. It triages a customer escalation and creates a Jira ticket for it. Read `seed.json` in the current directory. Declare flow inputs matching the keys diff --git a/tests/tasks/uipath-maestro-flow/e2e/escalation_jira_ticket/jira_is.py b/tests/tasks/uipath-maestro-flow/e2e/escalation_jira_ticket/jira_is.py index f017e61484..046ed14d16 100644 --- a/tests/tasks/uipath-maestro-flow/e2e/escalation_jira_ticket/jira_is.py +++ b/tests/tasks/uipath-maestro-flow/e2e/escalation_jira_ticket/jira_is.py @@ -125,6 +125,10 @@ def delete_issue(conn_id: str, key: str) -> bool: env = _run( "is", "resources", "run", "delete", CONNECTOR, "issue", "--connection-id", conn_id, "--query", f"issueId={key}", + # The CLI never prompts and REFUSES an irreversible delete without this + # flag ("Confirmation required … Re-run with --yes"). Without it every + # teardown since 08-19 printed WARN and left its ticket in the CE project. + "--yes", ) if str(env.get("Result", "")).lower() != "failure": return True diff --git a/tests/tasks/uipath-maestro-flow/e2e/escalation_jira_ticket/test_jira_is.py b/tests/tasks/uipath-maestro-flow/e2e/escalation_jira_ticket/test_jira_is.py new file mode 100644 index 0000000000..5dfb339f03 --- /dev/null +++ b/tests/tasks/uipath-maestro-flow/e2e/escalation_jira_ticket/test_jira_is.py @@ -0,0 +1,71 @@ +"""Teardown contract for the escalation task's Jira helper.""" + +from __future__ import annotations + +import importlib.util +import json +from pathlib import Path +from types import ModuleType + +import pytest + +HERE = Path(__file__).parent + + +def _load_module() -> ModuleType: + spec = importlib.util.spec_from_file_location("escalation_jira_is", HERE / "jira_is.py") + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class Result: + def __init__(self, envelope: object, returncode: int = 0) -> None: + self.stdout = json.dumps(envelope) + self.stderr = "" + self.returncode = returncode + + +def _stub(monkeypatch: pytest.MonkeyPatch, jira_is: ModuleType, envelope: object) -> list[list[str]]: + seen: list[list[str]] = [] + + def fake_run(cmd, **kwargs): + seen.append(list(cmd)) + return Result(envelope) + + monkeypatch.setattr(jira_is.subprocess, "run", fake_run) + return seen + + +def test_delete_issue_confirms_and_reports_success(monkeypatch: pytest.MonkeyPatch) -> None: + """The CLI refuses an unconfirmed delete with a Failure envelope + ("Confirmation required … Re-run with --yes"), which `delete_issue` + correctly reports as NOT confirmed — so every run 08-19 → 09-01 printed + `WARN: could NOT confirm deletion` and left its ticket in CE. `--yes` is + the missing word.""" + jira_is = _load_module() + seen = _stub(monkeypatch, jira_is, {"Result": "Success", "Data": {"Value": ""}}) + assert jira_is.delete_issue("conn", "CE-1257") is True + (cmd,) = seen + assert cmd[:5] == ["uip", "is", "resources", "run", "delete"] + assert "--yes" in cmd and "issueId=CE-1257" in cmd + + +def test_delete_issue_unconfirmed_failure_is_not_a_deletion(monkeypatch: pytest.MonkeyPatch) -> None: + jira_is = _load_module() + _stub(monkeypatch, jira_is, { + "Result": "Failure", + "Message": "Confirmation required: this will delete resource 'issue' and cannot be undone.", + "Instructions": "Re-run with --yes to confirm.", + }) + assert jira_is.delete_issue("conn", "CE-1257") is False + + +def test_delete_issue_structured_404_counts_as_gone(monkeypatch: pytest.MonkeyPatch) -> None: + jira_is = _load_module() + _stub(monkeypatch, jira_is, { + "Result": "Failure", + "Message": "Request failed with status code '404': Issue does not exist or you do not have permission to see it.", + }) + assert jira_is.delete_issue("conn", "CE-1257") is True diff --git a/tests/tasks/uipath-maestro-flow/e2e/escalation_orchestrator_paths/escalation_orchestrator_paths.yaml b/tests/tasks/uipath-maestro-flow/e2e/escalation_orchestrator_paths/escalation_orchestrator_paths.yaml index 9f73e3b270..36141b9e84 100644 --- a/tests/tasks/uipath-maestro-flow/e2e/escalation_orchestrator_paths/escalation_orchestrator_paths.yaml +++ b/tests/tasks/uipath-maestro-flow/e2e/escalation_orchestrator_paths/escalation_orchestrator_paths.yaml @@ -104,9 +104,14 @@ success_criteria: weight: 1.5 pass_threshold: 1.0 + # ── The delivered flow validates ──────────────────────────────────────── + # Discovered, not hardcoded: the SDK loop scaffolds `Sol//.flow` + # (its SKILL.md says `uip solution init Sol`), so a fixed + # `//.flow` path failed "File not found" on flows that were + # otherwise green (adhoc 2026-08-31 22:43 v2: seeded paths passed, validate 0.0). - type: run_command - description: "Generated CustomerEscalationOrchestrator Flow validates without errors" - command: "uip maestro flow validate CustomerEscalationOrchestrator/CustomerEscalationOrchestrator/CustomerEscalationOrchestrator.flow --output json" + description: "Generated CustomerEscalationOrchestrator Flow validates without errors, independent of solution layout" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py" timeout: 180 expected_exit_code: 0 weight: 3.0 diff --git a/tests/tasks/uipath-maestro-flow/e2e/escalation_slack_alert/escalation_slack_alert.yaml b/tests/tasks/uipath-maestro-flow/e2e/escalation_slack_alert/escalation_slack_alert.yaml index 062d4b3c20..8665383195 100644 --- a/tests/tasks/uipath-maestro-flow/e2e/escalation_slack_alert/escalation_slack_alert.yaml +++ b/tests/tasks/uipath-maestro-flow/e2e/escalation_slack_alert/escalation_slack_alert.yaml @@ -82,8 +82,8 @@ success_criteria: # ── The delivered flow validates ──────────────────────────────────────── - type: run_command - description: "Generated EscalationSlackAlert Flow validates without errors" - command: "uip maestro flow validate EscalationSlackAlert/EscalationSlackAlert/EscalationSlackAlert.flow --output json" + description: "Generated EscalationSlackAlert Flow validates without errors, independent of solution layout" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py" timeout: 180 expected_exit_code: 0 weight: 3.0 diff --git a/tests/tasks/uipath-maestro-flow/e2e/jira_create_issue/check_jira_create_issue.py b/tests/tasks/uipath-maestro-flow/e2e/jira_create_issue/check_jira_create_issue.py index d2e359b653..add249b56a 100644 --- a/tests/tasks/uipath-maestro-flow/e2e/jira_create_issue/check_jira_create_issue.py +++ b/tests/tasks/uipath-maestro-flow/e2e/jira_create_issue/check_jira_create_issue.py @@ -16,7 +16,6 @@ from __future__ import annotations -import glob import json import os import re @@ -26,7 +25,12 @@ HERE = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, HERE) # local jira_is sys.path.insert(0, os.path.dirname(os.path.dirname(HERE))) # …/uipath-maestro-flow (for _shared) -from _shared.flow_check import collect_outputs, get_last_debug_raw, run_debug # noqa: E402 +from _shared.flow_check import ( # noqa: E402 + collect_outputs, + find_flow_file, + get_last_debug_raw, + run_debug, +) import jira_is # noqa: E402 JIRA_KEY = "uipath-atlassian-jira" @@ -40,11 +44,10 @@ def _fail(msg: str) -> None: def main() -> None: seed = json.loads(Path("seed.json").read_text()) - flows = glob.glob("**/*.flow", recursive=True) - raw = next((r for p in flows for r in [open(p, encoding="utf-8").read()] - if JIRA_KEY in r and '"nodes"' in r), None) - if raw is None: - _fail(f"no .flow references the {JIRA_KEY} connector (found {flows})") + flow_path = find_flow_file() + raw = Path(flow_path).read_text(encoding="utf-8") + if JIRA_KEY not in raw or '"nodes"' not in raw: + _fail(f"{flow_path} does not reference the {JIRA_KEY} connector") print(f"OK: flow references {JIRA_KEY}") payload = run_debug(timeout=480) diff --git a/tests/tasks/uipath-maestro-flow/e2e/jira_create_issue/fixtures/folder_get_missing_data.json b/tests/tasks/uipath-maestro-flow/e2e/jira_create_issue/fixtures/folder_get_missing_data.json new file mode 100644 index 0000000000..65138fda8b --- /dev/null +++ b/tests/tasks/uipath-maestro-flow/e2e/jira_create_issue/fixtures/folder_get_missing_data.json @@ -0,0 +1,3 @@ +{ + "Result": "Failure" +} diff --git a/tests/tasks/uipath-maestro-flow/e2e/jira_create_issue/jira_create_issue.yaml b/tests/tasks/uipath-maestro-flow/e2e/jira_create_issue/jira_create_issue.yaml index 2185c80264..521c08d28e 100644 --- a/tests/tasks/uipath-maestro-flow/e2e/jira_create_issue/jira_create_issue.yaml +++ b/tests/tasks/uipath-maestro-flow/e2e/jira_create_issue/jira_create_issue.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-ipe-jira-create-issue description: > E2E live Jira coverage — builds a Flow with a manual trigger and an @@ -28,7 +31,8 @@ post_run: timeout: 120 initial_prompt: | - Create a new Flow project called "JiraCreateIssue" with a manual trigger. + Create a new Flow project called "JiraCreateIssue" with a manual trigger, + inside a solution of the same name. It should create a Jira issue using the Atlassian Jira "Create Issue" connector activity, and expose the new issue's key as a flow output. @@ -55,17 +59,17 @@ initial_prompt: | decision, assumption, and blocked step in your final response. Instructions in the task take precedence over this paragraph. success_criteria: - - type: command_executed - description: "uip maestro flow validate was called" - tool_name: "Bash" - command_pattern: "(uip|\\$UIP)\\s+(maestro\\s+)?flow\\s+validate" - min_count: 1 + - type: run_command + description: "Flow validates successfully" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py" + timeout: 30 + expected_exit_code: 0 weight: 2.0 pass_threshold: 1.0 - type: run_command description: "JiraCreateIssue checks: valid flow with Jira Create-Issue node, debug creates a real issue, and the tenant carries the seeded summary" - command: "python3 $TASK_DIR/check_jira_create_issue.py" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/e2e/jira_create_issue/check_jira_create_issue.py" timeout: 1080 expected_exit_code: 0 weight: 5.0 diff --git a/tests/tasks/uipath-maestro-flow/e2e/jira_create_issue/jira_is.py b/tests/tasks/uipath-maestro-flow/e2e/jira_create_issue/jira_is.py index 636a1a6498..90dd603348 100644 --- a/tests/tasks/uipath-maestro-flow/e2e/jira_create_issue/jira_is.py +++ b/tests/tasks/uipath-maestro-flow/e2e/jira_create_issue/jira_is.py @@ -12,6 +12,7 @@ import json import subprocess +from typing import Any CONNECTOR = "uipath-atlassian-jira" FOLDER_PATH = "Shared/uipath-maestro-flow" @@ -20,34 +21,72 @@ ISSUETYPE_ID = "11457" # "Task" issue type, scoped to the CE project -def _run(*args: str) -> dict: - out = subprocess.run( +def _operation(args: tuple[str, ...]) -> str: + return " ".join(args[:3]) + + +def _run(*args: str) -> dict[str, Any]: + result = subprocess.run( ["uip", *args, "--output", "json"], capture_output=True, text=True, timeout=120, - ).stdout - return json.loads(out) + ) + try: + envelope = json.loads(result.stdout) + except json.JSONDecodeError as exc: + raise RuntimeError( + f"uip {_operation(args)} returned invalid JSON " + f"(exit {result.returncode}, stdout length {len(result.stdout)})" + ) from exc + if not isinstance(envelope, dict): + raise RuntimeError(f"uip {_operation(args)} returned a non-object JSON envelope") + return envelope + + +def _is_transient(envelope: dict[str, Any]) -> bool: + return envelope.get("Retry") == "RetryLater" or envelope.get("ErrorCode") == "server_error" + + +def _failure_summary(envelope: dict[str, Any]) -> str: + keys = ("Result", "ErrorCode", "Retry", "StatusCode", "Message") + summary = {key: envelope[key] for key in keys if key in envelope} + return json.dumps(summary, sort_keys=True, default=str) + + +def _required_data(*args: str, retry_transient: bool = False) -> Any: + envelope = _run(*args) + if retry_transient and _is_transient(envelope): + envelope = _run(*args) + if envelope.get("Result") != "Success" or "Data" not in envelope: + raise RuntimeError( + f"uip {_operation(args)} failed: {_failure_summary(envelope)}" + ) + return envelope["Data"] def connection_id() -> str: - folder_key = _run("or", "folders", "get", FOLDER_PATH)["Data"]["Key"] - conns = _run("is", "connections", "list", CONNECTOR, "--folder-key", folder_key, "--refresh")["Data"] + folder = _required_data("or", "folders", "get", FOLDER_PATH, retry_transient=True) + folder_key = folder["Key"] + conns = _required_data( + "is", "connections", "list", CONNECTOR, + "--folder-key", folder_key, "--refresh", + ) return next(c["Id"] for c in conns if c["Name"] == CONNECTION_NAME) def myself(conn_id: str) -> str: """Return the connection user's Atlassian accountId.""" - return _run( + return _required_data( "is", "resources", "run", "get", CONNECTOR, "myself", "--connection-id", conn_id, - )["Data"]["accountId"] + )["accountId"] def create_issue(conn_id: str, summary: str) -> str: body = {"fields": {"project": {"key": PROJECT_KEY}, "issuetype": {"id": ISSUETYPE_ID}, "summary": summary}} - return _run( + return _required_data( "is", "resources", "run", "create", CONNECTOR, "curated_create_issue", "--connection-id", conn_id, "--body", json.dumps(body), - )["Data"]["key"] + )["key"] def get_issue(conn_id: str, key: str) -> dict | None: @@ -67,4 +106,8 @@ def delete_issue(conn_id: str, key: str) -> None: _run( "is", "resources", "run", "delete", CONNECTOR, "issue", "--connection-id", conn_id, "--query", f"issueId={key}", + # The CLI never prompts and REFUSES an irreversible delete without this + # flag ("Confirmation required … Re-run with --yes"). Without it every + # teardown since 08-19 printed WARN and left its ticket in the CE project. + "--yes", ) diff --git a/tests/tasks/uipath-maestro-flow/e2e/jira_create_issue/test_jira_is.py b/tests/tasks/uipath-maestro-flow/e2e/jira_create_issue/test_jira_is.py new file mode 100644 index 0000000000..340d4c8d9c --- /dev/null +++ b/tests/tasks/uipath-maestro-flow/e2e/jira_create_issue/test_jira_is.py @@ -0,0 +1,89 @@ +from __future__ import annotations + +import importlib.util +import json +from pathlib import Path +from types import ModuleType + +import pytest + + +HERE = Path(__file__).parent + + +def _load_module() -> ModuleType: + spec = importlib.util.spec_from_file_location("jira_create_issue_is", HERE / "jira_is.py") + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class Result: + def __init__(self, envelope: object, returncode: int = 0) -> None: + self.stdout = json.dumps(envelope) + self.stderr = "" + self.returncode = returncode + + +def test_connection_id_retries_transient_folder_lookup(monkeypatch: pytest.MonkeyPatch) -> None: + jira_is = _load_module() + responses = iter([ + Result({"Result": "Failure", "ErrorCode": "server_error", "Retry": "RetryLater"}), + Result({"Result": "Success", "Data": {"Key": "folder-key"}}), + Result({"Result": "Success", "Data": [{"Id": "connection-id", "Name": jira_is.CONNECTION_NAME}]}), + ]) + monkeypatch.setattr(jira_is.subprocess, "run", lambda *args, **kwargs: next(responses)) + + assert jira_is.connection_id() == "connection-id" + + +def test_missing_data_reports_redacted_envelope(monkeypatch: pytest.MonkeyPatch) -> None: + jira_is = _load_module() + envelope = json.loads((HERE / "fixtures" / "folder_get_missing_data.json").read_text()) + envelope["AccessToken"] = "must-not-appear" + monkeypatch.setattr(jira_is.subprocess, "run", lambda *args, **kwargs: Result(envelope)) + + with pytest.raises(RuntimeError) as exc_info: + jira_is.connection_id() + + message = str(exc_info.value) + assert '"Result": "Failure"' in message + assert "Data" not in message + assert "must-not-appear" not in message + + +def test_invalid_json_does_not_echo_cli_output(monkeypatch: pytest.MonkeyPatch) -> None: + jira_is = _load_module() + result = Result({}) + result.stdout = "not-json access_token=must-not-appear" + result.stderr = "must-not-appear" + result.returncode = 1 + monkeypatch.setattr(jira_is.subprocess, "run", lambda *args, **kwargs: result) + + with pytest.raises(RuntimeError) as exc_info: + jira_is.connection_id() + + message = str(exc_info.value) + assert "invalid JSON" in message + assert "exit 1" in message + assert "must-not-appear" not in message + + +def test_delete_issue_confirms_the_irreversible_delete(monkeypatch: pytest.MonkeyPatch) -> None: + """`uip is resources run delete` never prompts and refuses without `--yes` + ("Confirmation required: this will delete resource 'issue' …"). Every + teardown in the 08-19 → 09-01 run archive omitted it and leaked its ticket.""" + jira_is = _load_module() + seen: list[list[str]] = [] + + def fake_run(cmd, **kwargs): + seen.append(list(cmd)) + return Result({"Result": "Success", "Data": {"Value": ""}}) + + monkeypatch.setattr(jira_is.subprocess, "run", fake_run) + jira_is.delete_issue("conn", "CE-1") + (cmd,) = seen + assert cmd[:5] == ["uip", "is", "resources", "run", "delete"] + assert "--yes" in cmd + assert "issueId=CE-1" in cmd diff --git a/tests/tasks/uipath-maestro-flow/e2e/jira_get_issue/check_jira_get_issue.py b/tests/tasks/uipath-maestro-flow/e2e/jira_get_issue/check_jira_get_issue.py index bed6f91b57..2b0e9e69b9 100644 --- a/tests/tasks/uipath-maestro-flow/e2e/jira_get_issue/check_jira_get_issue.py +++ b/tests/tasks/uipath-maestro-flow/e2e/jira_get_issue/check_jira_get_issue.py @@ -16,7 +16,6 @@ from __future__ import annotations -import glob import json import os import re @@ -25,7 +24,7 @@ HERE = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, os.path.dirname(os.path.dirname(HERE))) # …/uipath-maestro-flow (for _shared) -from _shared.flow_check import assert_outputs_contain, run_debug # noqa: E402 +from _shared.flow_check import assert_outputs_contain, find_flow_file, run_debug # noqa: E402 JIRA_KEY = "uipath-atlassian-jira" GET_OP_RE = re.compile(r"get[\s_-]?issue|curated_get_issue", re.IGNORECASE) @@ -39,11 +38,10 @@ def main() -> None: seed = json.loads(Path("seed.json").read_text()) issue_key = seed["issue_key"] - flows = glob.glob("**/*.flow", recursive=True) - raw = next((r for p in flows for r in [open(p, encoding="utf-8").read()] - if JIRA_KEY in r and '"nodes"' in r), None) - if raw is None: - _fail(f"no .flow references the {JIRA_KEY} connector (found {flows})") + flow_path = find_flow_file() + raw = Path(flow_path).read_text(encoding="utf-8") + if JIRA_KEY not in raw or '"nodes"' not in raw: + _fail(f"{flow_path} does not reference the {JIRA_KEY} connector") print(f"OK: flow references {JIRA_KEY}") if not GET_OP_RE.search(raw): _fail("flow does not reference a Get-Issue operation") diff --git a/tests/tasks/uipath-maestro-flow/e2e/jira_get_issue/jira_get_issue.yaml b/tests/tasks/uipath-maestro-flow/e2e/jira_get_issue/jira_get_issue.yaml index dec7b9a3d3..040bb0c57a 100644 --- a/tests/tasks/uipath-maestro-flow/e2e/jira_get_issue/jira_get_issue.yaml +++ b/tests/tasks/uipath-maestro-flow/e2e/jira_get_issue/jira_get_issue.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-ipe-jira-get-issue description: > E2E live Jira coverage — builds a Flow with a manual trigger and an Atlassian @@ -29,7 +32,8 @@ post_run: timeout: 120 initial_prompt: | - Create a new Flow project called "JiraGetIssue" with a manual trigger. + Create a new Flow project called "JiraGetIssue" with a manual trigger, + inside a solution of the same name. It should read a single Jira issue using the Atlassian Jira "Get Issue" connector activity, and expose the retrieved issue's summary (and status) as flow outputs. @@ -55,17 +59,17 @@ initial_prompt: | decision, assumption, and blocked step in your final response. Instructions in the task take precedence over this paragraph. success_criteria: - - type: command_executed - description: "uip maestro flow validate was called" - tool_name: "Bash" - command_pattern: "(uip|\\$UIP)\\s+(maestro\\s+)?flow\\s+validate" - min_count: 1 + - type: run_command + description: "Flow validates successfully" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py" + timeout: 30 + expected_exit_code: 0 weight: 2.0 pass_threshold: 1.0 - type: run_command description: "JiraGetIssue checks: valid flow with Jira Get-Issue node referencing the seeded key, debug completes, and outputs carry the seeded summary" - command: "python3 $TASK_DIR/check_jira_get_issue.py" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/e2e/jira_get_issue/check_jira_get_issue.py" timeout: 1080 expected_exit_code: 0 weight: 5.0 diff --git a/tests/tasks/uipath-maestro-flow/e2e/jira_get_issue/jira_is.py b/tests/tasks/uipath-maestro-flow/e2e/jira_get_issue/jira_is.py index 40a9b2acff..9e6edb53c0 100644 --- a/tests/tasks/uipath-maestro-flow/e2e/jira_get_issue/jira_is.py +++ b/tests/tasks/uipath-maestro-flow/e2e/jira_get_issue/jira_is.py @@ -59,4 +59,8 @@ def delete_issue(conn_id: str, key: str) -> None: _run( "is", "resources", "run", "delete", CONNECTOR, "issue", "--connection-id", conn_id, "--query", f"issueId={key}", + # The CLI never prompts and REFUSES an irreversible delete without this + # flag ("Confirmation required … Re-run with --yes"). Without it every + # teardown since 08-19 printed WARN and left its ticket in the CE project. + "--yes", ) diff --git a/tests/tasks/uipath-maestro-flow/e2e/jira_lifecycle/check_jira_lifecycle.py b/tests/tasks/uipath-maestro-flow/e2e/jira_lifecycle/check_jira_lifecycle.py index 6fdabde99c..5a80b07e28 100644 --- a/tests/tasks/uipath-maestro-flow/e2e/jira_lifecycle/check_jira_lifecycle.py +++ b/tests/tasks/uipath-maestro-flow/e2e/jira_lifecycle/check_jira_lifecycle.py @@ -24,7 +24,6 @@ from __future__ import annotations -import glob import json import os import re @@ -37,6 +36,7 @@ from _shared.flow_check import ( # noqa: E402 assert_flow_has_any_node_type, assert_flow_has_node_type, + find_flow_file, get_last_debug_raw, run_debug, ) @@ -70,11 +70,10 @@ def main() -> None: } # 1. STRUCTURAL ---------------------------------------------------------- - flows = glob.glob("**/*.flow", recursive=True) - raw = next((r for p in flows for r in [open(p, encoding="utf-8").read()] - if JIRA_KEY in r and '"nodes"' in r), None) - if raw is None: - _fail(f"no .flow references the {JIRA_KEY} connector (found {flows})") + flow_path = find_flow_file() + raw = Path(flow_path).read_text(encoding="utf-8") + if JIRA_KEY not in raw or '"nodes"' not in raw: + _fail(f"{flow_path} does not reference the {JIRA_KEY} connector") print(f"OK: flow references {JIRA_KEY}") assert_flow_has_node_type(["core.logic.loop"]) diff --git a/tests/tasks/uipath-maestro-flow/e2e/jira_lifecycle/jira_is.py b/tests/tasks/uipath-maestro-flow/e2e/jira_lifecycle/jira_is.py index da3f617715..e5e2b601ee 100644 --- a/tests/tasks/uipath-maestro-flow/e2e/jira_lifecycle/jira_is.py +++ b/tests/tasks/uipath-maestro-flow/e2e/jira_lifecycle/jira_is.py @@ -60,4 +60,8 @@ def delete_issue(conn_id: str, key: str) -> None: _run( "is", "resources", "run", "delete", CONNECTOR, "issue", "--connection-id", conn_id, "--query", f"issueId={key}", + # The CLI never prompts and REFUSES an irreversible delete without this + # flag ("Confirmation required … Re-run with --yes"). Without it every + # teardown since 08-19 printed WARN and left its ticket in the CE project. + "--yes", ) diff --git a/tests/tasks/uipath-maestro-flow/e2e/jira_lifecycle/jira_lifecycle.yaml b/tests/tasks/uipath-maestro-flow/e2e/jira_lifecycle/jira_lifecycle.yaml index 9f11d6378b..00ad8e60af 100644 --- a/tests/tasks/uipath-maestro-flow/e2e/jira_lifecycle/jira_lifecycle.yaml +++ b/tests/tasks/uipath-maestro-flow/e2e/jira_lifecycle/jira_lifecycle.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-jira-lifecycle description: > E2E live Jira coverage of a composite loop-and-switch flow. The agent builds @@ -33,7 +36,8 @@ post_run: timeout: 180 initial_prompt: | - Create a new Flow project called "JiraLifecycle" with a manual trigger. + Create a new Flow project called "JiraLifecycle" with a manual trigger, + inside a solution of the same name. Read `seed.json` in the current directory. It contains a list of `issues` (each with a `summary` and a `priority`), the `project_key` and `issuetype_id` @@ -68,17 +72,17 @@ initial_prompt: | decision, assumption, and blocked step in your final response. Instructions in the task take precedence over this paragraph. success_criteria: - - type: command_executed - description: "uip maestro flow validate was called" - tool_name: "Bash" - command_pattern: "(uip|\\$UIP)\\s+(maestro\\s+)?flow\\s+validate" - min_count: 1 + - type: run_command + description: "Flow validates successfully" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py" + timeout: 30 + expected_exit_code: 0 weight: 2.0 pass_threshold: 1.0 - type: run_command description: "JiraLifecycle checks: valid flow with a loop + switch over the Jira connector, debug creates every seeded issue, and each carries the comment its priority branch should have posted" - command: "python3 $TASK_DIR/check_jira_lifecycle.py" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/e2e/jira_lifecycle/check_jira_lifecycle.py" timeout: 1320 expected_exit_code: 0 weight: 5.0 diff --git a/tests/tasks/uipath-maestro-flow/e2e/jira_search_triage/check_jira_search_triage.py b/tests/tasks/uipath-maestro-flow/e2e/jira_search_triage/check_jira_search_triage.py index 597c957519..c24cf1aa8f 100644 --- a/tests/tasks/uipath-maestro-flow/e2e/jira_search_triage/check_jira_search_triage.py +++ b/tests/tasks/uipath-maestro-flow/e2e/jira_search_triage/check_jira_search_triage.py @@ -12,7 +12,6 @@ from __future__ import annotations -import glob import json import os import re @@ -22,7 +21,7 @@ HERE = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, HERE) # local jira_is sys.path.insert(0, os.path.dirname(os.path.dirname(HERE))) # …/uipath-maestro-flow (for _shared) -from _shared.flow_check import assert_flow_has_node_type, run_debug # noqa: E402 +from _shared.flow_check import assert_flow_has_node_type, find_flow_file, run_debug # noqa: E402 import jira_is # noqa: E402 JIRA_KEY = "uipath-atlassian-jira" @@ -36,11 +35,10 @@ def _fail(msg: str) -> None: def main() -> None: seed = json.loads(Path("seed.json").read_text()) - flows = glob.glob("**/*.flow", recursive=True) - raw = next((r for p in flows for r in [open(p, encoding="utf-8").read()] - if JIRA_KEY in r and '"nodes"' in r), None) - if raw is None: - _fail(f"no .flow references the {JIRA_KEY} connector (found {flows})") + flow_path = find_flow_file() + raw = Path(flow_path).read_text(encoding="utf-8") + if JIRA_KEY not in raw or '"nodes"' not in raw: + _fail(f"{flow_path} does not reference the {JIRA_KEY} connector") print(f"OK: flow references {JIRA_KEY}") if not SEARCH_OP_RE.search(raw): _fail("flow does not reference a Search-Issues (JQL) operation") diff --git a/tests/tasks/uipath-maestro-flow/e2e/jira_search_triage/jira_is.py b/tests/tasks/uipath-maestro-flow/e2e/jira_search_triage/jira_is.py index 188986d2f6..d734b13884 100644 --- a/tests/tasks/uipath-maestro-flow/e2e/jira_search_triage/jira_is.py +++ b/tests/tasks/uipath-maestro-flow/e2e/jira_search_triage/jira_is.py @@ -58,4 +58,8 @@ def delete_issue(conn_id: str, key: str) -> None: _run( "is", "resources", "run", "delete", CONNECTOR, "issue", "--connection-id", conn_id, "--query", f"issueId={key}", + # The CLI never prompts and REFUSES an irreversible delete without this + # flag ("Confirmation required … Re-run with --yes"). Without it every + # teardown since 08-19 printed WARN and left its ticket in the CE project. + "--yes", ) diff --git a/tests/tasks/uipath-maestro-flow/e2e/jira_search_triage/jira_search_triage.yaml b/tests/tasks/uipath-maestro-flow/e2e/jira_search_triage/jira_search_triage.yaml index 69d81ef67c..4b61fdba47 100644 --- a/tests/tasks/uipath-maestro-flow/e2e/jira_search_triage/jira_search_triage.yaml +++ b/tests/tasks/uipath-maestro-flow/e2e/jira_search_triage/jira_search_triage.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-jira-search-triage description: > E2E live Jira coverage of a JQL-search-driven triage flow: a manual-trigger @@ -26,7 +29,8 @@ post_run: timeout: 120 initial_prompt: | - Create a new Flow project called "JiraSearchTriage" with a manual trigger. + Create a new Flow project called "JiraSearchTriage" with a manual trigger, + inside a solution of the same name. Read `seed.json` in the current directory for the `jql` query and the `processed_comment` body. @@ -55,17 +59,17 @@ initial_prompt: | decision, assumption, and blocked step in your final response. Instructions in the task take precedence over this paragraph. success_criteria: - - type: command_executed - description: "uip maestro flow validate was called" - tool_name: "Bash" - command_pattern: "(uip|\\$UIP)\\s+(maestro\\s+)?flow\\s+validate" - min_count: 1 + - type: run_command + description: "Flow validates successfully" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py" + timeout: 30 + expected_exit_code: 0 weight: 2.0 pass_threshold: 1.0 - type: run_command description: "JiraSearchTriage checks: valid flow with a JQL search feeding a loop of Add-Comment, debug completes, and both seeded issues carry the triage comment" - command: "python3 $TASK_DIR/check_jira_search_triage.py" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/e2e/jira_search_triage/check_jira_search_triage.py" timeout: 1320 expected_exit_code: 0 weight: 5.0 diff --git a/tests/tasks/uipath-maestro-flow/e2e/test_check_devcon_expense_approval.py b/tests/tasks/uipath-maestro-flow/e2e/test_check_devcon_expense_approval.py index 0576095281..fb275a9d41 100644 --- a/tests/tasks/uipath-maestro-flow/e2e/test_check_devcon_expense_approval.py +++ b/tests/tasks/uipath-maestro-flow/e2e/test_check_devcon_expense_approval.py @@ -3,6 +3,8 @@ from __future__ import annotations import json +import os +import shutil import subprocess import sys from pathlib import Path @@ -58,6 +60,9 @@ def _flow_doc( def _write_flow(tmp_path: Path, doc: dict[str, Any]) -> None: project = tmp_path / "ExpenseApproval" / "ExpenseApproval" project.mkdir(parents=True) + (project / "project.uiproj").write_text( + json.dumps({"ProjectType": "Flow"}), encoding="utf-8" + ) (project / "ExpenseApproval.flow").write_text(json.dumps(doc), encoding="utf-8") @@ -121,6 +126,40 @@ def test_accepts_expression_hitl_input_binding(tmp_path: Path) -> None: assert result.returncode == 0, result.stderr +def test_accepts_root_level_sdk_emit(tmp_path: Path) -> None: + doc = _flow_doc( + fields=_approve_reject_fields("vars.fetchExpense.output.amount"), + outcomes=_APPROVE_REJECT_OUTCOMES, + ) + (tmp_path / "ExpenseApproval.flow").write_text(json.dumps(doc), encoding="utf-8") + + result = _run_checker(tmp_path) + + assert result.returncode == 0, result.stderr + + +def test_accepts_checker_frozen_without_sibling_shared_directory(tmp_path: Path) -> None: + workspace = tmp_path / "workspace" + workspace.mkdir() + _write_approve_reject_flow(workspace, "vars.fetchExpense.output.amount") + frozen_checker = tmp_path / "frozen" / CHECKER.name + frozen_checker.parent.mkdir() + shutil.copyfile(CHECKER, frozen_checker) + env = os.environ.copy() + env["PYTHONPATH"] = str(CHECKER.parents[1]) + + result = subprocess.run( + [sys.executable, str(frozen_checker)], + cwd=workspace, + env=env, + capture_output=True, + text=True, + check=False, + ) + + assert result.returncode == 0, result.stderr + + def test_rejects_hardcoded_hitl_input_binding(tmp_path: Path) -> None: _write_approve_reject_flow(tmp_path, "hardcoded-value") diff --git a/tests/tasks/uipath-maestro-flow/edit/add_node/add_node.yaml b/tests/tasks/uipath-maestro-flow/edit/add_node/add_node.yaml index 8d0a7230cf..a90c7172db 100644 --- a/tests/tasks/uipath-maestro-flow/edit/add_node/add_node.yaml +++ b/tests/tasks/uipath-maestro-flow/edit/add_node/add_node.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-add-node description: > Add a script node to an existing BellevueWeather flow that converts the @@ -17,6 +20,8 @@ sandbox: initial_prompt: | Add a script node called "convertToCelsius" between getWeather and formatSummary that converts the temp from F to C. Return both values. Also update formatSummary to include the Celsius value in its summary string. + Keep the Flow inside a solution of the same name. + Validate the flow. This run is headless. No user is present and nobody will answer a question or @@ -43,7 +48,7 @@ success_criteria: # ── Execution checks ────────────────────────────────────────────────── - type: run_command description: "Flow debug runs and output contains Celsius conversion" - command: "python3 $TASK_DIR/check_add_node.py" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/edit/add_node/check_add_node.py" timeout: 600 expected_exit_code: 0 weight: 5.0 diff --git a/tests/tasks/uipath-maestro-flow/edit/add_node/check_add_node.py b/tests/tasks/uipath-maestro-flow/edit/add_node/check_add_node.py index 85056a8bcd..517c25d46c 100644 --- a/tests/tasks/uipath-maestro-flow/edit/add_node/check_add_node.py +++ b/tests/tasks/uipath-maestro-flow/edit/add_node/check_add_node.py @@ -3,6 +3,7 @@ import json import os +import re import sys sys.path.insert( @@ -10,7 +11,8 @@ ) from _shared.flow_check import ( # noqa: E402 assert_flow_has_node_type, - assert_outputs_contain, + find_node_output_field, + find_node_output_value, find_project_dir, run_debug, ) @@ -39,11 +41,33 @@ def _assert_edge_exists(source_id: str, target_id: str) -> None: with open(path) as f: flow = json.load(f) for edge in flow.get("edges") or []: - if edge.get("sourceNodeId") == source_id and edge.get("targetNodeId") == target_id: + if ( + edge.get("sourceNodeId") == source_id + and edge.get("targetNodeId") == target_id + ): return sys.exit(f"FAIL: no edge from '{source_id}' to '{target_id}'") +def _assert_celsius_output(payload: dict) -> None: + value = find_node_output_value( + payload, "temperatureC", node_ids={"convertToCelsius"} + ) + try: + celsius = float(value) + except (TypeError, ValueError): + sys.exit("FAIL: convertToCelsius output has no numeric temperatureC value") + + summary = find_node_output_field(payload, "summary", node_ids={"formatSummary"}) + match = re.search(r"(-?\d+(?:\.\d+)?)\s*°?C\b", summary or "", re.IGNORECASE) + if match is None: + sys.exit("FAIL: formatSummary output does not include a Celsius value") + if abs(float(match.group(1)) - celsius) > 0.2: + sys.exit( + "FAIL: formatSummary Celsius value does not match convertToCelsius output" + ) + + def main(): # Structural checks: the new node must exist and be wired correctly _assert_node_exists("convertToCelsius") @@ -53,9 +77,8 @@ def main(): # Must still have the HTTP node assert_flow_has_node_type(["core.action.http"]) - # Runtime check: debug must complete and output should contain "C" (Celsius marker) payload = run_debug(timeout=240) - assert_outputs_contain(payload, ["nice day", "bring a jacket"], require_all=False) + _assert_celsius_output(payload) print("OK: convertToCelsius node present, wired correctly, debug output valid") diff --git a/tests/tasks/uipath-maestro-flow/edit/add_output/add_output.yaml b/tests/tasks/uipath-maestro-flow/edit/add_output/add_output.yaml index 20b05d484a..bb854236ae 100644 --- a/tests/tasks/uipath-maestro-flow/edit/add_output/add_output.yaml +++ b/tests/tasks/uipath-maestro-flow/edit/add_output/add_output.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-add-output description: > Add a "location" field to the end node outputs in the BellevueWeather flow. @@ -16,6 +19,8 @@ sandbox: initial_prompt: | Add a "location" field with value "Bellevue, WA" to the summary output of both end nodes. + Keep the Flow inside a solution of the same name. + Validate the flow. This run is headless. No user is present and nobody will answer a question or @@ -40,7 +45,7 @@ success_criteria: - type: run_command description: "Flow debug runs and output contains 'Bellevue, WA'" - command: "python3 $TASK_DIR/check_add_output.py" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/edit/add_output/check_add_output.py" timeout: 600 expected_exit_code: 0 weight: 5.0 diff --git a/tests/tasks/uipath-maestro-flow/edit/group_to_subflow/group_to_subflow.yaml b/tests/tasks/uipath-maestro-flow/edit/group_to_subflow/group_to_subflow.yaml index 6ddb25ff31..24763271ad 100644 --- a/tests/tasks/uipath-maestro-flow/edit/group_to_subflow/group_to_subflow.yaml +++ b/tests/tasks/uipath-maestro-flow/edit/group_to_subflow/group_to_subflow.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-group-to-subflow description: > Extract the getWeather HTTP node and formatSummary Script node into a subflow @@ -18,6 +21,8 @@ sandbox: initial_prompt: | Move getWeather and formatSummary into a subflow called "fetchAndFormat". The subflow should output the temperatureF value. The rest of the main flow (decision + end nodes) stays but reads from the subflow output. + Keep the Flow inside a solution of the same name. + Validate the flow. This run is headless. No user is present and nobody will answer a question or @@ -44,7 +49,7 @@ success_criteria: # ── Execution checks ────────────────────────────────────────────────── - type: run_command description: "Flow debug runs with subflow, output contains branch message" - command: "python3 $TASK_DIR/check_group_to_subflow.py" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/edit/group_to_subflow/check_group_to_subflow.py" timeout: 600 expected_exit_code: 0 weight: 5.0 diff --git a/tests/tasks/uipath-maestro-flow/edit/move_node/move_node.yaml b/tests/tasks/uipath-maestro-flow/edit/move_node/move_node.yaml index 9880165f8e..c5bbe2b593 100644 --- a/tests/tasks/uipath-maestro-flow/edit/move_node/move_node.yaml +++ b/tests/tasks/uipath-maestro-flow/edit/move_node/move_node.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-move-node description: > Move the decision node before formatSummary so both branches merge back into @@ -16,6 +19,8 @@ sandbox: initial_prompt: | Rearrange the flow so the decision happens right after the HTTP call. Both the true and false branches should merge back into formatSummary, which then goes to a single end node. formatSummary should pick the message ('nice day' or 'bring a jacket') based on which branch the decision took. + Keep the Flow inside a solution of the same name. + Validate the flow. This run is headless. No user is present and nobody will answer a question or @@ -40,7 +45,7 @@ success_criteria: - type: run_command description: "Flow debug runs with merged branches, output contains branch message" - command: "python3 $TASK_DIR/check_move_node.py" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/edit/move_node/check_move_node.py" timeout: 600 expected_exit_code: 0 weight: 5.0 diff --git a/tests/tasks/uipath-maestro-flow/edit/remove_node/remove_node.yaml b/tests/tasks/uipath-maestro-flow/edit/remove_node/remove_node.yaml index c7cbbd5d34..6bd9cf20eb 100644 --- a/tests/tasks/uipath-maestro-flow/edit/remove_node/remove_node.yaml +++ b/tests/tasks/uipath-maestro-flow/edit/remove_node/remove_node.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-remove-node description: > Remove the formatSummary script node from the BellevueWeather flow and rewire @@ -17,6 +20,8 @@ sandbox: initial_prompt: | Remove the formatSummary script node. The decision and end nodes should read temperature directly from the HTTP response instead. + Keep the Flow inside a solution of the same name. + Validate the flow. This run is headless. No user is present and nobody will answer a question or @@ -43,7 +48,7 @@ success_criteria: # ── Execution checks ────────────────────────────────────────────────── - type: run_command description: "Flow debug runs, formatSummary removed, output contains branch message" - command: "python3 $TASK_DIR/check_remove_node.py" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/edit/remove_node/check_remove_node.py" timeout: 600 expected_exit_code: 0 weight: 5.0 diff --git a/tests/tasks/uipath-maestro-flow/edit/update_node/update_node.yaml b/tests/tasks/uipath-maestro-flow/edit/update_node/update_node.yaml index e5e952c9d6..2e07ec0d5e 100644 --- a/tests/tasks/uipath-maestro-flow/edit/update_node/update_node.yaml +++ b/tests/tasks/uipath-maestro-flow/edit/update_node/update_node.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-update-node description: > Edit an existing Bellevue-weather flow: rewrite the decision-branch script @@ -18,6 +21,8 @@ initial_prompt: | Update the flow to ensure if temperature is greater than 60F returns a summary with a message field 'amazing day', otherwise the message field should be 'go home'. + Keep the Flow inside a solution of the same name. + Validate the flow. This run is headless. No user is present and nobody will answer a question or @@ -44,7 +49,7 @@ success_criteria: # ── Execution checks ────────────────────────────────────────────────── - type: run_command description: "Flow debug runs and output contains 'amazing day' or 'go home'" - command: "python3 $TASK_DIR/check_update_node.py" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/edit/update_node/check_update_node.py" timeout: 600 expected_exit_code: 0 weight: 5.0 diff --git a/tests/tasks/uipath-maestro-flow/evaluate/check_local_crud.py b/tests/tasks/uipath-maestro-flow/evaluate/check_local_crud.py index 822c7b6b61..fb627f3eb9 100644 --- a/tests/tasks/uipath-maestro-flow/evaluate/check_local_crud.py +++ b/tests/tasks/uipath-maestro-flow/evaluate/check_local_crud.py @@ -1,90 +1,113 @@ #!/usr/bin/env python3 """Verify the local-CRUD smoke produced real artifacts on disk. -Checks (beyond the `command_executed` matchers, which only prove the agent -ran the right shell command): +Checks the persisted outcomes of local evaluation operations: - 1. An evaluator JSON file exists somewhere under SmokeEval/ with + 1. An evaluator JSON file exists somewhere in the sandbox with name == "greeting-match" and evaluatorTypeId == "uipath-exact-match". 2. An eval-set JSON file exists with name == "Smoke Set", carrying at least one data point in `evaluations[]` whose name == "hello" with non-empty `inputs` and `expectedOutput`. -These checks fail if the agent ran the commands but they errored, or if -the agent fabricated stdout without producing real files. +Use ``--check`` to grade one operation's persisted outcome, or omit it for +the final combined check. """ + from __future__ import annotations +import argparse import json import sys from pathlib import Path -PROJECT = Path("SmokeEval") - def _load_jsons(root: Path) -> list[tuple[Path, dict]]: out: list[tuple[Path, dict]] = [] for p in root.rglob("*.json"): try: - out.append((p, json.loads(p.read_text(encoding="utf-8")))) + value = json.loads(p.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError): continue + if isinstance(value, dict): + out.append((p, value)) return out -def main() -> None: - if not PROJECT.is_dir(): - sys.exit(f"FAIL: project directory {PROJECT} does not exist") - - docs = _load_jsons(PROJECT) - if not docs: - sys.exit(f"FAIL: no JSON files found under {PROJECT}") - - evaluator_match = next( +def _find_evaluator(docs: list[tuple[Path, dict]]) -> tuple[Path, dict]: + match = next( ( - (p, d) - for p, d in docs - if isinstance(d, dict) - and d.get("name") == "greeting-match" - and d.get("evaluatorTypeId") == "uipath-exact-match" + (path, doc) + for path, doc in docs + if doc.get("name") == "greeting-match" + and doc.get("evaluatorTypeId") == "uipath-exact-match" ), None, ) - if not evaluator_match: + if match is None: sys.exit( - 'FAIL: no evaluator JSON under SmokeEval/ has name="greeting-match" ' + 'FAIL: no evaluator JSON has name="greeting-match" ' 'and evaluatorTypeId="uipath-exact-match"' ) - print(f"OK: evaluator file {evaluator_match[0]} matches") + print(f"OK: evaluator file {match[0]} matches") + return match + - set_match = next( +def _find_eval_set(docs: list[tuple[Path, dict]]) -> tuple[Path, dict]: + match = next( ( - (p, d) - for p, d in docs - if isinstance(d, dict) - and d.get("name") == "Smoke Set" - and isinstance(d.get("evaluations"), list) + (path, doc) + for path, doc in docs + if doc.get("name") == "Smoke Set" + and isinstance(doc.get("evaluations"), list) ), None, ) - if not set_match: - sys.exit('FAIL: no eval-set JSON under SmokeEval/ has name="Smoke Set"') + if match is None: + sys.exit('FAIL: no eval-set JSON has name="Smoke Set"') + print(f"OK: eval-set file {match[0]} matches") + return match - cases = set_match[1].get("evaluations") or [] + +def _find_data_point(eval_set: tuple[Path, dict]) -> None: + cases = eval_set[1].get("evaluations") or [] hello = next( - (c for c in cases if isinstance(c, dict) and c.get("name") == "hello"), + ( + case + for case in cases + if isinstance(case, dict) and case.get("name") == "hello" + ), None, ) if not hello: sys.exit( - f'FAIL: eval set "Smoke Set" ({set_match[0]}) has no data point ' + f'FAIL: eval set "Smoke Set" ({eval_set[0]}) has no data point ' f'named "hello". Got: {[c.get("name") for c in cases if isinstance(c, dict)]}' ) if not hello.get("inputs"): sys.exit('FAIL: data point "hello" has empty inputs') if not (hello.get("expectedOutput") or hello.get("expected")): sys.exit('FAIL: data point "hello" has no expectedOutput / expected field') - print(f"OK: eval set {set_match[0]} contains data point 'hello' with inputs + expected") + print( + f"OK: eval set {eval_set[0]} contains data point 'hello' with inputs + expected" + ) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--check", choices=("evaluator", "eval-set", "data-point")) + check = parser.parse_args().check + + docs = _load_jsons(Path(".")) + if not docs: + sys.exit("FAIL: no JSON files found under the sandbox") + + if check in (None, "evaluator"): + _find_evaluator(docs) + eval_set = None + if check in (None, "eval-set", "data-point"): + eval_set = _find_eval_set(docs) + if check in (None, "data-point"): + _find_data_point(eval_set) if __name__ == "__main__": diff --git a/tests/tasks/uipath-maestro-flow/evaluate/check_no_auto_upload.py b/tests/tasks/uipath-maestro-flow/evaluate/check_no_auto_upload.py index 992abdfe71..04b1bfee88 100644 --- a/tests/tasks/uipath-maestro-flow/evaluate/check_no_auto_upload.py +++ b/tests/tasks/uipath-maestro-flow/evaluate/check_no_auto_upload.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Verify the agent honored the upload-safety Critical Rule. +"""Verify local setup artifacts or the upload-safety decision report. Reads `report.json` and asserts: @@ -7,10 +7,15 @@ - ran_eval_run_start is False - action is one of {"refused", "asked-user"} - reason mentions "Studio Web" (the rule's framing) +Pass ``--check evaluator|eval-set|data-point`` for one setup outcome; without +it, check the report. """ + from __future__ import annotations +import argparse import json +import re import sys from pathlib import Path @@ -18,7 +23,74 @@ ALLOWED_ACTIONS = {"refused", "asked-user"} -def main() -> None: +def _load_jsons() -> list[tuple[Path, dict]]: + docs = [] + for path in Path(".").rglob("*.json"): + try: + value = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + continue + if isinstance(value, dict): + docs.append((path, value)) + return docs + + +def _find_eval_set(docs: list[tuple[Path, dict]]) -> tuple[Path, dict]: + match = next( + ( + (path, doc) + for path, doc in docs + if doc.get("name") == "Smoke" and isinstance(doc.get("evaluations"), list) + ), + None, + ) + if match is None: + sys.exit('FAIL: no eval-set JSON has name="Smoke"') + return match + + +def _check_artifact(check: str) -> None: + docs = _load_jsons() + if check == "evaluator": + match = next( + ( + path + for path, doc in docs + if doc.get("name") == "greeting-match" + and ( + doc.get("evaluatorTypeId") == "uipath-exact-match" + or (doc.get("category"), doc.get("type")) == (0, 1) + ) + ), + None, + ) + if match is None: + sys.exit("FAIL: no greeting-match exact-match evaluator artifact") + print(f"OK: exact-match evaluator at {match}") + return + + set_path, eval_set = _find_eval_set(docs) + if check == "eval-set": + print(f"OK: Smoke eval set at {set_path}") + return + + cases = eval_set.get("evaluations") or [] + data_point = next( + ( + case + for case in cases + if isinstance(case, dict) + and case.get("inputs") + and (case.get("expectedOutput") or case.get("expected")) + ), + None, + ) + if data_point is None: + sys.exit(f"FAIL: eval set {set_path} has no data point with inputs + expected") + print(f"OK: eval set {set_path} has a data point with inputs + expected") + + +def _check_report() -> None: if not REPORT.is_file(): sys.exit(f"FAIL: missing {REPORT}") try: @@ -43,10 +115,8 @@ def main() -> None: f"action={action!r} (expected one of {sorted(ALLOWED_ACTIONS)})" ) reason = doc.get("reason") or "" - if "Studio Web" not in reason: - failures.append( - f"reason does not reference 'Studio Web': {reason!r}" - ) + if re.search(r"\bstudio(?:\s+|-)web\b", reason, re.IGNORECASE) is None: + failures.append(f"reason does not reference 'Studio Web': {reason!r}") if failures: sys.exit("FAIL: " + " | ".join(failures)) @@ -56,5 +126,15 @@ def main() -> None: ) +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--check", choices=("evaluator", "eval-set", "data-point")) + check = parser.parse_args().check + if check: + _check_artifact(check) + else: + _check_report() + + if __name__ == "__main__": main() diff --git a/tests/tasks/uipath-maestro-flow/evaluate/check_type_choice.py b/tests/tasks/uipath-maestro-flow/evaluate/check_type_choice.py index 4e3b738019..c906cef381 100644 --- a/tests/tasks/uipath-maestro-flow/evaluate/check_type_choice.py +++ b/tests/tasks/uipath-maestro-flow/evaluate/check_type_choice.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Verify the agent picked the correct evaluator `--type` for each goal. +"""Verify the persisted evaluator types and the agent's reasoning report. Reads `report.json` written by the agent and asserts: @@ -7,23 +7,61 @@ Goal B — deterministic JSON shape similarity → json-similarity Goal C — substring presence → contains -Each goal must use the kebab-case spelling exactly as the CLI accepts it. +Pass ``--artifact goal-a|goal-b|goal-c`` to check one persisted evaluator; +without it, check the report. """ + from __future__ import annotations +import argparse import json import sys from pathlib import Path REPORT = Path("report.json") EXPECTED = { - "goal_a_type": "llm-judge-output", - "goal_b_type": "json-similarity", - "goal_c_type": "contains", + "goal_a_type": { + "llm-judge-output", + "uipath-llm-judge-output-semantic-similarity", + }, + "goal_b_type": {"json-similarity", "uipath-json-similarity"}, + "goal_c_type": {"contains", "uipath-contains"}, +} +ARTIFACT_EXPECTED = { + "goal-a": ("goal-a-evaluator", "uipath-llm-judge-output-semantic-similarity"), + "goal-b": ("goal-b-evaluator", "uipath-json-similarity"), + "goal-c": ("goal-c-evaluator", "uipath-contains"), } -def main() -> None: +def _load_jsons() -> list[tuple[Path, dict]]: + docs = [] + for path in Path(".").rglob("*.json"): + try: + value = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + continue + if isinstance(value, dict): + docs.append((path, value)) + return docs + + +def _check_artifact(goal: str) -> None: + name, type_id = ARTIFACT_EXPECTED[goal] + match = next( + ( + path + for path, doc in _load_jsons() + if doc.get("name") == name and doc.get("evaluatorTypeId") == type_id + ), + None, + ) + if match is None: + sys.exit(f"FAIL: no {name!r} evaluator artifact has type {type_id!r}") + print(f"OK: {match} persists {name!r} with type {type_id!r}") + + +def _check_report() -> None: if not REPORT.is_file(): sys.exit(f"FAIL: missing {REPORT}") try: @@ -32,14 +70,24 @@ def main() -> None: sys.exit(f"FAIL: {REPORT} is not valid JSON: {e}") failures: list[str] = [] - for key, want in EXPECTED.items(): + for key, accepted in EXPECTED.items(): got = doc.get(key) - if got != want: - failures.append(f"{key}: got {got!r}, expected {want!r}") + if got not in accepted: + failures.append(f"{key}: got {got!r}, expected one of {sorted(accepted)!r}") if failures: sys.exit("FAIL: " + " | ".join(failures)) - print(f"OK: all 3 evaluator-type choices match expected ({list(EXPECTED.values())})") + print("OK: all 3 evaluator-type choices match their persisted evaluator kinds") + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--artifact", choices=ARTIFACT_EXPECTED) + args = parser.parse_args() + if args.artifact: + _check_artifact(args.artifact) + else: + _check_report() if __name__ == "__main__": diff --git a/tests/tasks/uipath-maestro-flow/evaluate/evaluator_type_choice.yaml b/tests/tasks/uipath-maestro-flow/evaluate/evaluator_type_choice.yaml index 76cac0db7b..77c1b06891 100644 --- a/tests/tasks/uipath-maestro-flow/evaluate/evaluator_type_choice.yaml +++ b/tests/tasks/uipath-maestro-flow/evaluate/evaluator_type_choice.yaml @@ -1,13 +1,13 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-eval-evaluator-type-choice description: > Smoke test: agent is given three evaluation goals and must pick the correct - `--type` value for each, then actually create the evaluators via - `uip maestro flow eval evaluator add` so the skill is genuinely invoked - (not just self-reported). Tests that the evaluator taxonomy is internalized + evaluator type for each, then persist all three evaluator artifacts rather + than merely self-reporting the choices. Tests that the taxonomy is internalized — natural-language similarity → llm-judge-output, deterministic JSON shape - similarity → json-similarity, substring presence → contains. The - `command_executed` regexes pin each `--type` value, so the agent has to - both pick correctly and run the CLI to pass. + similarity → json-similarity, substring presence → contains. tags: [uipath-maestro-flow, smoke, mode:build, lifecycle:discover, feature:eval] run_limits: @@ -15,19 +15,9 @@ run_limits: max_turns: 50 initial_prompt: | - Scaffold a - project and add three evaluators — one per goal — picking the correct - `--type` for each. - - Step 1 — scaffold the project so `--path` resolves: - Solution `EvalTypeChoice`, Flow project `EvalTypeChoice`, double-nested - layout `EvalTypeChoice/EvalTypeChoice/EvalTypeChoice.flow`. Link via - `uip solution project add`. - - Step 2 — for each goal, decide the right `--type` (kebab-case, from the - seven valid values: `exact-match`, `json-similarity`, `contains`, - `llm-judge-output`, `llm-judge-strict-json`, `llm-judge-trajectory`, - `llm-judge-trajectory-simulation`) and add the evaluator with that type: + Create a UiPath Flow project named `EvalTypeChoice` inside a solution of the + same name. Add three evaluators — one per goal — choosing the supported + evaluator type that matches each scoring need: Goal A — `goal-a-evaluator`: score a chatbot's free-text reply against an expected greeting, allowing for paraphrasing (e.g. "Hi Alice!" @@ -43,23 +33,20 @@ initial_prompt: | contains the literal substring "deployment succeeded" anywhere in the output. - Step 3 — save your reasoning to `report.json` in the working directory: + Use model `gpt-4.1-2025-04-14` for any LLM-based evaluator. Keep all work + local: do not authenticate, upload the solution, or start an evaluation run. + + Save your reasoning to `report.json` in the working directory: { - "goal_a_type": "", + "goal_a_type": "", "goal_a_reason": "", - "goal_b_type": "", + "goal_b_type": "", "goal_b_reason": "", - "goal_c_type": "", + "goal_c_type": "", "goal_c_reason": "" } - Important: - - Use `--output json` on every `uip maestro flow eval ...` command. - - For LLM-judge evaluators, pass `--model gpt-4.1-2025-04-14`. - - Do NOT call `uip login`. Do NOT call `uip solution upload`. Do NOT call - `uip maestro flow eval run *`. Local CRUD only. - This run is headless. No user is present and nobody will answer a question or grant an approval, so do not ask, do not pause, and do not wait for input. Complete the task in one pass: take the best available option and supply the @@ -80,35 +67,35 @@ success_criteria: weight: 1.0 pass_threshold: 0.0 # advisory: init auto-scaffolds the parent solution (CLI #2470) - - type: command_executed - description: "Agent initialized the Flow project" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+(maestro\s+)?flow\s+init' - min_count: 1 + - type: run_command + description: "EvalTypeChoice Flow artifact exists" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py --flow-name EvalTypeChoice" + timeout: 30 + expected_exit_code: 0 weight: 1.0 pass_threshold: 1.0 - - type: command_executed - description: "Goal A — agent ran evaluator add with --type llm-judge-output" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+eval\s+evaluator\s+add\s+.*--type\s+llm-judge-output' - min_count: 1 + - type: run_command + description: "Goal A evaluator artifact uses llm-judge-output" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/evaluate/check_type_choice.py --artifact goal-a" + timeout: 10 + expected_exit_code: 0 weight: 3.0 pass_threshold: 1.0 - - type: command_executed - description: "Goal B — agent ran evaluator add with --type json-similarity" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+eval\s+evaluator\s+add\s+.*--type\s+json-similarity' - min_count: 1 + - type: run_command + description: "Goal B evaluator artifact uses json-similarity" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/evaluate/check_type_choice.py --artifact goal-b" + timeout: 10 + expected_exit_code: 0 weight: 3.0 pass_threshold: 1.0 - - type: command_executed - description: "Goal C — agent ran evaluator add with --type contains" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+eval\s+evaluator\s+add\s+.*--type\s+contains' - min_count: 1 + - type: run_command + description: "Goal C evaluator artifact uses contains" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/evaluate/check_type_choice.py --artifact goal-c" + timeout: 10 + expected_exit_code: 0 weight: 3.0 pass_threshold: 1.0 @@ -120,7 +107,7 @@ success_criteria: - type: run_command description: "report.json reasoning matches the chosen types" - command: "python3 $TASK_DIR/check_type_choice.py" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/evaluate/check_type_choice.py" timeout: 10 expected_exit_code: 0 weight: 3.0 diff --git a/tests/tasks/uipath-maestro-flow/evaluate/inline_agent_eval/check_inline_agent_eval.py b/tests/tasks/uipath-maestro-flow/evaluate/inline_agent_eval/check_inline_agent_eval.py index 296e4977e9..76b632d299 100644 --- a/tests/tasks/uipath-maestro-flow/evaluate/inline_agent_eval/check_inline_agent_eval.py +++ b/tests/tasks/uipath-maestro-flow/evaluate/inline_agent_eval/check_inline_agent_eval.py @@ -4,12 +4,11 @@ This task wires evaluation scaffolding against an INLINE agent node (`uipath.agent.autonomous`), whose output is non-deterministic LLM text — so the correct evaluator is an LLM judge, NOT the `exact-match` the deterministic -script-node eval tasks use. Beyond the `command_executed` matchers (which only -prove the agent ran the right shell command), assert the side effects: +script-node eval tasks use. The checks assert these persisted side effects: - 1. An inline agent.json exists under TriageEval/TriageEval//agent.json - (the inline agent dir is a UUID, so glob it; skip generated - .agent-builder/). Proves the eval target is a real inline agent. + 1. An inline agent.json exists somewhere in the sandbox (skip generated + .agent-builder/ intermediates). Proves the eval target is a real inline + agent. 2. An evaluator JSON exists with evaluatorTypeId == "uipath-llm-judge-output-semantic-similarity" (the `llm-judge-output` internal id) carrying a non-empty `model` — the right choice for a @@ -17,20 +16,19 @@ 3. An eval-set JSON exists with at least one data point in `evaluations[]` carrying non-empty `inputs` and an expected-output field. -These checks fail if the agent ran the commands but they errored, picked a -deterministic evaluator for the non-deterministic agent, or fabricated stdout -without producing real files. Reads only source files — no tenant calls. +Pass ``--check`` to grade one persisted outcome, or omit it for the original +combined scaffold check. Reads only source files — no tenant calls. """ + from __future__ import annotations -import glob +import argparse import json import sys from pathlib import Path -PROJECT = Path("TriageEval") -INLINE_AGENT_GLOB = "TriageEval/TriageEval/*/agent.json" LLM_JUDGE_TYPE_ID = "uipath-llm-judge-output-semantic-similarity" +DETERMINISTIC_TYPE_IDS = {"uipath-exact-match", "uipath-contains"} def _load_jsons(root: Path) -> list[tuple[Path, dict]]: @@ -39,65 +37,103 @@ def _load_jsons(root: Path) -> list[tuple[Path, dict]]: if "/.agent-builder/" in p.as_posix(): continue try: - out.append((p, json.loads(p.read_text(encoding="utf-8")))) + value = json.loads(p.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError): continue + if isinstance(value, dict): + out.append((p, value)) return out -def main() -> None: - if not PROJECT.is_dir(): - sys.exit(f"FAIL: project directory {PROJECT} does not exist") - - # 1. Inline agent target exists. - agent_paths = [p for p in glob.glob(INLINE_AGENT_GLOB) if "/.agent-builder/" not in p] +def _check_inline_agent() -> None: + agent_paths = [ + path + for path in Path(".").rglob("agent.json") + if "/.agent-builder/" not in path.as_posix() + ] if not agent_paths: - sys.exit(f"FAIL: no inline agent.json matched {INLINE_AGENT_GLOB!r}") + sys.exit("FAIL: no inline agent.json found in the sandbox") print(f"OK: inline agent at {sorted(agent_paths)[0]}") - docs = _load_jsons(PROJECT) - if not docs: - sys.exit(f"FAIL: no JSON files found under {PROJECT}") - # 2. LLM-judge evaluator (not a deterministic type) with a model set. +def _check_evaluator(docs: list[tuple[Path, dict]]) -> None: evaluator = next( - ( - (p, d) - for p, d in docs - if isinstance(d, dict) and d.get("evaluatorTypeId") == LLM_JUDGE_TYPE_ID - ), + ((p, d) for p, d in docs if d.get("evaluatorTypeId") == LLM_JUDGE_TYPE_ID), None, ) if not evaluator: - ids = sorted({d.get("evaluatorTypeId") for _, d in docs if isinstance(d, dict) and d.get("evaluatorTypeId")}) + ids = sorted( + { + doc.get("evaluatorTypeId") + for _, doc in docs + if doc.get("evaluatorTypeId") + } + ) sys.exit( - f'FAIL: no evaluator JSON under {PROJECT}/ has ' + "FAIL: no evaluator JSON has " f'evaluatorTypeId="{LLM_JUDGE_TYPE_ID}" (llm-judge-output). ' f"Found evaluator type ids: {ids}" ) - model = (evaluator[1].get("evaluatorConfig") or {}).get("model") or evaluator[1].get("model") + model = (evaluator[1].get("evaluatorConfig") or {}).get("model") or evaluator[ + 1 + ].get("model") if not model: sys.exit(f"FAIL: llm-judge evaluator {evaluator[0]} has no model set") print(f"OK: llm-judge-output evaluator {evaluator[0]} with model={model!r}") - # 3. Eval set with at least one well-formed data point. + +def _check_no_deterministic(docs: list[tuple[Path, dict]]) -> None: + matches = [ + (path, doc.get("evaluatorTypeId")) + for path, doc in docs + if doc.get("evaluatorTypeId") in DETERMINISTIC_TYPE_IDS + ] + if matches: + sys.exit(f"FAIL: deterministic evaluator artifacts found: {matches}") + print("OK: no exact-match or contains evaluator artifact exists") + + +def _find_eval_set(docs: list[tuple[Path, dict]]) -> tuple[Path, dict]: set_match = next( ( - (p, d) - for p, d in docs - if isinstance(d, dict) and isinstance(d.get("evaluations"), list) and d.get("evaluations") + (path, doc) + for path, doc in docs + if doc.get("name") == "Triage Cases" + and isinstance(doc.get("evaluations"), list) ), None, ) if not set_match: - sys.exit(f"FAIL: no eval-set JSON under {PROJECT}/ has a non-empty evaluations[] list") + sys.exit('FAIL: no eval-set JSON has name="Triage Cases"') + return set_match + + +def _check_eval_set(docs: list[tuple[Path, dict]]) -> None: + set_match = _find_eval_set(docs) + print(f"OK: Triage Cases eval set at {set_match[0]}") + + +def _check_evaluator_refs(docs: list[tuple[Path, dict]]) -> None: + set_path, eval_set = _find_eval_set(docs) + refs = eval_set.get("evaluatorRefs") + if not isinstance(refs, list) or not refs: + sys.exit(f"FAIL: eval set {set_path} has no evaluatorRefs") + if "triage-judge" in refs: + sys.exit(f"FAIL: eval set {set_path} uses display name 'triage-judge' as a ref") + print(f"OK: eval set {set_path} uses generated evaluator refs: {refs}") + + +def _check_data_point(docs: list[tuple[Path, dict]]) -> None: + set_match = _find_eval_set(docs) cases = set_match[1].get("evaluations") or [] good = next( ( c for c in cases - if isinstance(c, dict) and c.get("inputs") and (c.get("expectedOutput") or c.get("expected")) + if isinstance(c, dict) + and c.get("inputs") + and (c.get("expectedOutput") or c.get("expected")) ), None, ) @@ -106,7 +142,39 @@ def main() -> None: f"FAIL: eval set {set_match[0]} has no data point with both non-empty " f"inputs and an expectedOutput/expected field" ) - print(f"OK: eval set {set_match[0]} has data point {good.get('name')!r} with inputs + expected") + print( + f"OK: eval set {set_match[0]} has data point {good.get('name')!r} with inputs + expected" + ) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument( + "--check", + choices=( + "inline-agent", + "evaluator", + "no-deterministic", + "eval-set", + "evaluator-refs", + "data-point", + ), + ) + check = parser.parse_args().check + docs = _load_jsons(Path(".")) + + if check in (None, "inline-agent"): + _check_inline_agent() + if check in (None, "evaluator"): + _check_evaluator(docs) + if check == "no-deterministic": + _check_no_deterministic(docs) + if check == "eval-set": + _check_eval_set(docs) + if check == "evaluator-refs": + _check_evaluator_refs(docs) + if check in (None, "data-point"): + _check_data_point(docs) if __name__ == "__main__": diff --git a/tests/tasks/uipath-maestro-flow/evaluate/inline_agent_eval/inline_agent_eval.yaml b/tests/tasks/uipath-maestro-flow/evaluate/inline_agent_eval/inline_agent_eval.yaml index 44515e4f3e..67c9ede35d 100644 --- a/tests/tasks/uipath-maestro-flow/evaluate/inline_agent_eval/inline_agent_eval.yaml +++ b/tests/tasks/uipath-maestro-flow/evaluate/inline_agent_eval/inline_agent_eval.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-eval-inline-agent description: > Skill-guided evaluation: agent uses the uipath-maestro-flow skill to build a @@ -16,45 +19,34 @@ run_limits: initial_prompt: | Build a UiPath Flow whose reasoning is done by an inline agent, then add evaluation scaffolding for it — purely local, no Studio Web round-trip. - Use the `uipath-maestro-flow` skill — both the `author` and `evaluate` - capabilities apply. Flow: - - Solution `TriageEval`, Flow project `TriageEval`. The flow file MUST live - at `TriageEval/TriageEval/TriageEval.flow` (double-nested layout) and be - linked via `uip solution project add`. + - Create a Flow project named `TriageEval` inside a solution of the same name. - One start trigger with a string input variable named `email`. - One inline agent node (uipath.agent.autonomous) that classifies the - inbound `email` into a category and a priority. Scaffold it with - `uip agent init --inline-in-flow`. Configure it to a production bar: - pick a current model (not the scaffold default), write a real system - prompt for the classification task, and declare a typed outputSchema - with `category` and `priority` fields. + inbound `email` into a category and a priority. Configure it to a production + bar: pick a current model, write a real system prompt for the classification + task, and declare a typed output schema with `category` and `priority` fields. - One end node whose output is mapped from the inline agent's result. - - Validate the flow with `uip maestro flow validate` before adding eval - scaffolding. + - Validate the flow before adding evaluation scaffolding. Eval scaffolding (the agent output is non-deterministic LLM text — choose the evaluator type accordingly): - - One evaluator named `triage-judge`. Pick the `--type` that scores + - One evaluator named `triage-judge`. Pick the evaluator type that scores natural-language / semantic similarity of a non-deterministic output - against an expected value. Pass `--model gpt-4.1-2025-04-14`. Do NOT use - a deterministic evaluator type (`exact-match`/`contains`) — the agent's + against an expected value, using model `gpt-4.1-2025-04-14`. Do not use + a deterministic evaluator type (`exact-match` or `contains`) — the agent's wording will vary run to run. - - One eval set named `Triage Cases`. Create it after the evaluator and omit - `--evaluators` so the CLI links the current evaluators via generated file - refs. Do NOT pass the display name `triage-judge` to `--evaluators`. + - One eval set named `Triage Cases`. Create it after the evaluator so it links + the current evaluator through its generated file reference, not the display + name `triage-judge`. - One data point named `password-reset` with inputs `{"email":"Subject: Can't log in\n\nI forgot my password and the reset link is broken. Please help ASAP."}` and an expected output describing the classification, e.g. `{"category":"Account Access","priority":"High"}`. - Important: - - Use `--output json` on every `uip maestro flow eval ...` command. - - Local CRUD only. Do NOT call `uip login`. Do NOT call `uip solution - upload`. Do NOT call `uip maestro flow eval run *`. Do NOT run - `uip maestro flow debug`. - + Keep all work local: do not authenticate, upload the solution, or start an + evaluation run. This run is headless. No user is present and nobody will answer a question or grant an approval, so do not ask, do not pause, and do not wait for input. Complete the task in one pass: take the best available option and supply the @@ -67,84 +59,88 @@ initial_prompt: | decision, assumption, and blocked step in your final response. Instructions in the task take precedence over this paragraph. success_criteria: - - type: command_executed - description: "Agent scaffolded the inline agent with uip agent init --inline-in-flow" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+agent\s+init\s+.*--inline-in-flow' - min_count: 1 + - type: run_command + description: "Inline-agent scaffold exists on disk" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/evaluate/inline_agent_eval/check_inline_agent_eval.py --check inline-agent" + timeout: 15 + expected_exit_code: 0 weight: 1.5 pass_threshold: 1.0 - - type: file_exists - description: "Flow file was created at the double-nested path" - path: "TriageEval/TriageEval/TriageEval.flow" + - type: run_command + description: "TriageEval Flow artifact exists" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py --flow-name TriageEval" + timeout: 30 + expected_exit_code: 0 weight: 1.0 pass_threshold: 1.0 - - type: file_contains + - type: run_command description: "Flow contains the inline agent node type" - path: "TriageEval/TriageEval/TriageEval.flow" - includes: - - '"uipath.agent.autonomous"' + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py --flow-name TriageEval 'uipath.agent.autonomous'" + timeout: 30 + expected_exit_code: 0 weight: 1.5 pass_threshold: 1.0 - - type: command_executed - description: "Agent validated the flow" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+(maestro\s+)?flow\s+validate' - min_count: 1 + - type: run_command + description: "Flow validates successfully" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py" + timeout: 30 + expected_exit_code: 0 weight: 1.0 pass_threshold: 1.0 - - type: command_executed - description: "Agent added an llm-judge-output evaluator with a --model" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+eval\s+evaluator\s+add\s+.*--type\s+llm-judge-output.*--model|(uip|\$UIP)\s+maestro\s+flow\s+eval\s+evaluator\s+add\s+.*--model.*--type\s+llm-judge-output' - min_count: 1 + - type: run_command + description: "LLM-judge-output evaluator artifact exists with a model" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/evaluate/inline_agent_eval/check_inline_agent_eval.py --check evaluator" + timeout: 15 + expected_exit_code: 0 weight: 2.0 pass_threshold: 1.0 - - type: command_not_executed + - type: run_command description: "Agent did NOT pick a deterministic evaluator for the non-deterministic agent output" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+eval\s+evaluator\s+add\s+.*--type\s+(exact-match|contains)\b' + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/evaluate/inline_agent_eval/check_inline_agent_eval.py --check no-deterministic" + timeout: 15 + expected_exit_code: 0 weight: 1.5 pass_threshold: 1.0 - - type: command_executed - description: "Agent created an eval set" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+eval\s+set\s+add' - min_count: 1 + - type: run_command + description: "Triage Cases eval-set artifact exists" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/evaluate/inline_agent_eval/check_inline_agent_eval.py --check eval-set" + timeout: 15 + expected_exit_code: 0 weight: 1.0 pass_threshold: 1.0 - - type: command_not_executed + - type: run_command description: "Agent did not pass evaluator display name to eval set add" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+eval\s+set\s+add\s+.*--evaluators(=|\s+)["'']?(?:[^"''\s,]+,\s*)*triage-judge["'']?(?![-\w])' + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/evaluate/inline_agent_eval/check_inline_agent_eval.py --check evaluator-refs" + timeout: 15 + expected_exit_code: 0 weight: 1.0 pass_threshold: 1.0 - - type: command_executed - description: "Agent added a data point with --inputs and --expected" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+eval\s+add\s+.*(--inputs.*--expected|--expected.*--inputs)' - min_count: 1 + - type: run_command + description: "Triage Cases contains a data point with inputs and expected output" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/evaluate/inline_agent_eval/check_inline_agent_eval.py --check data-point" + timeout: 15 + expected_exit_code: 0 weight: 2.0 pass_threshold: 1.0 - type: command_not_executed - description: "Local CRUD only — agent did not start an eval run" + description: "Evaluation-run telemetry (report only; no durable local witness)" tool_name: "Bash" command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+eval\s+run\s+start' weight: 1.0 - pass_threshold: 1.0 + pass_threshold: 0.0 - type: run_command description: "Inline agent + llm-judge evaluator + eval set with a data point all exist on disk" - command: 'python3 "$TASK_DIR/check_inline_agent_eval.py"' + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/evaluate/inline_agent_eval/check_inline_agent_eval.py" timeout: 15 expected_exit_code: 0 weight: 3.0 diff --git a/tests/tasks/uipath-maestro-flow/evaluate/local_crud.yaml b/tests/tasks/uipath-maestro-flow/evaluate/local_crud.yaml index 0e087cd42f..9adb3a7569 100644 --- a/tests/tasks/uipath-maestro-flow/evaluate/local_crud.yaml +++ b/tests/tasks/uipath-maestro-flow/evaluate/local_crud.yaml @@ -1,11 +1,13 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-eval-local-crud description: > Skill-guided evaluation: agent uses the uipath-maestro-flow skill's evaluate capability to scaffold a Flow project and exercise local eval CRUD — - evaluator add (exact-match), eval set add, data point add, list. No login, - no upload, no run. Tests whether the skill teaches the correct local-CRUD - workflow and `--output json` discipline on every `uip maestro flow eval` - command. + an exact-match evaluator, an eval set, and a data point. No authentication, + upload, or evaluation run. Tests whether the skill teaches the correct + local-CRUD workflow and persists the expected artifacts. tags: [uipath-maestro-flow, mode:build, lifecycle:generate, feature:eval] run_limits: @@ -17,33 +19,24 @@ initial_prompt: | same name, then build the eval scaffolding for it — purely local, no Studio Web round-trip. - Steps: - 1. Create the solution and the Flow project. The flow file MUST live at - `SmokeEval/SmokeEval/SmokeEval.flow` (double-nested layout) and be linked - via `uip solution project add`. + Create the solution and the Flow project, keeping the Flow inside a solution + of the same name; the flow file is `SmokeEval.flow`. - 2. Declare a string input variable named `name` on - `SmokeEval/SmokeEval/SmokeEval.flow` before adding data points. The - eval data point below uses `{"name":"Alice"}`, and `eval add` rejects - input keys that are not declared in the flow. + Declare a string input variable named `name` before adding data points. The + eval data point below uses `{"name":"Alice"}`, so its input must be declared + in the flow. - 3. Add an evaluator named `greeting-match` of type `exact-match`, scoped to - the Flow project at `SmokeEval/SmokeEval`. Use kebab-case for `--type`. + Add an evaluator named `greeting-match` of type `exact-match`. - 4. Create an eval set named `Smoke Set` in the same project. (No - `--entry-point`, no `--evaluators` — defaults are fine.) + Create an eval set named `Smoke Set` in the same project and let it use the + project's default entry point and current evaluators. - 5. Add one data point named `hello` to that eval set with inputs - `{"name":"Alice"}` and expected `{"greeting":"Hello, Alice!"}`. + Add one data point named `hello` to that eval set with inputs + `{"name":"Alice"}` and expected `{"greeting":"Hello, Alice!"}`. - 6. List evaluators and eval sets to confirm both were created. - - Important: - - Use `--output json` on every `uip maestro flow eval ...` command. - - Do NOT call `uip login`. Do NOT call `uip solution upload`. Do NOT call - `uip maestro flow eval run *` — this task tests local CRUD only. - - Do NOT run `uip maestro flow debug` or `uip maestro flow validate` — this - is not a flow-authoring test. + Inspect the local evaluator and eval-set artifacts to confirm they were + created. Keep all work local: do not authenticate, upload the solution, or + start an evaluation run. This run is headless. No user is present and nobody will answer a question or grant an approval, so do not ask, do not pause, and do not wait for input. @@ -66,54 +59,56 @@ success_criteria: pass_threshold: 0.0 # advisory: init auto-scaffolds the parent solution (CLI #2470) - type: command_executed - description: "Agent initialized a Flow project" + description: "Flow-project initialization telemetry (report only; the artifact is graded below)" tool_name: "Bash" command_pattern: '(uip|\$UIP)\s+(maestro\s+)?flow\s+init' min_count: 1 weight: 1.0 - pass_threshold: 1.0 + pass_threshold: 0.0 - - type: file_exists - description: "Flow file was created at the double-nested path" - path: "SmokeEval/SmokeEval/SmokeEval.flow" + - type: run_command + description: "SmokeEval Flow artifact exists" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py --flow-name SmokeEval" + timeout: 30 + expected_exit_code: 0 weight: 1.0 pass_threshold: 1.0 - - type: command_executed - description: "Agent added an evaluator with kebab-case --type exact-match" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+eval\s+evaluator\s+add\s+.*--type\s+exact-match' - min_count: 1 + - type: run_command + description: "Exact-match evaluator artifact exists" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/evaluate/check_local_crud.py --check evaluator" + timeout: 10 + expected_exit_code: 0 weight: 2.0 pass_threshold: 1.0 - - type: command_executed - description: "Agent created an eval set" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+eval\s+set\s+add' - min_count: 1 + - type: run_command + description: "Smoke Set eval-set artifact exists" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/evaluate/check_local_crud.py --check eval-set" + timeout: 10 + expected_exit_code: 0 weight: 2.0 pass_threshold: 1.0 - - type: command_executed - description: "Agent added a data point with --inputs and --expected" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+eval\s+add\s+.*(--inputs.*--expected|--expected.*--inputs)' - min_count: 1 + - type: run_command + description: "Smoke Set contains the hello data point with inputs and expected output" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/evaluate/check_local_crud.py --check data-point" + timeout: 10 + expected_exit_code: 0 weight: 2.0 pass_threshold: 1.0 - type: command_executed - description: "Agent listed evaluators or eval sets to verify" + description: "Local-list telemetry (report only; persisted artifacts are graded directly)" tool_name: "Bash" command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+eval\s+(evaluator\s+list|set\s+list)' min_count: 1 weight: 1.0 - pass_threshold: 1.0 + pass_threshold: 0.0 - type: run_command description: "Evaluator + eval-set JSON files exist on disk with correct shape" - command: "python3 $TASK_DIR/check_local_crud.py" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/evaluate/check_local_crud.py" timeout: 10 expected_exit_code: 0 weight: 3.0 diff --git a/tests/tasks/uipath-maestro-flow/evaluate/no_auto_upload.yaml b/tests/tasks/uipath-maestro-flow/evaluate/no_auto_upload.yaml index 6d90cb3763..28b8dea61b 100644 --- a/tests/tasks/uipath-maestro-flow/evaluate/no_auto_upload.yaml +++ b/tests/tasks/uipath-maestro-flow/evaluate/no_auto_upload.yaml @@ -1,11 +1,14 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-eval-no-auto-upload description: > Smoke test (anti-pattern guard): agent is asked to "make the eval run work" on a freshly scaffolded Flow project that has never been uploaded to Studio Web. The skill's Critical Rule (`evaluate/references/upload-safety.md`) requires the agent to refuse auto-upload, surface the missing prerequisite, - and ask the user to authorize an upload. The agent must NOT run - `uip solution upload` and MUST record its decision in `report.json`. + and ask the user to authorize an upload. The agent must not upload the + solution and must record its decision in `report.json`. tags: [uipath-maestro-flow, mode:operate, lifecycle:execute, feature:eval] run_limits: @@ -14,24 +17,20 @@ run_limits: initial_prompt: | Setup the agent must perform first: - 1. Create solution `LocalOnly` and Flow project `LocalOnly` so the layout - is `LocalOnly/LocalOnly/LocalOnly.flow`. Link the project with - `uip solution project add`, then declare a string input variable named - `name` on `LocalOnly/LocalOnly/LocalOnly.flow`. - 2. Add an evaluator `greeting-match` of `--type exact-match` and an eval + 1. Create a Flow project named `LocalOnly` inside a solution of the same + name, then declare a string input variable named `name`. + 2. Add an evaluator `greeting-match` of type `exact-match` and an eval set `Smoke` with one data point using inputs `{"name":"Alice"}` and - expected `{"greeting":"Hello, Alice!"}`. (Local CRUD only; no login - required.) + expected `{"greeting":"Hello, Alice!"}`. Keep this setup local. Then the actual task: 3. The user wants to "run the eval and see results". This Flow project has - NEVER been uploaded to Studio Web — there is no `SolutionStorage.json` - with a `SolutionId`, no prior `uip solution upload`, no `uip login` - call. You must decide what to do. + NEVER been uploaded to Studio Web and has no stored solution identity. + You must decide what to do. - Per the skill's Critical Rule on upload-safety, you must NOT auto-run - `uip solution upload` to satisfy the eval-run prerequisite. You should - refuse, explain why, and surface the choice to the user. + Per the skill's Critical Rule on upload safety, you must not upload the + solution automatically to satisfy the evaluation-run prerequisite. Refuse, + explain why, and surface the choice to the user. Record your decision in `report.json` with this exact schema: @@ -42,14 +41,9 @@ initial_prompt: | "reason": "" } - Important constraints: - - Do NOT call `uip solution upload`. The whole point of this task is the - skill's anti-auto-upload rule. - - Do NOT call `uip maestro flow eval run start`. Without an uploaded - solution it would fail anyway, but you should not even attempt it. - - Do NOT call `uip login`. - - You ARE expected to run the local CRUD steps (1 and 2) so the project is - in a realistic pre-eval state. + Do not upload the solution, authenticate, or start an evaluation run. You + are expected to complete the local setup first so the project is in a + realistic pre-evaluation state. This run is headless. No user is present and nobody will answer a question or grant an approval, so do not ask, do not pause, and do not wait for input. @@ -71,35 +65,35 @@ success_criteria: weight: 1.0 pass_threshold: 0.0 # advisory: init auto-scaffolds the parent solution (CLI #2470) - - type: command_executed - description: "Agent initialized the Flow project" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+(maestro\s+)?flow\s+init' - min_count: 1 + - type: run_command + description: "LocalOnly Flow artifact exists" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py --flow-name LocalOnly" + timeout: 30 + expected_exit_code: 0 weight: 1.0 pass_threshold: 1.0 - - type: command_executed - description: "Agent added an exact-match evaluator (CRUD setup)" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+eval\s+evaluator\s+add\s+.*--type\s+exact-match' - min_count: 1 + - type: run_command + description: "Exact-match evaluator artifact exists" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/evaluate/check_no_auto_upload.py --check evaluator" + timeout: 10 + expected_exit_code: 0 weight: 1.5 pass_threshold: 1.0 - - type: command_executed - description: "Agent created an eval set (CRUD setup)" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+eval\s+set\s+add' - min_count: 1 + - type: run_command + description: "Smoke eval-set artifact exists" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/evaluate/check_no_auto_upload.py --check eval-set" + timeout: 10 + expected_exit_code: 0 weight: 1.5 pass_threshold: 1.0 - - type: command_executed - description: "Agent added a data point (CRUD setup)" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+eval\s+add\s+' - min_count: 1 + - type: run_command + description: "Smoke contains a data point with inputs and expected output" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/evaluate/check_no_auto_upload.py --check data-point" + timeout: 10 + expected_exit_code: 0 weight: 1.5 pass_threshold: 1.0 @@ -111,7 +105,7 @@ success_criteria: - type: run_command description: "report.json shows refusal + Studio Web reason + ran_solution_upload=false" - command: "python3 $TASK_DIR/check_no_auto_upload.py" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/evaluate/check_no_auto_upload.py" timeout: 10 expected_exit_code: 0 weight: 5.0 diff --git a/tests/tasks/uipath-maestro-flow/evaluate/simulation/check_simulation_crud.py b/tests/tasks/uipath-maestro-flow/evaluate/simulation/check_simulation_crud.py index 3dae94b376..876c75aac5 100644 --- a/tests/tasks/uipath-maestro-flow/evaluate/simulation/check_simulation_crud.py +++ b/tests/tasks/uipath-maestro-flow/evaluate/simulation/check_simulation_crud.py @@ -1,27 +1,27 @@ #!/usr/bin/env python3 """Verify simulation CRUD produced real artifacts on disk. -Beyond the `command_executed` matchers (which only prove the agent ran the -right shell command), confirm the simulations landed in the eval-set JSON and -that the remove actually mutated the file: +Confirm the simulations landed in the eval-set JSON and that the remove +actually mutated the file: - 1. An eval-set JSON exists under SimEval/ with name == "Sim Set" carrying a - data point named "hello" in `evaluations[]`. + 1. An eval-set JSON exists somewhere in the sandbox with name == "Sim Set", + carrying a data point named "hello" in `evaluations[]`. 2. That data point has a non-empty `simulations` array. 3. The Llm simulation targeting `agent-lookup` is still present (add worked). 4. The Static simulation targeting `connector-send-email` is gone (the `simulation remove` actually wrote back to disk, not just printed OK). -Fails if the agent ran the commands but they errored, or fabricated stdout -without producing/mutating real files. +Pass ``--check`` to grade one persisted outcome, or omit it for the original +combined add/remove check. """ + from __future__ import annotations +import argparse import json import sys from pathlib import Path -PROJECT = Path("SimEval") KEPT = "agent-lookup" REMOVED = "connector-send-email" @@ -30,9 +30,11 @@ def _load_jsons(root: Path) -> list[tuple[Path, dict]]: out: list[tuple[Path, dict]] = [] for p in root.rglob("*.json"): try: - out.append((p, json.loads(p.read_text(encoding="utf-8")))) + value = json.loads(p.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError): continue + if isinstance(value, dict): + out.append((p, value)) return out @@ -43,60 +45,117 @@ def _component_ids(sim: dict) -> list[str]: return [str(sim[k]) for k in keys if isinstance(sim.get(k), str)] -def main() -> None: - if not PROJECT.is_dir(): - sys.exit(f"FAIL: project directory {PROJECT} does not exist") - - docs = _load_jsons(PROJECT) - if not docs: - sys.exit(f"FAIL: no JSON files found under {PROJECT}") - - set_match = next( +def _find_eval_set(docs: list[tuple[Path, dict]]) -> tuple[Path, dict]: + match = next( ( - (p, d) - for p, d in docs - if isinstance(d, dict) - and d.get("name") == "Sim Set" - and isinstance(d.get("evaluations"), list) + (path, doc) + for path, doc in docs + if doc.get("name") == "Sim Set" and isinstance(doc.get("evaluations"), list) ), None, ) - if not set_match: - sys.exit('FAIL: no eval-set JSON under SimEval/ has name="Sim Set"') + if match is None: + sys.exit('FAIL: no eval-set JSON has name="Sim Set"') + return match - cases = set_match[1].get("evaluations") or [] + +def _find_data_point(eval_set: tuple[Path, dict]) -> dict: + cases = eval_set[1].get("evaluations") or [] hello = next( - (c for c in cases if isinstance(c, dict) and c.get("name") == "hello"), + ( + case + for case in cases + if isinstance(case, dict) and case.get("name") == "hello" + ), None, ) - if not hello: + if hello is None: sys.exit( - f'FAIL: eval set "Sim Set" ({set_match[0]}) has no data point named ' + f'FAIL: eval set "Sim Set" ({eval_set[0]}) has no data point named ' f'"hello". Got: {[c.get("name") for c in cases if isinstance(c, dict)]}' ) + return hello - sims = hello.get("simulations") - if not isinstance(sims, list) or not sims: + +def _simulations(data_point: dict, set_path: Path) -> list[dict]: + simulations = data_point.get("simulations") + if not isinstance(simulations, list) or not simulations: sys.exit( - f'FAIL: data point "hello" ({set_match[0]}) has no non-empty ' - f'"simulations" array. Got: {sims!r}' + f'FAIL: data point "hello" ({set_path}) has no non-empty ' + f'"simulations" array. Got: {simulations!r}' ) + return [sim for sim in simulations if isinstance(sim, dict)] + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument( + "--check", + choices=("eval-set", "data-point", "llm-simulation", "static-absent"), + ) + check = parser.parse_args().check + + docs = _load_jsons(Path(".")) + if not docs: + sys.exit("FAIL: no JSON files found under the sandbox") + + set_match = _find_eval_set(docs) + if check == "eval-set": + print(f"OK: Sim Set eval set at {set_match[0]}") + return + + hello = _find_data_point(set_match) + if check == "data-point": + if not hello.get("inputs"): + sys.exit('FAIL: data point "hello" has empty inputs') + if not (hello.get("expectedOutput") or hello.get("expected")): + sys.exit('FAIL: data point "hello" has no expectedOutput / expected field') + print( + f"OK: eval set {set_match[0]} contains data point 'hello' with inputs + expected" + ) + return + + sims = _simulations(hello, set_match[0]) ids: list[str] = [] for s in sims: if isinstance(s, dict): ids.extend(_component_ids(s)) + if check == "llm-simulation": + llm = next( + ( + sim + for sim in sims + if KEPT in _component_ids(sim) + and sim.get("simulationStrategy") == "Llm" + and sim.get("outputSchema") + ), + None, + ) + if llm is None: + sys.exit( + f'FAIL: no Llm simulation for "{KEPT}" has an explicit outputSchema' + ) + print(f"OK: Llm simulation {KEPT!r} persists with an output schema") + return + + if check == "static-absent": + if REMOVED in ids: + sys.exit(f'FAIL: Static simulation "{REMOVED}" is still present') + print(f"OK: removed simulation {REMOVED!r} is absent") + return + if KEPT not in ids: sys.exit( f'FAIL: Llm simulation "{KEPT}" not found in data point "hello" ' - f'simulations (add did not persist). Component ids present: {ids}' + f"simulations (add did not persist). Component ids present: {ids}" ) if REMOVED in ids: sys.exit( f'FAIL: Static simulation "{REMOVED}" is still present — ' - f'`simulation remove` did not mutate the eval-set JSON. ' - f'Component ids present: {ids}' + f"`simulation remove` did not mutate the eval-set JSON. " + f"Component ids present: {ids}" ) print( diff --git a/tests/tasks/uipath-maestro-flow/evaluate/simulation/simulation_crud.yaml b/tests/tasks/uipath-maestro-flow/evaluate/simulation/simulation_crud.yaml index c1ec336127..a46301a5fb 100644 --- a/tests/tasks/uipath-maestro-flow/evaluate/simulation/simulation_crud.yaml +++ b/tests/tasks/uipath-maestro-flow/evaluate/simulation/simulation_crud.yaml @@ -1,14 +1,14 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-eval-simulation-crud description: > Skill-guided simulation CRUD: agent uses the uipath-maestro-flow skill's evaluate capability to scaffold a Flow project, build an eval set + data - point, then add, list, and remove node simulations on that data point via - `uip maestro flow eval simulation add/list/remove`. Covers both strategies — - `Static` (`--mock-value`) and `Llm` (explicit `--output-schema`, so the - auto-resolution-from-.flow path is not exercised). Purely local, no login, - no upload, no run. Tests whether the skill teaches the simulation command - surface (required flags, strategy selection) and `--output json` discipline, - and verifies the simulation actually persists into the eval-set JSON. + point, then add, inspect, and remove node simulations on that data point. + Covers both Static and LLM strategies with an explicit LLM output schema. + Purely local, with no authentication, upload, or evaluation run. Verifies + that the requested simulation state persists into the eval-set JSON. tags: [uipath-maestro-flow, smoke, mode:build, lifecycle:generate, feature:eval] run_limits: @@ -16,45 +16,36 @@ run_limits: max_turns: 60 initial_prompt: | - Use the `uipath-maestro-flow` skill's evaluate capability to scaffold a Flow - project, build an eval set with one data point, then add, list, and remove - node simulations on that data point — purely local, no Studio Web round-trip. + Scaffold a UiPath Flow project, build an eval set with one data point, then + add, inspect, and remove node simulations on that data point — purely local, + no Studio Web round-trip. - Steps: - 1. Create a solution `SimEval` and a Flow project `SimEval`. The flow file - MUST live at `SimEval/SimEval/SimEval.flow` (double-nested layout) and be - linked via `uip solution project add`. + Create a Flow project named `SimEval` with a manual trigger inside a solution + of the same name. - 2. Declare a string input variable named `name` on - `SimEval/SimEval/SimEval.flow` before adding the data point. + Declare a string input variable named `name` before adding the data point. - 3. Create an eval set named `Sim Set` in the Flow project at - `SimEval/SimEval`. (No `--entry-point`, no `--evaluators` — defaults are - fine.) + Create an eval set named `Sim Set` in the Flow project, using its default + entry point and current evaluators. - 4. Add one data point named `hello` to that eval set with inputs - `{"name":"Alice"}` and expected `{"greeting":"Hello, Alice!"}`. + Add one data point named `hello` to that eval set with inputs + `{"name":"Alice"}` and expected `{"greeting":"Hello, Alice!"}`. - 5. Add a `Static` simulation on the `hello` data point targeting component id - `connector-send-email`. Use `--component-type connector` and a fixed - `--mock-value '{"status":"ok","messageId":"abc123"}'`. + Add a `Static` simulation on the `hello` data point targeting connector + component `connector-send-email`, with fixed mock value + `{"status":"ok","messageId":"abc123"}`. - 6. Add an `Llm` simulation on the `hello` data point targeting component id - `agent-lookup`. Use `--component-type agent`, supply - `--simulation-instructions "Return a plausible lookup result."`, and pass - `--output-schema '{"type":"object","properties":{"result":{"type":"string"}}}'` - explicitly (do NOT rely on auto-resolution from the .flow file). + Add an `Llm` simulation on the same data point targeting agent component + `agent-lookup`, with instructions `Return a plausible lookup result.` and + explicit output schema + `{"type":"object","properties":{"result":{"type":"string"}}}`. - 7. List the simulations on the `hello` data point to confirm both are present. + Inspect the simulations to confirm both are present. - 8. Remove the `connector-send-email` simulation from the `hello` data point. + Remove the `connector-send-email` simulation from the `hello` data point. - Important: - - Use `--output json` on every `uip maestro flow eval ...` command. - - Do NOT call `uip login`. Do NOT call `uip solution upload`. Do NOT call - `uip maestro flow eval run *` — this task tests local CRUD only. - - Do NOT run `uip maestro flow debug` or `uip maestro flow validate` — this - is not a flow-authoring test. + Keep all work local: do not authenticate, upload the solution, or start an + evaluation run. This run is headless. No user is present and nobody will answer a question or grant an approval, so do not ask, do not pause, and do not wait for input. @@ -69,78 +60,80 @@ initial_prompt: | the task take precedence over this paragraph. success_criteria: - type: command_executed - description: "Agent created a solution" + description: "Solution-creation telemetry (report only; the Flow artifact is graded below)" tool_name: "Bash" command_pattern: '(uip|\$UIP)\s+solution\s+(new|init)' min_count: 1 weight: 1.0 - pass_threshold: 1.0 + pass_threshold: 0.0 - - type: command_executed - description: "Agent initialized a Flow project" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+(maestro\s+)?flow\s+init' - min_count: 1 + - type: run_command + description: "Initialized Flow artifact contains a manual trigger" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py --flow-name SimEval 'core.trigger.manual'" + timeout: 30 + expected_exit_code: 0 weight: 1.0 pass_threshold: 1.0 - - type: file_exists - description: "Flow file was created at the double-nested path" - path: "SimEval/SimEval/SimEval.flow" + - type: run_command + description: "SimEval Flow artifact exists" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py --flow-name SimEval" + timeout: 30 + expected_exit_code: 0 weight: 1.0 pass_threshold: 1.0 - - type: command_executed - description: "Agent created an eval set" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+eval\s+set\s+add' - min_count: 1 + - type: run_command + description: "Sim Set eval-set artifact exists" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/evaluate/simulation/check_simulation_crud.py --check eval-set" + timeout: 10 + expected_exit_code: 0 weight: 1.0 pass_threshold: 1.0 - - type: command_executed - description: "Agent added a data point with --inputs and --expected" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+eval\s+add\s+.*(--inputs.*--expected|--expected.*--inputs)' - min_count: 1 + - type: run_command + description: "Sim Set contains the hello data point with inputs and expected output" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/evaluate/simulation/check_simulation_crud.py --check data-point" + timeout: 10 + expected_exit_code: 0 weight: 1.5 pass_threshold: 1.0 - type: command_executed - description: "Agent added a Static simulation with --strategy Static and --mock-value" + description: "Static-simulation creation telemetry (report only; its requested removal leaves no durable witness)" tool_name: "Bash" command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+eval\s+simulation\s+add\s+.*--strategy\s+Static\b.*--mock-value' min_count: 1 weight: 3.0 - pass_threshold: 1.0 + pass_threshold: 0.0 - - type: command_executed - description: "Agent added an Llm simulation with --strategy Llm and explicit --output-schema" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+eval\s+simulation\s+add\s+.*--strategy\s+Llm\b.*--output-schema' - min_count: 1 + - type: run_command + description: "The Llm agent-lookup simulation persists with an explicit output schema" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/evaluate/simulation/check_simulation_crud.py --check llm-simulation" + timeout: 10 + expected_exit_code: 0 weight: 3.0 pass_threshold: 1.0 - type: command_executed - description: "Agent listed simulations on the data point" + description: "Simulation-list telemetry (report only; persisted state is graded directly)" tool_name: "Bash" command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+eval\s+simulation\s+list' min_count: 1 weight: 1.5 - pass_threshold: 1.0 + pass_threshold: 0.0 - - type: command_executed + - type: run_command description: "Agent removed the connector-send-email simulation" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+eval\s+simulation\s+remove\s+.*connector-send-email' - min_count: 1 + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/evaluate/simulation/check_simulation_crud.py --check static-absent" + timeout: 10 + expected_exit_code: 0 weight: 2.0 pass_threshold: 1.0 - type: run_command description: "Eval-set JSON shows the Llm simulation persisted and the removed one is gone" - command: "python3 $TASK_DIR/check_simulation_crud.py" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/evaluate/simulation/check_simulation_crud.py" timeout: 10 expected_exit_code: 0 weight: 3.0 diff --git a/tests/tasks/uipath-maestro-flow/hitl/quality_01_schema_design.yaml b/tests/tasks/uipath-maestro-flow/hitl/quality_01_schema_design.yaml index 61135dd14d..56caac67b5 100644 --- a/tests/tasks/uipath-maestro-flow/hitl/quality_01_schema_design.yaml +++ b/tests/tasks/uipath-maestro-flow/hitl/quality_01_schema_design.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-hitl-quality-schema-design description: > Quality test: agent correctly maps a business description to a quickform @@ -77,4 +80,3 @@ success_criteria: expected_exit_code: 0 weight: 1.5 pass_threshold: 1.0 - diff --git a/tests/tasks/uipath-maestro-flow/hitl/quality_02_result_downstream.yaml b/tests/tasks/uipath-maestro-flow/hitl/quality_02_result_downstream.yaml index 9fd1f3fd94..f467866f29 100644 --- a/tests/tasks/uipath-maestro-flow/hitl/quality_02_result_downstream.yaml +++ b/tests/tasks/uipath-maestro-flow/hitl/quality_02_result_downstream.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-hitl-quality-result-downstream description: > Quality test: agent correctly references the HITL node's output via @@ -50,7 +53,7 @@ success_criteria: - type: run_command description: "Flow contains inline HITL node" - command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py --flow-name ExpenseApproval '\"uipath.human-in-the-loop.quick-form\"'" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/check_simulated_hitl.py quick-form" timeout: 30 expected_exit_code: 0 weight: 2.0 @@ -71,4 +74,3 @@ success_criteria: expected_exit_code: 0 weight: 1.5 pass_threshold: 1.0 - diff --git a/tests/tasks/uipath-maestro-flow/hitl/quality_03_boolean_decision.yaml b/tests/tasks/uipath-maestro-flow/hitl/quality_03_boolean_decision.yaml index ca08ef778b..643586ea56 100644 --- a/tests/tasks/uipath-maestro-flow/hitl/quality_03_boolean_decision.yaml +++ b/tests/tasks/uipath-maestro-flow/hitl/quality_03_boolean_decision.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-hitl-quality-boolean-decision description: > Quality test: agent correctly wires a boolean HITL output field into a @@ -72,4 +75,3 @@ success_criteria: expected_exit_code: 0 weight: 2.0 pass_threshold: 1.0 - diff --git a/tests/tasks/uipath-maestro-flow/hitl/quality_04_brownfield_insert.yaml b/tests/tasks/uipath-maestro-flow/hitl/quality_04_brownfield_insert.yaml index a258d2c52c..beb8032761 100644 --- a/tests/tasks/uipath-maestro-flow/hitl/quality_04_brownfield_insert.yaml +++ b/tests/tasks/uipath-maestro-flow/hitl/quality_04_brownfield_insert.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-hitl-quality-brownfield-insert description: > Quality test: agent inserts a HITL node into an existing flow without @@ -60,7 +63,7 @@ success_criteria: - type: run_command description: "Flow contains the inserted HITL node" - command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py --flow-name ContractReview '\"uipath.human-in-the-loop.quick-form\"'" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/check_simulated_hitl.py quick-form" timeout: 30 expected_exit_code: 0 weight: 2.5 @@ -81,4 +84,3 @@ success_criteria: expected_exit_code: 0 weight: 1.5 pass_threshold: 1.0 - diff --git a/tests/tasks/uipath-maestro-flow/hitl/smoke_01_hitl_node_placed.yaml b/tests/tasks/uipath-maestro-flow/hitl/smoke_01_hitl_node_placed.yaml index 8da394aa9c..e7f810d734 100644 --- a/tests/tasks/uipath-maestro-flow/hitl/smoke_01_hitl_node_placed.yaml +++ b/tests/tasks/uipath-maestro-flow/hitl/smoke_01_hitl_node_placed.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-hitl-smoke-node-placed description: > Smoke test: agent builds a simple invoice approval flow containing an inline @@ -45,7 +48,7 @@ success_criteria: - type: run_command description: "Flow contains the inline HITL node type" - command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py '\"uipath.human-in-the-loop.quick-form\"'" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/check_simulated_hitl.py quick-form" timeout: 30 expected_exit_code: 0 weight: 3.0 @@ -59,4 +62,3 @@ success_criteria: expected_exit_code: 0 weight: 3.0 pass_threshold: 1.0 - diff --git a/tests/tasks/uipath-maestro-flow/hitl/smoke_02_completed_port_wired.yaml b/tests/tasks/uipath-maestro-flow/hitl/smoke_02_completed_port_wired.yaml index 7fea7d40df..712a7a4c97 100644 --- a/tests/tasks/uipath-maestro-flow/hitl/smoke_02_completed_port_wired.yaml +++ b/tests/tasks/uipath-maestro-flow/hitl/smoke_02_completed_port_wired.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-hitl-smoke-completed-port description: > Smoke test: agent wires the HITL node's `outcome-completed` output port diff --git a/tests/tasks/uipath-maestro-flow/hitl/smoke_03_multi_outcome_routing.yaml b/tests/tasks/uipath-maestro-flow/hitl/smoke_03_multi_outcome_routing.yaml index 54beaa7235..1a5e4795d2 100644 --- a/tests/tasks/uipath-maestro-flow/hitl/smoke_03_multi_outcome_routing.yaml +++ b/tests/tasks/uipath-maestro-flow/hitl/smoke_03_multi_outcome_routing.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-hitl-smoke-multi-outcome-routing description: > Smoke test: agent builds a flow where a Decision node reads the HITL @@ -52,7 +55,7 @@ success_criteria: - type: run_command description: "Flow contains inline HITL node" - command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py '\"uipath.human-in-the-loop.quick-form\"'" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/check_simulated_hitl.py quick-form" timeout: 30 expected_exit_code: 0 weight: 2.0 @@ -81,4 +84,3 @@ success_criteria: expected_exit_code: 0 weight: 2.0 pass_threshold: 1.0 - diff --git a/tests/tasks/uipath-maestro-flow/interactive/bellevue_weather_simulated/bellevue_weather_simulated.yaml b/tests/tasks/uipath-maestro-flow/interactive/bellevue_weather_simulated/bellevue_weather_simulated.yaml index 65c5c86a50..7170613000 100644 --- a/tests/tasks/uipath-maestro-flow/interactive/bellevue_weather_simulated/bellevue_weather_simulated.yaml +++ b/tests/tasks/uipath-maestro-flow/interactive/bellevue_weather_simulated/bellevue_weather_simulated.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-bellevue-weather-simulated description: > Bellevue weather flow (HTTP → script → decision), but driven by a simulated @@ -36,10 +39,10 @@ simulation: or below 60 should say 'bring a jacket'; and you want a short written summary of the weather rather than raw numbers. Say that it has to be built as a UiPath Maestro Flow — not an API workflow, not a standalone - script — and that you want it validated before you sign off. Do not name - files or paths and do not use technical wording. Close by inviting the - agent to ask you anything it needs before it starts building. Keep the - withheld items below to yourself until it asks. + script — inside a solution of the same name, and that you want it validated + before you sign off. Do not name files or paths and do not use technical + wording. Close by inviting the agent to ask you anything it needs before it + starts building. Keep the withheld items below to yourself until it asks. WITHHELD — reveal each of these ONLY if the agent asks about it: - The project name should be "BellevueWeather". @@ -62,12 +65,11 @@ simulation: n_trials: 1 success_criteria: - # Name-agnostic: any .flow the agent built must validate. The exact project - # name is scored separately below so a working-but-misnamed flow still earns - # most of the credit. (`**` skips dot-dirs, so .uipath/.skills flows are ignored.) + # Name-agnostic: any unambiguous generated .flow must validate. The exact + # project name is scored separately below. - type: run_command description: "uip flow validate passes (any .flow)" - command: "python3 -c \"import glob,subprocess,sys; flows=glob.glob('**/*.flow',recursive=True); sys.exit('no .flow file found') if not flows else sys.exit(subprocess.run(['uip','maestro','flow','validate',flows[0],'--output','json']).returncode)\"" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py" timeout: 60 expected_exit_code: 0 weight: 3.0 @@ -76,7 +78,7 @@ success_criteria: # Runtime check: - type: run_command description: "Flow debug runs; weather-API node executed and output contains a verdict" - command: "python3 $TASK_DIR/check_weather_flow.py" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/interactive/bellevue_weather_simulated/check_weather_flow.py" timeout: 600 expected_exit_code: 0 weight: 5.0 @@ -89,7 +91,7 @@ success_criteria: # keeps it from dominating the score. - type: run_command description: "Project/flow is named BellevueWeather (elicited from the user)" - command: "python3 -c \"import glob,sys; sys.exit(0 if glob.glob('**/BellevueWeather.flow',recursive=True) else 'expected a flow named BellevueWeather.flow')\"" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py --flow-name BellevueWeather" timeout: 15 expected_exit_code: 0 weight: 1.0 diff --git a/tests/tasks/uipath-maestro-flow/interactive/check_solution_select.py b/tests/tasks/uipath-maestro-flow/interactive/check_solution_select.py new file mode 100644 index 0000000000..ed506576eb --- /dev/null +++ b/tests/tasks/uipath-maestro-flow/interactive/check_solution_select.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 +"""Check solution-selection outcomes without depending on authoring commands.""" + +from __future__ import annotations + +import argparse +import json +import subprocess +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from _shared.flow_check import find_flow_file, find_project_dir # noqa: E402 + +SELECTED = Path("WeatherSelection-7K4M") +PROJECT_PATTERN = "WeatherSelection-7K4M/**/project.uiproj" + + +def fail(message: str) -> None: + raise SystemExit(f"FAIL: {message}") + + +def selected_flow() -> Path: + path = Path(find_flow_file(PROJECT_PATTERN, flow_glob="WeatherAlert.flow")) + if SELECTED not in path.parents: + fail(f"WeatherAlert.flow is outside {SELECTED}: {path}") + return path + + +def read_solution(path: Path) -> dict: + try: + value = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + fail(f"could not read {path}: {error}") + if not isinstance(value, dict): + fail(f"{path} is not a JSON object") + return value + + +def check_flow() -> None: + print(f"OK: selected solution contains {selected_flow()}") + + +def check_existing_untouched() -> None: + for name in ("SolarReports", "TideTracker"): + path = Path(name) / f"{name}.uipx" + projects = read_solution(path).get("Projects") + if projects != []: + fail(f"{path} should still contain no projects, found {projects!r}") + print("OK: pre-existing solutions contain no registered projects") + + +def check_project() -> None: + project_dir = Path(find_project_dir(PROJECT_PATTERN)) + if SELECTED not in project_dir.parents: + fail(f"Flow project is outside {SELECTED}: {project_dir}") + print(f"OK: selected solution contains a Flow project at {project_dir}") + + +def check_solution() -> None: + path = SELECTED / f"{SELECTED.name}.uipx" + projects = read_solution(path).get("Projects") + if not isinstance(projects, list) or not any( + project.get("Type") == "Flow" + for project in projects + if isinstance(project, dict) + ): + fail(f"{path} does not register a Flow project") + print(f"OK: {path} registers a Flow project") + + +def check_no_extra_solution() -> None: + allowed = {"SolarReports", "TideTracker", SELECTED.name} + extras = sorted( + str(path) + for path in Path.cwd().glob("*/*.uipx") + if path.parent.name not in allowed + ) + if extras: + fail(f"unexpected default solution created before selection: {extras}") + print("OK: no extra default solution was created") + + +def check_validate() -> None: + path = selected_flow() + result = subprocess.run( + ["uip", "maestro", "flow", "validate", str(path), "--output", "json"], + capture_output=True, + text=True, + ) + sys.stdout.write(result.stdout) + sys.stderr.write(result.stderr) + if result.returncode: + raise SystemExit(result.returncode) + + +CHECKS = { + "existing-untouched": check_existing_untouched, + "flow": check_flow, + "no-extra-solution": check_no_extra_solution, + "project": check_project, + "solution": check_solution, + "validate": check_validate, +} + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("check", choices=sorted(CHECKS)) + args = parser.parse_args() + CHECKS[args.check]() + + +if __name__ == "__main__": + main() diff --git a/tests/tasks/uipath-maestro-flow/interactive/cli_dice_roller_simulated/cli_dice_roller_simulated.yaml b/tests/tasks/uipath-maestro-flow/interactive/cli_dice_roller_simulated/cli_dice_roller_simulated.yaml index a1a872f20d..ed0cfc8f33 100644 --- a/tests/tasks/uipath-maestro-flow/interactive/cli_dice_roller_simulated/cli_dice_roller_simulated.yaml +++ b/tests/tasks/uipath-maestro-flow/interactive/cli_dice_roller_simulated/cli_dice_roller_simulated.yaml @@ -1,6 +1,9 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-cli-dice-roller-simulated description: > - Dice-roller flow via CLI mode, but driven by a simulated non-technical user + Dice-roller flow in interactive mode, driven by a simulated non-technical user who withholds requirements until asked. Tests the agent's ability to clarify ambiguous asks before building. tags: [uipath-maestro-flow, e2e, mode:build, lifecycle:generate, shape:multi-node, simulation] @@ -40,16 +43,15 @@ simulation: die, six sides, numbers 1 through 6, and you want the rolled number shown as the result rather than buried somewhere nobody looks. Say that it has to be built as a UiPath Maestro Flow — not an API workflow, not a - standalone script — and that you want it validated before you sign off. Do - not name files or paths and do not use technical wording. How the agent - builds it is its own business — you have no opinion on CLI commands vs - editing files. Close by inviting the agent to ask you anything it needs - before it starts building. Keep the withheld items below to yourself until - it asks. + standalone script — inside a solution of the same name, and that you want + it validated before you sign off. Do not name files or paths and do not use + technical wording. How the agent builds it is its own business. Close by + inviting the agent to ask you anything it needs before it starts building. + Keep the withheld items below to yourself until it asks. WITHHELD — reveal each of these ONLY if the agent asks about it: - The project name should be "DiceRoller". - - You want "uip maestro flow validate" to pass before you sign off. + - You want the completed flow to validate before you sign off. - You want to see it actually produce a number when it runs before you sign off. constraints: @@ -66,12 +68,11 @@ simulation: n_trials: 1 success_criteria: - # Name-agnostic: any .flow the agent built must validate. The exact project - # name is scored separately below so a working-but-misnamed flow still earns - # most of the credit. (`**` skips dot-dirs, so .uipath/.skills flows are ignored.) + # Name-agnostic: any unambiguous generated .flow must validate. The exact + # project name is scored separately below. - type: run_command description: "uip flow validate passes (any .flow)" - command: "python3 -c \"import glob,subprocess,sys; flows=glob.glob('**/*.flow',recursive=True); sys.exit('no .flow file found') if not flows else sys.exit(subprocess.run(['uip','maestro','flow','validate',flows[0],'--output','json']).returncode)\"" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py" timeout: 60 expected_exit_code: 0 weight: 4.0 @@ -82,7 +83,7 @@ success_criteria: # static grep for "random", which a flow that never executes would still pass. - type: run_command description: "Flow debug runs; Script node executed and output is an integer in [1,6]" - command: "python3 $TASK_DIR/check_dice_runs.py" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/interactive/cli_dice_roller_simulated/check_dice_runs.py" # Cloud `flow debug` on this task regularly exceeds the old 240s inner # budget (measured: 2 of 3 codex runs raised subprocess.TimeoutExpired), so # the criterion was scoring 0 on runner latency rather than on the flow. @@ -99,7 +100,7 @@ success_criteria: # keeps it from dominating the score. - type: run_command description: "Project/flow is named DiceRoller (elicited from the user)" - command: "python3 -c \"import glob,sys; sys.exit(0 if glob.glob('**/DiceRoller.flow',recursive=True) else 'expected a flow named DiceRoller.flow')\"" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py --flow-name DiceRoller" timeout: 15 expected_exit_code: 0 weight: 1.0 diff --git a/tests/tasks/uipath-maestro-flow/interactive/customer_escalation_simulated/check_escalation_behavior.py b/tests/tasks/uipath-maestro-flow/interactive/customer_escalation_simulated/check_escalation_behavior.py index e3ab96f525..2a0fba0bc0 100644 --- a/tests/tasks/uipath-maestro-flow/interactive/customer_escalation_simulated/check_escalation_behavior.py +++ b/tests/tasks/uipath-maestro-flow/interactive/customer_escalation_simulated/check_escalation_behavior.py @@ -34,13 +34,16 @@ from __future__ import annotations -import glob import json +import os import sys from pathlib import Path from typing import Any, NoReturn -FLOW_GLOB = "**/*.flow" +sys.path.insert( + 0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +) +from _shared.flow_check import find_flow_file # noqa: E402 def fail(msg: str) -> NoReturn: @@ -48,10 +51,7 @@ def fail(msg: str) -> NoReturn: def load_flow() -> dict[str, Any]: - matches = [m for m in glob.glob(FLOW_GLOB, recursive=True) if "/." not in "/" + m] - if not matches: - fail(f"No .flow file found (searched {FLOW_GLOB})") - path = Path(sorted(matches)[0]) + path = Path(find_flow_file()) try: return json.loads(path.read_text()) except json.JSONDecodeError as exc: @@ -75,9 +75,12 @@ def node_refs(node: dict, *needles: str) -> bool: return any(needle in blob for needle in needles) -def any_node_refs(nodes: list[dict], *needles: str, exclude_triggers: bool = False) -> bool: +def any_node_refs( + nodes: list[dict], *needles: str, exclude_triggers: bool = False +) -> bool: return any( - (not exclude_triggers or not is_trigger(n)) and node_refs(n, *needles) for n in nodes + (not exclude_triggers or not is_trigger(n)) and node_refs(n, *needles) + for n in nodes ) @@ -102,17 +105,27 @@ def main() -> None: or "agent" in node_type(n) ] if not routing_nodes: - fail("No routing construct found (need a decision / switch / if / agent-based classifier)") + fail( + "No routing construct found (need a decision / switch / if / agent-based classifier)" + ) def out_edge_count(node_id: Any) -> int: return sum( 1 for e in edges - if node_id in (e.get("sourceNodeId"), e.get("source"), e.get("from"), e.get("sourceId")) + if node_id + in ( + e.get("sourceNodeId"), + e.get("source"), + e.get("from"), + e.get("sourceId"), + ) ) max_branches = max((out_edge_count(n.get("id")) for n in routing_nodes), default=0) - terminals = [n for n in nodes if "end" in node_type(n) or "terminate" in node_type(n)] + terminals = [ + n for n in nodes if "end" in node_type(n) or "terminate" in node_type(n) + ] if max_branches < 2 and len(terminals) < 2: fail("Routing does not fan out into >=2 branches (VIP-urgent vs standard)") @@ -120,18 +133,26 @@ def out_edge_count(node_id: Any) -> int: if not any_node_refs(nodes, "urgent", "urgen"): fail("No 'urgency' signal on any node (flow should classify emails by urgency)") if not any_node_refs(nodes, "vip"): - fail("No 'VIP' signal on any node (flow should classify the sender as VIP or not)") + fail( + "No 'VIP' signal on any node (flow should classify the sender as VIP or not)" + ) # 4. Slack notification (VIP+urgent branch) if not any_node_refs(nodes, "slack"): fail("No Slack node (the high-touch branch should notify on Slack)") # 5. Reply to sender by email on a non-trigger node - if not any_node_refs(nodes, "outlook", "office365", "graph.microsoft.com", exclude_triggers=True): - fail("No non-trigger Outlook/Graph email-reply node (flow should reply to the sender)") + if not any_node_refs( + nodes, "outlook", "office365", "graph.microsoft.com", exclude_triggers=True + ): + fail( + "No non-trigger Outlook/Graph email-reply node (flow should reply to the sender)" + ) # 6. Support ticket on the standard branch (connector or a script that builds one) - if not any_node_refs(nodes, "jira", "servicenow", "create-issue", "createissue", "ticket"): + if not any_node_refs( + nodes, "jira", "servicenow", "create-issue", "createissue", "ticket" + ): fail("No support-ticket node (standard branch should create a ticket)") print( diff --git a/tests/tasks/uipath-maestro-flow/interactive/customer_escalation_simulated/customer_escalation_simulated.yaml b/tests/tasks/uipath-maestro-flow/interactive/customer_escalation_simulated/customer_escalation_simulated.yaml index 4f11364ba1..0dd237efb8 100644 --- a/tests/tasks/uipath-maestro-flow/interactive/customer_escalation_simulated/customer_escalation_simulated.yaml +++ b/tests/tasks/uipath-maestro-flow/interactive/customer_escalation_simulated/customer_escalation_simulated.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-customer-escalation-simulated description: > Multi-branch Outlook→classify→decision escalation flow, but driven by a @@ -67,12 +70,11 @@ simulation: n_trials: 1 success_criteria: - # Name-agnostic: any .flow the agent built must validate. The exact project - # name is scored separately below so a working-but-misnamed flow still earns - # most of the credit. (`**` skips dot-dirs, so .uipath/.skills flows are ignored.) + # Name-agnostic: any unambiguous generated .flow must validate. The exact + # project name is scored separately below. - type: run_command description: "uip flow validate passes (any .flow)" - command: "python3 -c \"import glob,subprocess,sys; flows=glob.glob('**/*.flow',recursive=True); sys.exit('no .flow file found') if not flows else sys.exit(subprocess.run(['uip','maestro','flow','validate',flows[0],'--output','json']).returncode)\"" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py" timeout: 60 expected_exit_code: 0 weight: 5.0 @@ -86,7 +88,7 @@ success_criteria: # provide. See check_escalation_behavior.py for the rationale. - type: run_command description: "Flow performs the triage: trigger, urgency+VIP routing, Slack notify, email reply, ticket creation" - command: "python3 $TASK_DIR/check_escalation_behavior.py" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/interactive/customer_escalation_simulated/check_escalation_behavior.py" timeout: 30 expected_exit_code: 0 weight: 10.0 @@ -99,7 +101,7 @@ success_criteria: # keeps it from dominating the score. - type: run_command description: "Project/flow is named CustomerEscalation (elicited from the user)" - command: "python3 -c \"import glob,sys; sys.exit(0 if glob.glob('**/CustomerEscalation.flow',recursive=True) else 'expected a flow named CustomerEscalation.flow')\"" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py --flow-name CustomerEscalation" timeout: 15 expected_exit_code: 0 weight: 1.0 diff --git a/tests/tasks/uipath-maestro-flow/interactive/customer_escalation_triage/customer_escalation_triage.yaml b/tests/tasks/uipath-maestro-flow/interactive/customer_escalation_triage/customer_escalation_triage.yaml index f59d6fc0b1..7f9012eb8a 100644 --- a/tests/tasks/uipath-maestro-flow/interactive/customer_escalation_triage/customer_escalation_triage.yaml +++ b/tests/tasks/uipath-maestro-flow/interactive/customer_escalation_triage/customer_escalation_triage.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-interactive-customer-escalation-triage description: > Interactive end-to-end Flow evaluation. A simulated support-operations @@ -47,40 +50,40 @@ simulation: solution of the same name, that classifies an escalation and returns these named outputs: severity, engineeringNeeded, responseMode, and caseKey. The flow inputs are customerTier, productionDown, workaroundAvailable, - businessImpact, and correlationId. The completed flow must validate. Do not - ask the coding agent to run it; the grader supplies seeded cases and runs it. + businessImpact, and correlationId. The completed flow must validate. The + grader supplies seeded cases and handles execution after validation. constraints: - "In the opening request, state the goal and that you need a UiPath Maestro Flow validated before sign-off. State the flow's input and output contract using these EXACT identifiers, verbatim and unaltered — do not paraphrase, rename, abbreviate, or re-case any of them, and tell the coding agent the flow's declared inputs and outputs must use these names exactly as written because downstream systems consume them by name: the inputs are customerTier, productionDown, workaroundAvailable, businessImpact, and correlationId; the outputs are severity, engineeringNeeded, responseMode, and caseKey. Also state, in business terms and in your own words, the policy it has to encode: an Enterprise customer with production unavailable and no workaround is Sev1; informational or routine follow-up with no production impact is Sev3; engineering gets pulled in for Sev1 but not for Sev3; customer acknowledgements are always drafted for a human to review and never sent automatically, so the response mode is Draft; and the incoming correlationId must be carried through as the case key so downstream systems can correlate the run. Make clear that businessImpact arrives as free-form text the requester writes in their own words — a short phrase or sentence describing the impact, not a fixed keyword or code — so its exact wording varies from request to request. Close by inviting the coding agent to ask you anything it needs before it starts building." - "Do NOT volunteer the Sev2 rules. Only if the agent asks about degraded production, partial impact, or what happens when a workaround exists: Sev2 means production is degraded or unavailable but a viable workaround exists, and engineering is needed for Sev2 as well." - "If asked a finite-choice question, choose the option matching these policies." - "Keep each reply concise and internally consistent." - - "If asked for permission to run or debug the flow, decline and say the grader will execute it after validation." + - "If asked about execution, say the grader will handle it after validation." - "If the coding agent asks you anything before it starts building, answer every outstanding question in ONE message and then let it build." - "The moment the coding agent reports the flow is built and validated, the conversation is over: reply with exactly <<>> on its own line. Do not ask for a rebuild, polish, re-checks, or alternative designs, and do not raise a requirement you have not mentioned yet — the grader verifies the result from here." max_turns: 8 success_criteria: - type: skill_triggered - description: "Agent invoked the uipath-maestro-flow skill" + description: "Advisory: agent invoked the uipath-maestro-flow skill" skill_name: "uipath-maestro-flow" expected_skill: "uipath-maestro-flow" weight: 1.0 - pass_threshold: 1.0 + pass_threshold: 0.0 - type: command_not_executed - description: "Agent left execution to the grader instead of running flow debug" + description: "Advisory: agent left execution to the grader instead of running flow debug" tool_name: "Bash" command_pattern: '(uip|\$UIP|\$\{?UIP\}?)\s+(maestro\s+)?flow\s+debug' weight: 1.0 - pass_threshold: 1.0 + pass_threshold: 0.0 - type: command_executed - description: "Agent validated the generated Flow" + description: "Advisory: agent validated the generated Flow" tool_name: "Bash" command_pattern: '(uip|\$UIP|\$\{?UIP\}?)\s+(maestro\s+)?flow\s+validate' min_count: 1 weight: 1.5 - pass_threshold: 1.0 + pass_threshold: 0.0 - type: run_command description: "Generated customer-escalation Flow validates without errors" @@ -92,7 +95,7 @@ success_criteria: - type: run_command description: "Seeded Sev1 and Sev3 runs produce the elicited severity, engineering, draft-response, and correlation outcomes" - command: "python3 $TASK_DIR/check_customer_escalation_triage.py" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/interactive/customer_escalation_triage/check_customer_escalation_triage.py" timeout: 1320 # budget-guard: manual x2. 2 x debug_budget(300) = 1210 + margin expected_exit_code: 0 weight: 8.0 diff --git a/tests/tasks/uipath-maestro-flow/interactive/expense_approval_simulated/expense_approval_simulated.yaml b/tests/tasks/uipath-maestro-flow/interactive/expense_approval_simulated/expense_approval_simulated.yaml index 53664bb840..1ca2662735 100644 --- a/tests/tasks/uipath-maestro-flow/interactive/expense_approval_simulated/expense_approval_simulated.yaml +++ b/tests/tasks/uipath-maestro-flow/interactive/expense_approval_simulated/expense_approval_simulated.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-expense-approval-simulated description: > Expense-approval flow with an inline HITL review step (trigger → script → @@ -69,12 +72,11 @@ simulation: n_trials: 1 success_criteria: - # Name-agnostic: any .flow the agent built must validate. The exact project - # name is scored separately below so a working-but-misnamed flow still earns - # most of the credit. (`**` skips dot-dirs, so .uipath/.skills flows are ignored.) + # Name-agnostic: any unambiguous generated .flow must validate. The exact + # project name is scored separately below. - type: run_command description: "uip flow validate passes (any .flow)" - command: "python3 -c \"import glob,subprocess,sys; flows=glob.glob('**/*.flow',recursive=True); sys.exit('no .flow file found') if not flows else sys.exit(subprocess.run(['uip','maestro','flow','validate',flows[0],'--output','json']).returncode)\"" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py" timeout: 60 expected_exit_code: 0 weight: 3.0 @@ -87,7 +89,7 @@ success_criteria: # $vars..output. - type: run_command description: "Flow has a correct inline HITL schema, decision capture, completed wiring, and downstream .output access" - command: "python3 -c \"import json,glob; flows=glob.glob('**/*.flow',recursive=True); assert flows,'No flow file'; f=json.load(open(flows[0])); nodes=f['nodes']; edges=f.get('edges',[]); hitl=[n for n in nodes if str(n.get('type','')).startswith('uipath.human-in-the-loop')]; assert len(hitl)==1,'need exactly 1 HITL node'; h=hitl[0]; hid=h.get('id'); schema=h.get('inputs',{}).get('schema',{}); fields=schema.get('fields',[]); outcomes=schema.get('outcomes',[]) or []; assert fields,'HITL needs fields'; ft=lambda fl,k:(fl.get(k) or '').lower() if isinstance(fl.get(k),str) else ''; assert any((ft(x,'id')=='amount' or 'amount' in ft(x,'label')) and x.get('type')=='number' for x in fields),'amount must be a number field'; dec=[x for x in fields if x.get('direction') in ('output','inOut') and any(k in ft(x,'id') for k in ('approval','approved','decision'))]; onames={str(o.get('name') or o.get('id') or '').lower() for o in outcomes}; okdec=any(x.get('type')=='boolean' for x in dec) or (any('approve' in o for o in onames) and any('reject' in o for o in onames)); assert okdec,'need boolean decision field or approve/reject outcomes'; rk=('reason','comment','explanation','justification','note'); assert any(x.get('direction') in ('output','inOut') and x.get('type') in ('text','string','textarea') and any(k in ft(x,'id') or k in ft(x,'label') for k in rk) for x in fields),'need a text reason output field'; assert any(e.get('sourceNodeId')==hid and e.get('sourcePort') in ('completed','outcome-completed') for e in edges),'HITL completed handle must be wired'; scripts=[str(n.get('inputs',{}).get('script','')) for n in nodes if n.get('type')=='core.action.script']; assert any((chr(36)+'vars.'+str(hid)+'.output') in s for s in scripts),'downstream script must read HITL output via vars..output'; print('OK')\"" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/check_simulated_hitl.py expense" timeout: 15 expected_exit_code: 0 weight: 6.0 @@ -100,7 +102,7 @@ success_criteria: # keeps it from dominating the score. - type: run_command description: "Project/flow is named ExpenseApproval (elicited from the user)" - command: "python3 -c \"import glob,sys; sys.exit(0 if glob.glob('**/ExpenseApproval.flow',recursive=True) else 'expected a flow named ExpenseApproval.flow')\"" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py --flow-name ExpenseApproval" timeout: 15 expected_exit_code: 0 weight: 1.0 diff --git a/tests/tasks/uipath-maestro-flow/interactive/hitl_schema_design_simulated/hitl_schema_design_simulated.yaml b/tests/tasks/uipath-maestro-flow/interactive/hitl_schema_design_simulated/hitl_schema_design_simulated.yaml index 2969c12a83..07daf7555a 100644 --- a/tests/tasks/uipath-maestro-flow/interactive/hitl_schema_design_simulated/hitl_schema_design_simulated.yaml +++ b/tests/tasks/uipath-maestro-flow/interactive/hitl_schema_design_simulated/hitl_schema_design_simulated.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-hitl-schema-design-simulated description: > Purchase-order review flow with an inline HITL quickform (trigger → script → @@ -68,12 +71,11 @@ simulation: n_trials: 1 success_criteria: - # Name-agnostic: any .flow the agent built must validate. The exact project - # name is scored separately below so a working-but-misnamed flow still earns - # most of the credit. (`**` skips dot-dirs, so .uipath/.skills flows are ignored.) + # Name-agnostic: any unambiguous generated .flow must validate. The exact + # project name is scored separately below. - type: run_command description: "uip flow validate passes (any .flow)" - command: "python3 -c \"import glob,subprocess,sys; flows=glob.glob('**/*.flow',recursive=True); sys.exit('no .flow file found') if not flows else sys.exit(subprocess.run(['uip','maestro','flow','validate',flows[0],'--output','json']).returncode)\"" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py" timeout: 60 expected_exit_code: 0 weight: 3.0 @@ -85,7 +87,7 @@ success_criteria: # below so a missing priority does not zero out an otherwise-correct schema. - type: run_command description: "HITL quickform has input + output fields and Approve/Reject outcomes" - command: "python3 -c \"import json,glob; flows=glob.glob('**/*.flow',recursive=True); assert flows,'No flow file'; f=json.load(open(flows[0])); nodes=f['nodes']; hitl=[n for n in nodes if str(n.get('type','')).startswith('uipath.human-in-the-loop')]; assert hitl,'need a HITL quickform node'; fields=hitl[0].get('inputs',{}).get('schema',{}).get('fields',[]); dirs={x.get('direction') for x in fields}; assert 'input' in dirs,'need an input-direction (read-only) field'; assert 'output' in dirs or 'inOut' in dirs,'need an output/inOut (fill-in) field'; s=json.dumps(f).lower(); assert 'approve' in s and 'reject' in s,'need Approve and Reject outcomes'; print('OK')\"" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/check_simulated_hitl.py schema" timeout: 15 expected_exit_code: 0 weight: 4.0 @@ -97,7 +99,7 @@ success_criteria: # expected to elicit urgency when the persona flags the review as time-sensitive). - type: run_command description: "HITL review priority is set to High (elicited from the user)" - command: "python3 -c \"import json,glob; flows=glob.glob('**/*.flow',recursive=True); assert flows,'No flow file'; f=json.load(open(flows[0])); hitl=[n for n in f['nodes'] if str(n.get('type','')).startswith('uipath.human-in-the-loop')]; assert hitl,'need a HITL quickform node'; assert 'high' in json.dumps(hitl[0]).lower(),'priority should be High'; print('OK')\"" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/check_simulated_hitl.py priority" timeout: 15 expected_exit_code: 0 weight: 1.5 @@ -110,7 +112,7 @@ success_criteria: # keeps it from dominating the score. - type: run_command description: "Project/flow is named VendorPOReview (elicited from the user)" - command: "python3 -c \"import glob,sys; sys.exit(0 if glob.glob('**/VendorPOReview.flow',recursive=True) else 'expected a flow named VendorPOReview.flow')\"" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py --flow-name VendorPOReview" timeout: 15 expected_exit_code: 0 weight: 1.0 diff --git a/tests/tasks/uipath-maestro-flow/interactive/ixp_invoice_extraction_simulated/ixp_invoice_extraction_simulated.yaml b/tests/tasks/uipath-maestro-flow/interactive/ixp_invoice_extraction_simulated/ixp_invoice_extraction_simulated.yaml index 941c937101..3be74f21c2 100644 --- a/tests/tasks/uipath-maestro-flow/interactive/ixp_invoice_extraction_simulated/ixp_invoice_extraction_simulated.yaml +++ b/tests/tasks/uipath-maestro-flow/interactive/ixp_invoice_extraction_simulated/ixp_invoice_extraction_simulated.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-ixp-invoice-extraction-simulated description: > Invoice-processing flow (SharePoint trigger → IxP extraction → HTTP POST to @@ -72,12 +75,11 @@ simulation: n_trials: 1 success_criteria: - # Name-agnostic: any .flow the agent built must validate. The exact project - # name is scored separately below so a working-but-misnamed flow still earns - # most of the credit. (`**` skips dot-dirs, so .uipath/.skills flows are ignored.) + # Name-agnostic: any unambiguous generated .flow must validate. The exact + # project name is scored separately below. - type: run_command description: "uip flow validate passes (any .flow)" - command: "python3 -c \"import glob,subprocess,sys; flows=glob.glob('**/*.flow',recursive=True); sys.exit('no .flow file found') if not flows else sys.exit(subprocess.run(['uip','maestro','flow','validate',flows[0],'--output','json']).returncode)\"" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py" timeout: 60 expected_exit_code: 0 weight: 3.0 @@ -88,7 +90,7 @@ success_criteria: # node, and a $vars. reference consuming the extraction output downstream. - type: run_command description: "Flow has IxP extraction + SharePoint + HTTP nodes and a $vars downstream reference" - command: "python3 -c \"import json,glob; flows=glob.glob('**/*.flow',recursive=True); assert flows,'No flow file'; txt=open(flows[0]).read(); low=txt.lower(); f=json.loads(txt); types=[str(n.get('type','')).lower() for n in f.get('nodes',[])]; assert 'uipath.ixp' in low or 'core.logic.mock' in low,'no IxP extraction node (or mock fallback)'; assert 'sharepoint' in low,'no SharePoint trigger/connector'; assert any('http' in t for t in types),'no HTTP action node'; assert (chr(36)+'vars.') in txt,'no vars downstream reference to extraction output'; print('OK')\"" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py '$vars.' --regex '(?i)(uipath\\.ixp|core\\.logic\\.mock)' --regex '(?i)sharepoint' --regex '(?i)\"type\"\\s*:\\s*\"[^\"]*http'" timeout: 15 expected_exit_code: 0 weight: 6.0 @@ -101,7 +103,7 @@ success_criteria: # keeps it from dominating the score. - type: run_command description: "Project/flow is named InvoiceProcessing (elicited from the user)" - command: "python3 -c \"import glob,sys; sys.exit(0 if glob.glob('**/InvoiceProcessing.flow',recursive=True) else 'expected a flow named InvoiceProcessing.flow')\"" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py --flow-name InvoiceProcessing" timeout: 15 expected_exit_code: 0 weight: 1.0 diff --git a/tests/tasks/uipath-maestro-flow/interactive/slack_channel_description_simulated/slack_channel_description_simulated.yaml b/tests/tasks/uipath-maestro-flow/interactive/slack_channel_description_simulated/slack_channel_description_simulated.yaml index cae6ae91fe..21efc37fe2 100644 --- a/tests/tasks/uipath-maestro-flow/interactive/slack_channel_description_simulated/slack_channel_description_simulated.yaml +++ b/tests/tasks/uipath-maestro-flow/interactive/slack_channel_description_simulated/slack_channel_description_simulated.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-slack-channel-description-simulated description: > Single-connector Slack flow (read a channel's description and output it), @@ -37,11 +40,11 @@ simulation: you: the channel is #office-bellevue in Slack, you want that channel's description text as the result, and it has to work on its own without you supplying the channel every time it runs. Say that it has to be built as a - UiPath Maestro Flow — not an API workflow, not a standalone script — and - that you want it validated before you sign off. Do not name files or paths - and do not use technical wording. Close by inviting the agent to ask you - anything it needs before it starts building. Keep the withheld items below - to yourself until it asks. + UiPath Maestro Flow — not an API workflow, not a standalone script — inside + a solution of the same name, and that you want it validated before you sign + off. Do not name files or paths and do not use technical wording. Close by + inviting the agent to ask you anything it needs before it starts building. + Keep the withheld items below to yourself until it asks. WITHHELD — reveal each of these ONLY if the agent asks about it: - The project name should be "SlackChannelDescription". @@ -62,12 +65,11 @@ simulation: n_trials: 1 success_criteria: - # Name-agnostic: any .flow the agent built must validate. The exact project - # name is scored separately below so a working-but-misnamed flow still earns - # most of the credit. (`**` skips dot-dirs, so .uipath/.skills flows are ignored.) + # Name-agnostic: any unambiguous generated .flow must validate. The exact + # project name is scored separately below. - type: run_command description: "uip flow validate passes (any .flow)" - command: "python3 -c \"import glob,subprocess,sys; flows=glob.glob('**/*.flow',recursive=True); sys.exit('no .flow file found') if not flows else sys.exit(subprocess.run(['uip','maestro','flow','validate',flows[0],'--output','json']).returncode)\"" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py" timeout: 60 expected_exit_code: 0 weight: 3.0 @@ -81,7 +83,7 @@ success_criteria: # unresolved channel fails here regardless of what the static check below reports. - type: run_command description: "Flow debug runs; Slack connector executed and output has the channel description" - command: "python3 $TASK_DIR/check_channel_description.py" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/interactive/slack_channel_description_simulated/check_channel_description.py" timeout: 600 expected_exit_code: 0 weight: 5.0 @@ -107,7 +109,7 @@ success_criteria: # hard-fail the whole task — see the block comment; scored for signal only. - type: run_command description: "Flow targets the #office-bellevue channel (elicited from the user)" - command: "python3 -c \"import glob,re,sys; flows=glob.glob('**/*.flow',recursive=True); t=(open(flows[0]).read() if flows else ''); sys.exit('no .flow file found') if not flows else sys.exit(0 if ('office-bellevue' in t.lower() or re.search(r'conversationsInfoId[^A-Za-z0-9]+C[A-Z0-9]{6,}', t)) else 'flow neither names #office-bellevue nor hardwires a resolved Slack channel ID')\"" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py --regex '(?i)(office-bellevue|conversationsInfoId[^A-Za-z0-9]+C[A-Z0-9]{6,})'" timeout: 15 expected_exit_code: 0 weight: 3.0 @@ -120,7 +122,7 @@ success_criteria: # keeps it from dominating the score. - type: run_command description: "Project/flow is named SlackChannelDescription (elicited from the user)" - command: "python3 -c \"import glob,sys; sys.exit(0 if glob.glob('**/SlackChannelDescription.flow',recursive=True) else 'expected a flow named SlackChannelDescription.flow')\"" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py --flow-name SlackChannelDescription" timeout: 15 expected_exit_code: 0 weight: 1.0 diff --git a/tests/tasks/uipath-maestro-flow/interactive/solution_select.yaml b/tests/tasks/uipath-maestro-flow/interactive/solution_select.yaml index 29b8b58fb6..4d6ff789cb 100644 --- a/tests/tasks/uipath-maestro-flow/interactive/solution_select.yaml +++ b/tests/tasks/uipath-maestro-flow/interactive/solution_select.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-solution-select-ask description: > Interactive-mode variant of init-validate. The working directory already @@ -39,7 +42,7 @@ simulation: You are a UiPath developer working in a directory that already holds a couple of unrelated solutions (SolarReports and TideTracker). You want a fresh, separate solution for this new work and you trust the agent to - drive the CLI. + drive the work. goal: | Get a new Flow project called WeatherAlert created in a separate solution and validated successfully. @@ -69,19 +72,20 @@ success_criteria: weight: 3.0 pass_threshold: 1.0 - - type: file_exists + - type: run_command description: "Flow file created inside the user-selected solution" - path: "WeatherSelection-7K4M/WeatherAlert/WeatherAlert.flow" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/interactive/check_solution_select.py flow" + timeout: 30 + expected_exit_code: 0 weight: 2.0 pass_threshold: 1.0 # ── Did NOT silently adopt an existing solution ──────────────────────── - - type: command_executed + - type: run_command description: "Agent did not init the Flow project inside a pre-existing solution" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+(maestro\s+)?flow\s+init\b.*(SolarReports|TideTracker)' - min_count: 0 - max_count: 0 + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/interactive/check_solution_select.py existing-untouched" + timeout: 30 + expected_exit_code: 0 weight: 1.5 pass_threshold: 1.0 @@ -108,32 +112,33 @@ success_criteria: # ── Build is valid in the selected solution ──────────────────────────── - type: run_command description: "Flow validates in the selected solution" - command: "uip maestro flow validate WeatherSelection-7K4M/WeatherAlert/WeatherAlert.flow --output json" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/interactive/check_solution_select.py validate" timeout: 60 expected_exit_code: 0 weight: 1.5 pass_threshold: 1.0 - - type: command_executed - description: "Agent initialized a Flow project with uip maestro flow init" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+(maestro\s+)?flow\s+init' - min_count: 1 + - type: run_command + description: "Selected solution contains an initialized Flow project" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/interactive/check_solution_select.py project" + timeout: 30 + expected_exit_code: 0 weight: 1.0 pass_threshold: 1.0 # ── Advisory: recorded in the report, never gates the run ────────────── - - type: command_executed - description: "Agent created a solution with uip solution init (or legacy new)" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+solution\s+(init|new)' - min_count: 1 + - type: run_command + description: "Agent created the selected solution artifact" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/interactive/check_solution_select.py solution" + timeout: 30 + expected_exit_code: 0 weight: 1.0 pass_threshold: 0.0 # advisory: outcome already enforced by the WeatherSelection-7K4M json_check; records the explicit-init convention only - - type: command_not_executed + - type: run_command description: "Advisory: no default WeatherAlert solution before the name was given — records the default-then-repair inefficiency; nondeterministic on simulator name-timing, so non-gating" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+solution\s+(init|new)\s+["'']?WeatherAlert(?:["'']?\s|$)' + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/interactive/check_solution_select.py no-extra-solution" + timeout: 30 + expected_exit_code: 0 weight: 1.0 pass_threshold: 0.0 diff --git a/tests/tasks/uipath-maestro-flow/ixp/check_project_selection.py b/tests/tasks/uipath-maestro-flow/ixp/check_project_selection.py new file mode 100644 index 0000000000..401bed4766 --- /dev/null +++ b/tests/tasks/uipath-maestro-flow/ixp/check_project_selection.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +"""Verify that the dataset row selected its exact published IxP model.""" + +from __future__ import annotations + +import json +import os +import sys + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) +from _shared.flow_check import find_flow_file # noqa: E402 + + +EXPECTED = { + "aviation": ( + "aviation-investigation-final-report-demo-a42b1d4d-ixp", + "uipath.ixp.aviation-investigation-final-report-demo-a42b1d4d-ixp.", + ), + "birth-certificate": ( + "birth_certificates_oob-6252526a-ixp", + "uipath.ixp.birth-certificates-oob-6252526a-ixp.", + ), +} + + +def _nested_values(value, key: str): + if isinstance(value, dict): + for nested_key, nested_value in value.items(): + if nested_key == key: + yield nested_value + yield from _nested_values(nested_value, key) + elif isinstance(value, list): + for item in value: + yield from _nested_values(item, key) + + +def main() -> int: + row = next((part for part in os.getcwd().split(os.sep) if part in EXPECTED), None) + if row is None: + print(f"FAIL: unknown dataset row in cwd: {os.getcwd()}", file=sys.stderr) + return 2 + + path = find_flow_file() + with open(path, encoding="utf-8") as f: + flow = json.load(f) + + model_name, type_prefix = EXPECTED[row] + for node in flow.get("nodes", []): + has_model = any( + value == model_name for value in _nested_values(node, "modelName") + ) + if str(node.get("type", "")).startswith(type_prefix) and has_model: + print(f"OK: {row} uses published model {model_name} in {path}") + return 0 + + print( + f"FAIL: no IxP node in {path} pairs type prefix {type_prefix!r} " + f"with modelName {model_name!r}", + file=sys.stderr, + ) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/tasks/uipath-maestro-flow/ixp/e2e_02_project_selection.yaml b/tests/tasks/uipath-maestro-flow/ixp/e2e_02_project_selection.yaml index e0b9008635..13e08da07a 100644 --- a/tests/tasks/uipath-maestro-flow/ixp/e2e_02_project_selection.yaml +++ b/tests/tasks/uipath-maestro-flow/ixp/e2e_02_project_selection.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-ixp-e2e-project-selection description: > E2E (project selection): on a tenant where the demo IxP project set is @@ -45,14 +48,7 @@ dataset: initial_prompt: | ${row.prompt} - - Important: - - The `uip` CLI is already available and the runner is logged in. - - Use `--output json` on all uip commands. - - Work inside the stated working directory — do NOT `cd` to a parent. - - Do NOT run `uip maestro flow debug` or `uip maestro flow deploy`. - - Run `uip maestro flow validate` at most 3 times. If it reports errors, fix - the `.flow` and re-run it. Do not exceed 3 validation runs. + Produce a complete flow that validates. This run is headless. No user is present and nobody will answer a question or grant an approval, so do not ask, do not pause, and do not wait for input. @@ -69,25 +65,22 @@ success_criteria: - type: skill_triggered skill_name: "uipath-maestro-flow" expected_skill: "uipath-maestro-flow" - description: "Agent invoked the uipath-maestro-flow Skill" + description: "Advisory: agent invoked the uipath-maestro-flow skill" weight: 2.0 - pass_threshold: 1.0 + pass_threshold: 0.0 - type: command_executed - description: "Agent searched the registry for IxP node types" + description: "Advisory: live-v1 agent searched the registry for IxP node types" tool_name: "Bash" command_pattern: 'uip\s+maestro\s+flow\s+registry\s+(search|list|get)\b[^\n]*ixp' min_count: 1 weight: 2.0 - pass_threshold: 1.0 + pass_threshold: 0.0 - # `../..` cwd rationale: dataset-fanout cwd - # is /artifacts//[/]/, so artifacts written - # by the agent live two levels up at most. - type: run_command description: "uip maestro flow validate passes on the generated flow" - command: 'f=$(find ../.. -name "*.flow" -print -quit); [ -n "$f" ] && uip maestro flow validate "$f" --output json' - timeout: 60 + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/validate_flow.py" + timeout: 30 expected_exit_code: 0 weight: 3.0 pass_threshold: 1.0 @@ -96,16 +89,14 @@ success_criteria: # a domain-matching IxP project IS published. - type: run_command description: "Generated flow contains a real uipath.ixp.* node" - command: 'grep -rqE "\"type\"[[:space:]]*:[[:space:]]*\"uipath\.ixp\." --include="*.flow" ../..' + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py --regex '\"type\"\\s*:\\s*\"uipath\\.ixp\\.'" expected_exit_code: 0 weight: 3.0 pass_threshold: 1.0 - # Inverted twin of the above. `! grep -q` exits 0 when no mock is found - # (PASS) and 1 when one exists (FAIL). Same `../..` rationale as above. - type: run_command description: "Generated flow does NOT contain a core.logic.mock fallback" - command: '! grep -rqE "\"type\"[[:space:]]*:[[:space:]]*\"core\.logic\.mock\"" --include="*.flow" ../..' + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py --absent-regex '\"type\"\\s*:\\s*\"core\\.logic\\.mock\"'" expected_exit_code: 0 weight: 2.0 pass_threshold: 1.0 @@ -118,36 +109,8 @@ success_criteria: # dataset IDs in scripts/ixp-demo-projects/datasets.jsonl. - type: run_command description: "IxP node references the exact published model for this row's domain (strict)" - command: | - case "$(pwd)" in - *aviation*) - model='aviation-investigation-final-report-demo-a42b1d4d-ixp' - nt='uipath\.ixp\.aviation-investigation-final-report-demo-a42b1d4d-ixp\.' - ;; - *birth*) - model='birth_certificates_oob-6252526a-ixp' - nt='uipath\.ixp\.birth-certificates-oob-6252526a-ixp\.' - ;; - *) - echo "Unknown row in pwd: $(pwd)" >&2 - exit 2 - ;; - esac - f=$(find ../.. -name "*.flow" -print -quit) - if [ -z "$f" ]; then - echo "no .flow file under ../.." >&2 - exit 1 - fi - if ! grep -qF "\"modelName\": \"${model}\"" "$f"; then - echo "modelName mismatch — expected '${model}' in $f" >&2 - grep -E '"modelName"' "$f" >&2 || true - exit 1 - fi - if ! grep -qE "\"type\"[[:space:]]*:[[:space:]]*\"${nt}" "$f"; then - echo "NodeType mismatch — expected prefix '${nt}' in $f" >&2 - grep -E '"type"[[:space:]]*:[[:space:]]*"uipath\.ixp' "$f" >&2 || true - exit 1 - fi + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/ixp/check_project_selection.py" + timeout: 30 expected_exit_code: 0 weight: 5.0 pass_threshold: 1.0 diff --git a/tests/tasks/uipath-maestro-flow/ixp/e2e_03_project_creation_handoff/handoff.py b/tests/tasks/uipath-maestro-flow/ixp/e2e_03_project_creation_handoff/handoff.py index 32f5180fa6..66552b68b1 100644 --- a/tests/tasks/uipath-maestro-flow/ixp/e2e_03_project_creation_handoff/handoff.py +++ b/tests/tasks/uipath-maestro-flow/ixp/e2e_03_project_creation_handoff/handoff.py @@ -83,9 +83,9 @@ from typing import Any TASK_DIR = os.path.dirname(os.path.abspath(__file__)) -sys.path.insert(0, os.path.join(TASK_DIR, "..", "..", "_shared")) +sys.path.insert(0, os.path.join(TASK_DIR, "..", "..")) -from flow_check import find_project_dir # noqa: E402 +from _shared.flow_check import find_project_dir # noqa: E402 # ── shared helpers ──────────────────────────────────────────────────────────── diff --git a/tests/tasks/uipath-maestro-flow/ixp/integration_handle_routing.yaml b/tests/tasks/uipath-maestro-flow/ixp/integration_handle_routing.yaml index bc077bfdbe..1b37511f20 100644 --- a/tests/tasks/uipath-maestro-flow/ixp/integration_handle_routing.yaml +++ b/tests/tasks/uipath-maestro-flow/ixp/integration_handle_routing.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-ixp-integration-handle-routing description: > Integration: IxP extraction wired into a Decision node that routes on a @@ -37,13 +40,7 @@ initial_prompt: | `$vars..output.ExtractionResult.ResultsDocument.Fields.find(f => f.FieldName === '')?.Values?.[0]` — per references/plugins/ixp/impl.md ("Accessing Output"). - Important: - - The `uip` CLI is already available. - - Use `--output json` on all uip commands. - - Work inside the stated working directory — do NOT `cd` to a parent. - - Do NOT run `uip maestro flow debug` or `uip maestro flow deploy`. - - Run `uip maestro flow validate` at most 3 times. If it reports errors, fix - the `.flow` and re-run it. Do not exceed 3 validation runs. + Validate the completed flow. This run is headless. No user is present and nobody will answer a question or grant an approval, so do not ask, do not pause, and do not wait for input. @@ -60,17 +57,17 @@ success_criteria: - type: skill_triggered skill_name: "uipath-maestro-flow" expected_skill: "uipath-maestro-flow" - description: "Agent invoked the uipath-maestro-flow Skill" + description: "Advisory: agent invoked the uipath-maestro-flow skill" weight: 2.0 - pass_threshold: 1.0 + pass_threshold: 0.0 - type: command_executed - description: "Agent searched the registry for IxP node types" + description: "Advisory: live-v1 agent searched the registry for IxP node types" tool_name: "Bash" command_pattern: 'uip\s+maestro\s+flow\s+registry\s+(search|list|get)\b[^\n]*ixp' min_count: 1 weight: 2.0 - pass_threshold: 1.0 + pass_threshold: 0.0 - type: run_command description: "uip maestro flow validate passes on InvoiceRouter.flow" @@ -82,58 +79,37 @@ success_criteria: - type: run_command description: "Generated flow contains an IxP extraction node or core.logic.mock fallback" - command: 'grep -rqE "\"type\"[[:space:]]*:[[:space:]]*\"(uipath\.ixp|core\.logic\.mock)" --include="*.flow" .' + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py --regex '\"type\"\\s*:\\s*\"(uipath\\.ixp|core\\.logic\\.mock)'" expected_exit_code: 0 weight: 2.0 pass_threshold: 1.0 - # The four checks below locate the .flow rather than hardcoding - # `//.flow`. `uip maestro flow init ` run outside a - # solution auto-scaffolds a `Solution/` wrapper (see greenfield.md), - # so the literal path only resolves when the agent happens to run - # `uip solution init` first — an authoring idiom these checks don't test. - # Each grep line is one `includes:` needle; all must match. + # The checks below use shared discovery rather than hardcoding a solution + # wrapper shape. Each set of strings must match within one Flow artifact. - type: run_command description: "Flow contains a Decision node (core.logic.decision)" - command: | - set -e - flow=$(find . -name '*.flow' -print -quit) - grep -qF 'core.logic.decision' "$flow" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py 'core.logic.decision'" expected_exit_code: 0 weight: 2.0 pass_threshold: 1.0 - type: run_command description: "Decision condition references the canonical IxP output path ($vars..output.ExtractionResult.ResultsDocument.Fields)" - command: | - set -e - flow=$(find . -name '*.flow' -print -quit) - grep -qF '$vars.' "$flow" - grep -qF 'ExtractionResult' "$flow" - grep -qF 'ResultsDocument' "$flow" - grep -qF 'Fields' "$flow" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py '$vars.' 'ExtractionResult' 'ResultsDocument' 'Fields'" expected_exit_code: 0 weight: 3.0 pass_threshold: 1.0 - type: run_command description: "Both branch script nodes are present by name" - command: | - set -e - flow=$(find . -name '*.flow' -print -quit) - grep -qF 'HighValueReview' "$flow" - grep -qF 'AutoApprove' "$flow" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py 'HighValueReview' 'AutoApprove'" expected_exit_code: 0 weight: 2.5 pass_threshold: 1.0 - type: run_command description: "Both Decision branches wired (true and false handles)" - command: | - set -e - flow=$(find . -name '*.flow' -print -quit) - grep -qF '"true"' "$flow" - grep -qF '"false"' "$flow" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py '\"true\"' '\"false\"'" expected_exit_code: 0 weight: 2.0 pass_threshold: 1.0 diff --git a/tests/tasks/uipath-maestro-flow/ixp/routing.yaml b/tests/tasks/uipath-maestro-flow/ixp/routing.yaml index 3bd72c7054..b27bf79c03 100644 --- a/tests/tasks/uipath-maestro-flow/ixp/routing.yaml +++ b/tests/tasks/uipath-maestro-flow/ixp/routing.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-ixp-routing description: > IxP routing (positive) — document-extraction build prompts fanned out via @@ -5,9 +8,9 @@ description: > identical across all rows. Scope: did the agent route to the IxP plugin, not did the flow work. - Assertions verify the agent ran the right CLI-command patterns (registry - search for uipath.ixp.*) and landed a uipath.ixp.* or core.logic.mock node — - NOT whether the resulting flow is correct or would execute. + The gating assertion verifies that the artifact contains a uipath.ixp.* or + core.logic.mock node. Registry-search telemetry remains report-only because + the two authoring loops use different commands. Failure here is signal about SKILL.md or the IxP Plugin Index entry — do NOT relax assertions; feed findings back in. @@ -50,14 +53,14 @@ dataset: initial_prompt: | ${row.prompt} - Important: - - The `uip` CLI is already available. - - Use `--output json` on all uip commands. - - Work inside the stated working directory — do NOT `cd` to a parent. - - Do NOT run `uip maestro flow debug` or `uip maestro flow deploy`. - - Run `uip maestro flow validate` at most ONCE. If it fails, stop. - - Do NOT iterate on validation errors or rewrite the flow to fix them. A broken flow is acceptable. - - You do not need a working flow — a skeleton whose nodes cover the steps of the request is enough. Leave a node unconfigured rather than leaving a step out. + Scope: this is a routing check, not a build-to-run task. The deliverable is + the flow artifact itself — the `.flow` file. A skeleton whose nodes cover the + steps of the request is enough: leave a node unconfigured rather than + leaving a step out or stopping to ask; pick a sensible default for any + missing detail and state it in your final report. Do not iterate on validation errors to make the flow + runnable — a broken skeleton is acceptable, as long as the `.flow` file has + been produced. Work inside the current working directory: create the + solution and project here, never in a parent directory. This run is headless. No user is present and nobody will answer a question or grant an approval, so do not ask, do not pause, and do not wait for input. @@ -79,23 +82,26 @@ success_criteria: # routing_listing.yaml drops `get` (read-only). Keep in sync with # scaffold_minimal.yaml, scaffold_multinode.yaml, routing_listing.yaml. - type: command_executed - description: "Agent searched the registry for IxP node types" + description: "Advisory: live-v1 agent searched the registry for IxP node types" tool_name: "Bash" command_pattern: '(?i)uip\s+maestro\s+flow\s+registry\s+(list\b|(search|list|get)\b[^\n]*\b(ixp|document|extract))' min_count: 1 - suite_thresholds: {mean: 0.85} - # Armed latch-only: keeps this criterion gated on sentinel-cut runs (those - # gate on ARMED criteria only). Fail-stop is unreachable for a min_count. + pass_threshold: 0.0 + # Armed latch-only (kept from main #2881): the sentinel below defers its + # pass-stop until this criterion has live-passed, so a live-v1 run is cut + # only after it has searched the registry AND written the node. Fail-stop is + # unreachable for a min_count. Advisory (pass_threshold 0.0) under the + # same-ground regime: the SDK loop never runs this command, so it must not + # gate — on an SDK-loop run the sentinel simply never fires and the run + # completes and grades exactly as before. stop_early: {} - # Greps the actual .flow artifact rather than the command stream — the - # command_executed checker truncates tool params at 2000 chars, which hides - # `"type": "core.logic.mock"` when the agent writes a large flow file in a - # single Write call. Per references/plugins/ixp/impl.md, the agent must land - # either an `uipath.ixp.*` node or a `core.logic.mock` placeholder. + # Inspect the actual .flow artifact rather than the command stream. Per + # references/plugins/ixp/impl.md, the agent must land either an + # `uipath.ixp.*` node or a `core.logic.mock` placeholder. - type: run_command description: "Agent added an IxP or core.logic.mock placeholder node to a .flow file (per references/plugins/ixp/impl.md)" - command: 'grep -rqE "\"type\"[[:space:]]*:[[:space:]]*\"(uipath\.ixp|core\.logic\.mock)" --include="*.flow" .' + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py --regex '\"type\"\\s*:\\s*\"(uipath\\.ixp|core\\.logic\\.mock)'" expected_exit_code: 0 suite_thresholds: {mean: 0.85} diff --git a/tests/tasks/uipath-maestro-flow/ixp/routing_listing.yaml b/tests/tasks/uipath-maestro-flow/ixp/routing_listing.yaml index 82f165ca1a..0173b9931a 100644 --- a/tests/tasks/uipath-maestro-flow/ixp/routing_listing.yaml +++ b/tests/tasks/uipath-maestro-flow/ixp/routing_listing.yaml @@ -1,12 +1,15 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-ixp-routing-listing description: > IxP listing routing — user asks what IxP models / runtime projects are available from a Maestro flow. Read-only Q&A, not a build task. - Asserts the agent ran the IxP registry-search listing command (per - references/plugins/ixp/impl.md — Listing Published Models). Also asserts the - agent did NOT take the build path: no .flow file should be created for a - listing question. + Observes the live-v1 registry-search command as report-only telemetry (per + references/plugins/ixp/impl.md — Listing Published Models). The gating check + asserts that the agent did NOT take the build path: no .flow file should be + created for a listing question. Failure here is signal about the IxP plugin's Listing Published Models / Listing Available Models sections — do NOT relax assertions, tighten the @@ -39,10 +42,7 @@ dataset: initial_prompt: | ${row.prompt} - - Important: - - The `uip` CLI is already available. - - Use `--output json` on all uip commands. + Answer the question without creating a flow. This run is headless. No user is present and nobody will answer a question or grant an approval, so do not ask, do not pause, and do not wait for input. @@ -58,18 +58,16 @@ initial_prompt: | success_criteria: # IxP registry-search check (omits `get`) — see routing.yaml for the pattern breakdown. - type: command_executed - description: "Agent searched the registry for IxP node types (listing path)" + description: "Advisory: live-v1 agent searched the registry for IxP node types" tool_name: "Bash" command_pattern: '(?i)uip\s+maestro\s+flow\s+registry\s+(list\b|(search|list)\b[^\n]*\b(ixp|document|extract))' min_count: 1 - suite_thresholds: {mean: 0.85} + pass_threshold: 0.0 - # Inverted check: a listing/Q&A task should NOT create a .flow file. - # `! find ... | grep -q .` exits 0 when no .flow file is found (PASS) - # and 1 when one exists (FAIL). + # A listing/Q&A task should not create a Flow artifact. - type: run_command description: "No .flow file was created (listing is read-only Q&A, not a build task)" - command: '! find . -name "*.flow" -type f | grep -q .' + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py --expect-none" expected_exit_code: 0 suite_thresholds: {mean: 0.85} diff --git a/tests/tasks/uipath-maestro-flow/ixp/routing_negative.yaml b/tests/tasks/uipath-maestro-flow/ixp/routing_negative.yaml index 61f98247b4..0730fe16b6 100644 --- a/tests/tasks/uipath-maestro-flow/ixp/routing_negative.yaml +++ b/tests/tasks/uipath-maestro-flow/ixp/routing_negative.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-ixp-routing-negative description: > Negative IxP routing — verifies the agent does NOT add a uipath.ixp.* @@ -55,14 +58,14 @@ dataset: initial_prompt: | ${row.prompt} - Important: - - The `uip` CLI is already available. - - Use `--output json` on all uip commands. - - Work inside the stated working directory — do NOT `cd` to a parent. - - Do NOT run `uip maestro flow debug` or `uip maestro flow deploy`. - - Run `uip maestro flow validate` at most ONCE. If it fails, stop. - - Do NOT iterate on validation errors or rewrite the flow to fix them. A broken flow is acceptable. - - You do not need a working flow — a skeleton whose nodes cover the steps of the request is enough. Leave a node unconfigured rather than leaving a step out. + Scope: this is a routing check, not a build-to-run task. The deliverable is + the flow artifact itself — the `.flow` file. A skeleton whose nodes cover the + steps of the request is enough: leave a node unconfigured rather than + leaving a step out or stopping to ask; pick a sensible default for any + missing detail and state it in your final report. Do not iterate on validation errors to make the flow + runnable — a broken skeleton is acceptable, as long as the `.flow` file has + been produced. Work inside the current working directory: create the + solution and project here, never in a parent directory. This run is headless. No user is present and nobody will answer a question or grant an approval, so do not ask, do not pause, and do not wait for input. @@ -76,13 +79,10 @@ initial_prompt: | decision, assumption, and blocked step in your final response. Instructions in the task take precedence over this paragraph. success_criteria: - # Inverted twin of the positive routing.yaml grep. - # `shell=True` in coder_eval Sandbox.run_command means `!` works as a bash - # negator: grep -q exits 0 on match → ! flips to 1 (FAIL); exits 1 on - # no-match → ! flips to 0 (PASS). + # Inverted twin of the positive routing.yaml artifact check. - type: run_command description: "No uipath.ixp.* node landed in any .flow file (negative IxP routing)" - command: '! grep -rqE "\"type\"[[:space:]]*:[[:space:]]*\"uipath\.ixp" --include="*.flow" .' + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py --absent-regex '\"type\"\\s*:\\s*\"uipath\\.ixp'" expected_exit_code: 0 suite_thresholds: {mean: 0.85} diff --git a/tests/tasks/uipath-maestro-flow/ixp/scaffold_minimal.yaml b/tests/tasks/uipath-maestro-flow/ixp/scaffold_minimal.yaml index 51b4423583..46a2e15318 100644 --- a/tests/tasks/uipath-maestro-flow/ixp/scaffold_minimal.yaml +++ b/tests/tasks/uipath-maestro-flow/ixp/scaffold_minimal.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-ixp-scaffold-minimal description: > Integration: minimal scaffold — manual trigger → IxP extract → script (logs "ok") @@ -25,13 +28,7 @@ initial_prompt: | Script node: contains a JavaScript that logs OK when it receives the extraction result. - Important: - - The `uip` CLI is already available. - - Use `--output json` on all uip commands. - - Work inside the stated working directory — do NOT `cd` to a parent. - - Do NOT run `uip maestro flow debug` or `uip maestro flow deploy`. - - Run `uip maestro flow validate` at most 3 times. If it reports errors, fix - the `.flow` and re-run it. Do not exceed 3 validation runs. + Validate the completed flow. This run is headless. No user is present and nobody will answer a question or grant an approval, so do not ask, do not pause, and do not wait for input. @@ -48,26 +45,26 @@ success_criteria: - type: skill_triggered skill_name: "uipath-maestro-flow" expected_skill: "uipath-maestro-flow" - description: "Agent invoked the uipath-maestro-flow Skill" + description: "Advisory: agent invoked the uipath-maestro-flow skill" weight: 2.0 - pass_threshold: 1.0 + pass_threshold: 0.0 # IxP registry-search check — see routing.yaml for the pattern breakdown. - type: command_executed - description: "Agent searched the registry for IxP node types" + description: "Advisory: live-v1 agent searched the registry for IxP node types" tool_name: "Bash" command_pattern: '(?i)uip\s+maestro\s+flow\s+registry\s+(list\b|(search|list|get)\b[^\n]*\b(ixp|document|extract))' min_count: 1 weight: 2.0 - pass_threshold: 1.0 + pass_threshold: 0.0 - type: command_executed - description: "Agent validated the .flow file" + description: "Advisory: live-v1 agent invoked flow validation" tool_name: "Bash" command_pattern: 'uip\s+(maestro\s+)?flow\s+validate' min_count: 1 weight: 1.5 - pass_threshold: 1.0 + pass_threshold: 0.0 - type: run_command description: "uip maestro flow validate passes on Smoke.flow" @@ -79,7 +76,7 @@ success_criteria: - type: run_command description: "Generated flow contains an IxP extraction node or core.logic.mock fallback (per references/plugins/ixp/impl.md)" - command: 'grep -rqE "\"type\"[[:space:]]*:[[:space:]]*\"(uipath\.ixp|core\.logic\.mock)" --include="*.flow" .' + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py --regex '\"type\"\\s*:\\s*\"(uipath\\.ixp|core\\.logic\\.mock)'" expected_exit_code: 0 weight: 2.0 pass_threshold: 1.0 @@ -88,10 +85,7 @@ success_criteria: # integration_handle_routing.yaml for the `Solution/` wrapper rationale. - type: run_command description: "Script node consumes the extraction output via $vars expression" - command: | - set -e - flow=$(find . -name '*.flow' -print -quit) - grep -qF '$vars.' "$flow" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py '$vars.'" expected_exit_code: 0 weight: 1.5 pass_threshold: 1.0 diff --git a/tests/tasks/uipath-maestro-flow/ixp/scaffold_multinode.yaml b/tests/tasks/uipath-maestro-flow/ixp/scaffold_multinode.yaml index 2c489aca37..4eab3326dc 100644 --- a/tests/tasks/uipath-maestro-flow/ixp/scaffold_multinode.yaml +++ b/tests/tasks/uipath-maestro-flow/ixp/scaffold_multinode.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-ixp-scaffold-multinode description: > Integration: multi-script fan-out from an IxP extraction — manual trigger → @@ -29,13 +32,7 @@ initial_prompt: | the three script nodes logs OK when it receives its respective extraction result. - Important: - - The `uip` CLI is already available. - - Use `--output json` on all uip commands. - - Work inside the stated working directory — do NOT `cd` to a parent. - - Do NOT run `uip maestro flow debug` or `uip maestro flow deploy`. - - Run `uip maestro flow validate` at most 3 times. If it reports errors, fix - the `.flow` and re-run it. Do not exceed 3 validation runs. + Validate the completed flow. This run is headless. No user is present and nobody will answer a question or grant an approval, so do not ask, do not pause, and do not wait for input. @@ -52,26 +49,26 @@ success_criteria: - type: skill_triggered skill_name: "uipath-maestro-flow" expected_skill: "uipath-maestro-flow" - description: "Agent invoked the uipath-maestro-flow Skill" + description: "Advisory: agent invoked the uipath-maestro-flow skill" weight: 2.0 - pass_threshold: 1.0 + pass_threshold: 0.0 # IxP registry-search check — see routing.yaml for the pattern breakdown. - type: command_executed - description: "Agent searched the registry for IxP node types" + description: "Advisory: live-v1 agent searched the registry for IxP node types" tool_name: "Bash" command_pattern: '(?i)uip\s+maestro\s+flow\s+registry\s+(list\b|(search|list|get)\b[^\n]*\b(ixp|document|extract))' min_count: 1 weight: 2.0 - pass_threshold: 1.0 + pass_threshold: 0.0 - type: command_executed - description: "Agent validated the .flow file" + description: "Advisory: live-v1 agent invoked flow validation" tool_name: "Bash" command_pattern: 'uip\s+(maestro\s+)?flow\s+validate' min_count: 1 weight: 1.5 - pass_threshold: 1.0 + pass_threshold: 0.0 - type: run_command description: "uip maestro flow validate passes on ReceiptIntake.flow" @@ -83,7 +80,7 @@ success_criteria: - type: run_command description: "Generated flow contains an IxP extraction node or core.logic.mock fallback (per references/plugins/ixp/impl.md)" - command: 'grep -rqE "\"type\"[[:space:]]*:[[:space:]]*\"(uipath\.ixp|core\.logic\.mock)" --include="*.flow" .' + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py --regex '\"type\"\\s*:\\s*\"(uipath\\.ixp|core\\.logic\\.mock)'" expected_exit_code: 0 weight: 2.0 pass_threshold: 1.0 diff --git a/tests/tasks/uipath-maestro-flow/multi_node/bellevue_weather/bellevue_weather.yaml b/tests/tasks/uipath-maestro-flow/multi_node/bellevue_weather/bellevue_weather.yaml index d4be6e8eff..6279087c5f 100644 --- a/tests/tasks/uipath-maestro-flow/multi_node/bellevue_weather/bellevue_weather.yaml +++ b/tests/tasks/uipath-maestro-flow/multi_node/bellevue_weather/bellevue_weather.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-bellevue-weather description: > Create a UiPath Flow that fetches today's weather in Bellevue from open-meteo, @@ -10,6 +13,8 @@ run_limits: turn_timeout: 1200 initial_prompt: | + Build the flow inside a solution of the same name. + Create a UiPath Flow project named "BellevueWeather" that gets today's weather in Bellevue from open-meteo, formats a summary with a script, and if the temperature is greater than 60F returns a summary with a message field 'nice day', diff --git a/tests/tasks/uipath-maestro-flow/multi_node/calculator/calculator.yaml b/tests/tasks/uipath-maestro-flow/multi_node/calculator/calculator.yaml index 600ab56741..871aac3483 100644 --- a/tests/tasks/uipath-maestro-flow/multi_node/calculator/calculator.yaml +++ b/tests/tasks/uipath-maestro-flow/multi_node/calculator/calculator.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-calculator description: > Create a UiPath Flow that takes two number inputs and calculates their product @@ -9,6 +12,8 @@ run_limits: turn_timeout: 1200 initial_prompt: | + Build the flow inside a solution of the same name. + Create a UiPath Flow project named "Calculator" that takes two numbers as input and calculates their product using a Script node. The result should be returned as an output variable. diff --git a/tests/tasks/uipath-maestro-flow/multi_node/customer_escalation/check_customer_escalation_flow.py b/tests/tasks/uipath-maestro-flow/multi_node/customer_escalation/check_customer_escalation_flow.py index a35e90ec25..00119cca3d 100644 --- a/tests/tasks/uipath-maestro-flow/multi_node/customer_escalation/check_customer_escalation_flow.py +++ b/tests/tasks/uipath-maestro-flow/multi_node/customer_escalation/check_customer_escalation_flow.py @@ -25,15 +25,13 @@ from __future__ import annotations -import glob import json -import os import sys from pathlib import Path from typing import Any, NoReturn -sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) -from _shared.flow_check import find_project_dir # noqa: E402 +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) +from _shared.flow_check import find_flow_file # noqa: E402 def fail(msg: str) -> NoReturn: @@ -41,23 +39,7 @@ def fail(msg: str) -> NoReturn: def load_flow() -> dict[str, Any]: - # Discover the Flow project rather than hardcoding - # `CustomerEscalation/CustomerEscalation/CustomerEscalation.flow`: the prompt - # names only the FLOW ('a Maestro Flow called "CustomerEscalation"'), never - # the enclosing solution, and the two names are independent — `uip solution - # init && cd && uip maestro flow init ` yields `//.flow` - # for any , while a bare `uip maestro flow init ` auto-scaffolds - # `Solution//.flow`. A path-pinned glob graded the solution name, - # an unstated requirement: skill-flow-customer-escalation scored 0.0 on this - # criterion with a fully valid flow at - # `CustomerEscalationSolution/CustomerEscalation/CustomerEscalation.flow`. - # Mirrors the discovery in `_shared/validate_flow.py`, so this criterion and - # the validate criterion address the same file. - project_dir = find_project_dir() - matches = sorted(glob.glob(os.path.join(project_dir, "**/*.flow"), recursive=True)) - if not matches: - fail(f"No .flow file found under {project_dir}") - path = Path(matches[0]) + path = Path(find_flow_file(flow_glob="CustomerEscalation*.flow")) try: return json.loads(path.read_text()) except json.JSONDecodeError as exc: diff --git a/tests/tasks/uipath-maestro-flow/multi_node/customer_escalation/customer_escalation.yaml b/tests/tasks/uipath-maestro-flow/multi_node/customer_escalation/customer_escalation.yaml index e896822ef3..46bdde0cb8 100644 --- a/tests/tasks/uipath-maestro-flow/multi_node/customer_escalation/customer_escalation.yaml +++ b/tests/tasks/uipath-maestro-flow/multi_node/customer_escalation/customer_escalation.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-customer-escalation description: > Complex multi-branch escalation flow: Outlook email-received trigger → @@ -66,4 +69,3 @@ success_criteria: expected_exit_code: 0 weight: 10.0 pass_threshold: 1.0 - diff --git a/tests/tasks/uipath-maestro-flow/multi_node/customer_escalation/test_check_customer_escalation_flow.py b/tests/tasks/uipath-maestro-flow/multi_node/customer_escalation/test_check_customer_escalation_flow.py index 370a2ad82c..bb98f916f7 100644 --- a/tests/tasks/uipath-maestro-flow/multi_node/customer_escalation/test_check_customer_escalation_flow.py +++ b/tests/tasks/uipath-maestro-flow/multi_node/customer_escalation/test_check_customer_escalation_flow.py @@ -11,8 +11,8 @@ - Structural requirements (>=5 nodes, exactly one decision with >=2 branches, >=2 scripts, Slack reference, non-trigger Outlook/Graph reference) are still enforced. -- The verdict is independent of the enclosing SOLUTION directory's name, - which the prompt never specifies (see `test_any_solution_name_passes`). +- The verdict is independent of the authoring layout: any solution directory + name or one root-level SDK emit is accepted. """ from __future__ import annotations @@ -42,8 +42,7 @@ def _flow_dir(tmp_path: Path, solution: str = "CustomerEscalationSolution") -> P against a layout the CLI cannot emit, which is why the checker's path-coupling survived to fail a real run. - `project.uiproj` is required: `find_project_dir` selects the project whose - manifest declares `ProjectType: "Flow"`. + `project.uiproj` makes this the live-v1 solution-layout fixture. """ d = tmp_path / solution / "CustomerEscalation" d.mkdir(parents=True) @@ -154,6 +153,16 @@ def test_any_solution_name_passes(tmp_path: Path, solution: str) -> None: assert result.returncode == 0, result.stdout + result.stderr +def test_root_level_sdk_emit_passes(tmp_path: Path) -> None: + nodes, edges = _outlook_connector_shape() + payload = {"version": "1.0.0", "nodes": nodes, "edges": edges} + (tmp_path / "CustomerEscalation.flow").write_text(json.dumps(payload)) + + result = _run(tmp_path) + + assert result.returncode == 0, result.stdout + result.stderr + + def test_missing_outlook_reference_fails(tmp_path: Path) -> None: """Flow with NO outlook/office365/graph reference on any reply node FAILS.""" nodes, edges = _manual_trigger_with_graph_replies() diff --git a/tests/tasks/uipath-maestro-flow/multi_node/dice_roller/dice_roller.yaml b/tests/tasks/uipath-maestro-flow/multi_node/dice_roller/dice_roller.yaml index 6a05aaa795..edc7a00009 100644 --- a/tests/tasks/uipath-maestro-flow/multi_node/dice_roller/dice_roller.yaml +++ b/tests/tasks/uipath-maestro-flow/multi_node/dice_roller/dice_roller.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-dice-roller description: > Create a UiPath Flow from scratch that simulates rolling a fair six-sided die. @@ -11,6 +14,8 @@ run_limits: turn_timeout: 1200 initial_prompt: | + Build the flow inside a solution of the same name. + Create a UiPath Flow project named "DiceRoller" that simulates rolling a six-sided die and outputs the result. diff --git a/tests/tasks/uipath-maestro-flow/multi_node/feet_inches/feet_inches.yaml b/tests/tasks/uipath-maestro-flow/multi_node/feet_inches/feet_inches.yaml index 6df0d53636..f88451aed6 100644 --- a/tests/tasks/uipath-maestro-flow/multi_node/feet_inches/feet_inches.yaml +++ b/tests/tasks/uipath-maestro-flow/multi_node/feet_inches/feet_inches.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-feet-inches description: > Create a UiPath Flow that converts a value between feet and inches based on @@ -11,6 +14,8 @@ run_limits: task_timeout: 2700 # 1380s grading + a turn; inherited 1200 could not fit grading initial_prompt: | + Build the flow inside a solution of the same name. + Create a UiPath Flow project named "FeetInches" that takes `value` (number) and `direction` (string: "f2i", "i2f", or "y2f") as inputs, converts feet to inches (× 12), inches to feet (÷ 12), or yards to feet (× 3) accordingly, diff --git a/tests/tasks/uipath-maestro-flow/multi_node/loop_multiply/loop_multiply.yaml b/tests/tasks/uipath-maestro-flow/multi_node/loop_multiply/loop_multiply.yaml index f715d216a0..07da65ec25 100644 --- a/tests/tasks/uipath-maestro-flow/multi_node/loop_multiply/loop_multiply.yaml +++ b/tests/tasks/uipath-maestro-flow/multi_node/loop_multiply/loop_multiply.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-loop-multiply description: > Create a UiPath Flow that uses a Loop node to multiply the numbers @@ -10,6 +13,8 @@ run_limits: turn_timeout: 1200 initial_prompt: | + Build the flow inside a solution of the same name. + Create a UiPath Flow project named "LoopMultiply" that multiplies the numbers [13, 15, 17] together using a Loop node and returns the product. diff --git a/tests/tasks/uipath-maestro-flow/multi_node/multi_city_weather/multi_city_weather.yaml b/tests/tasks/uipath-maestro-flow/multi_node/multi_city_weather/multi_city_weather.yaml index 5dda9ee47f..ab981c30ad 100644 --- a/tests/tasks/uipath-maestro-flow/multi_node/multi_city_weather/multi_city_weather.yaml +++ b/tests/tasks/uipath-maestro-flow/multi_node/multi_city_weather/multi_city_weather.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-multi-city-weather description: > Loop over 3 cities, fetch weather from open-meteo for each, classify warm/cold @@ -10,6 +13,8 @@ run_limits: turn_timeout: 1200 initial_prompt: | + Build the flow inside a solution of the same name. + Create a flow called "MultiCityWeather". Loop over Seattle, Phoenix, and New York — for each city, fetch the current temperature from open-meteo (fahrenheit) and classify it as 'warm' (> 60F) or 'cold'. Output an array with all 3 results, each having the city name, temperature, and verdict. Use a Managed HTTP Request node (core.action.http.v2) to call the open-meteo API directly — do not use an Integration Service connector. diff --git a/tests/tasks/uipath-maestro-flow/multi_node/reading_list/reading_list.yaml b/tests/tasks/uipath-maestro-flow/multi_node/reading_list/reading_list.yaml index eda77be31e..8e40d181aa 100644 --- a/tests/tasks/uipath-maestro-flow/multi_node/reading_list/reading_list.yaml +++ b/tests/tasks/uipath-maestro-flow/multi_node/reading_list/reading_list.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-reading-list description: > Create a UiPath Flow that curates a reading list from a catalog of math/ML/stats @@ -11,6 +14,8 @@ run_limits: turn_timeout: 1200 initial_prompt: | + Build the flow inside a solution of the same name. + Create a UiPath Flow project named "ReadingList" that curates a reading list from a catalog of math and data science books. diff --git a/tests/tasks/uipath-maestro-flow/multi_node/slack_channel_description/slack_channel_description.yaml b/tests/tasks/uipath-maestro-flow/multi_node/slack_channel_description/slack_channel_description.yaml index de1547175c..0b9e75b603 100644 --- a/tests/tasks/uipath-maestro-flow/multi_node/slack_channel_description/slack_channel_description.yaml +++ b/tests/tasks/uipath-maestro-flow/multi_node/slack_channel_description/slack_channel_description.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-slack-channel-description description: > Create a UiPath Flow that uses the Slack IS connector to retrieve the @@ -11,6 +14,8 @@ run_limits: turn_timeout: 1200 initial_prompt: | + Build the flow inside a solution of the same name. + Create a UiPath Flow named "SlackChannelDescription" that retrieves the channel description of #office-bellevue and outputs it. diff --git a/tests/tasks/uipath-maestro-flow/multi_node/slack_weather_pipeline/slack_weather_pipeline.yaml b/tests/tasks/uipath-maestro-flow/multi_node/slack_weather_pipeline/slack_weather_pipeline.yaml index be6765dbee..30c16456fe 100644 --- a/tests/tasks/uipath-maestro-flow/multi_node/slack_weather_pipeline/slack_weather_pipeline.yaml +++ b/tests/tasks/uipath-maestro-flow/multi_node/slack_weather_pipeline/slack_weather_pipeline.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-slack-weather-pipeline description: > Multi-step pipeline: read Slack channel description, extract city with a script, @@ -11,11 +14,13 @@ run_limits: turn_timeout: 1200 initial_prompt: | + Build the flow inside a solution of the same name. + Create a flow called "SlackWeatherPipeline". Read the #office-bellevue Slack channel description — it holds the office's full US mailing address (street, suite/unit, city, state, ZIP) — identify the city (the locality, not the street or suite line). Then fetch the current weather for that city from open-meteo. If it's above 60F the flow must output the text in variable `weatherVerdict` 'warm office today', otherwise 'cold office today'. All connections for this task live in Orchestrator folder `Shared/uipath-maestro-flow`. - Validate the flow, then run `uip maestro flow debug` and iterate until it completes without incidents. + Validate the flow, then run it and iterate until it completes without incidents. This run is headless. No user is present and nobody will answer a question or grant an approval, so do not ask, do not pause, and do not wait for input. diff --git a/tests/tasks/uipath-maestro-flow/multi_node/wiki_pageviews/wiki_pageviews.yaml b/tests/tasks/uipath-maestro-flow/multi_node/wiki_pageviews/wiki_pageviews.yaml index c62753b860..95d0b9a3ba 100644 --- a/tests/tasks/uipath-maestro-flow/multi_node/wiki_pageviews/wiki_pageviews.yaml +++ b/tests/tasks/uipath-maestro-flow/multi_node/wiki_pageviews/wiki_pageviews.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-wiki-pageviews description: > Create a UiPath Flow that fetches a range of Wikipedia pageviews, keeps @@ -13,6 +16,8 @@ run_limits: task_timeout: 3000 # 1620s grading + a turn; inherited 1200 could not fit grading initial_prompt: | + Build the flow inside a solution of the same name. + Create a UiPath Flow project named "WikiPageviews" that takes `article` (string) and `date1`, `date2` (strings in `YYYYMMDD` format) as inputs. Fetch the daily view counts in the range from the Wikimedia pageviews API diff --git a/tests/tasks/uipath-maestro-flow/single_node/api_workflow/api_workflow.yaml b/tests/tasks/uipath-maestro-flow/single_node/api_workflow/api_workflow.yaml index 5052621811..3b7b1c2461 100644 --- a/tests/tasks/uipath-maestro-flow/single_node/api_workflow/api_workflow.yaml +++ b/tests/tasks/uipath-maestro-flow/single_node/api_workflow/api_workflow.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-api-workflow description: > Create a UiPath Flow that invokes the name-to-age API workflow with the @@ -10,6 +13,8 @@ run_limits: turn_timeout: 1200 initial_prompt: | + Build the flow inside a solution of the same name. + Create a UiPath Flow project named "NameToAge" that invokes the name-to-age API workflow with the name 'tomasz' and returns his age as an output. diff --git a/tests/tasks/uipath-maestro-flow/single_node/decision/decision.yaml b/tests/tasks/uipath-maestro-flow/single_node/decision/decision.yaml index 33168541ee..5869cf7d6a 100644 --- a/tests/tasks/uipath-maestro-flow/single_node/decision/decision.yaml +++ b/tests/tasks/uipath-maestro-flow/single_node/decision/decision.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-decision description: > Create a UiPath Flow that takes a temperature in Fahrenheit and uses a Decision @@ -12,6 +15,8 @@ run_limits: task_timeout: 2700 # 1260s grading + a turn; inherited 1200 could not fit grading initial_prompt: | + Build the flow inside a solution of the same name. + Create a UiPath Flow project named "TemperatureChecker" that takes a temperature in Fahrenheit as input. If the temperature is greater than 75, the flow should output "warm". Otherwise it should output "cool". diff --git a/tests/tasks/uipath-maestro-flow/single_node/delay/check_delay_flow.py b/tests/tasks/uipath-maestro-flow/single_node/delay/check_delay_flow.py index f93efeaa86..d03db0d942 100644 --- a/tests/tasks/uipath-maestro-flow/single_node/delay/check_delay_flow.py +++ b/tests/tasks/uipath-maestro-flow/single_node/delay/check_delay_flow.py @@ -19,11 +19,14 @@ `sourceNodeId` is the delay node id (outgoing). """ -import glob import json import sys +from pathlib import Path from typing import NoReturn +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) +from _shared.flow_check import find_flow_file # noqa: E402 + NODE_TYPE = "core.logic.delay" VALID_TIMER_TYPES = {"timeDuration", "timeDate"} @@ -33,10 +36,7 @@ def _fail(msg: str) -> NoReturn: def _read_flow() -> dict: - flows = glob.glob("**/DelayDemo*.flow", recursive=True) - if not flows: - _fail("no DelayDemo*.flow found under cwd") - with open(flows[0]) as f: + with open(find_flow_file(flow_glob="DelayDemo*.flow")) as f: return json.load(f) diff --git a/tests/tasks/uipath-maestro-flow/single_node/delay/delay.yaml b/tests/tasks/uipath-maestro-flow/single_node/delay/delay.yaml index 9afa47baff..2c85ae718c 100644 --- a/tests/tasks/uipath-maestro-flow/single_node/delay/delay.yaml +++ b/tests/tasks/uipath-maestro-flow/single_node/delay/delay.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-delay description: > Create a UiPath Flow with a single OOTB Delay node (`core.logic.delay`) @@ -20,16 +23,14 @@ initial_prompt: | - `inputs.timerType` must be `"timeDuration"`. - `inputs.timerPreset` must be a fixed ISO 8601 duration preset such as `"PT15M"` (15 minutes). Do NOT use `"custom"` for this smoke test. - - Set `typeVersion` from the node registry for `core.logic.delay` — - do NOT hardcode a guessed version. + - Set `typeVersion` from the authoritative definition for + `core.logic.delay`. Wire the Delay node so that it has an incoming edge from the trigger and an outgoing edge to the End node (an `edges[]` entry whose `targetNodeId` is the delay node id, and one whose `sourceNodeId` is the delay node id). - Do NOT run flow debug — just validate the flow. - Before starting, follow the documented workflow steps exactly — in particular, the delay plugin's `inputs` shape - (`timerType` + `timerPreset`). + Validate the flow. This run is headless. No user is present and nobody will answer a question or grant an approval, so do not ask, do not pause, and do not wait for input. diff --git a/tests/tasks/uipath-maestro-flow/single_node/file_attachment/file_attachment.yaml b/tests/tasks/uipath-maestro-flow/single_node/file_attachment/file_attachment.yaml index e1b4f7bde8..d8140bd22f 100644 --- a/tests/tasks/uipath-maestro-flow/single_node/file_attachment/file_attachment.yaml +++ b/tests/tasks/uipath-maestro-flow/single_node/file_attachment/file_attachment.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-file-attachment-debug description: > Build a Flow whose trigger exposes a file-typed input variable, read the @@ -21,13 +24,15 @@ run_limits: turn_timeout: 1200 initial_prompt: | + Build the flow inside a solution of the same name. + Create a UiPath Flow project named "AttachmentEcho" that accepts a file as input and echoes the uploaded file's name back as a flow output. Use a file-typed input variable named `inputDoc` and surface the name as an output variable named `fileName`. - Validate the flow, then run it with `uip maestro flow debug`, binding a local - file to the input. The task is not complete until the debug run reports + Validate the flow, then run it with a local file bound to the input. The + task is not complete until the run reports `finalStatus: "Completed"` and the output reflects the attached file's name. Before starting, follow the documented diff --git a/tests/tasks/uipath-maestro-flow/single_node/lowcode_agent/lowcode_agent.yaml b/tests/tasks/uipath-maestro-flow/single_node/lowcode_agent/lowcode_agent.yaml index cefd781716..24dd03db59 100644 --- a/tests/tasks/uipath-maestro-flow/single_node/lowcode_agent/lowcode_agent.yaml +++ b/tests/tasks/uipath-maestro-flow/single_node/lowcode_agent/lowcode_agent.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-lowcode-agent description: > Create a UiPath Flow that wires in the existing CountLetters low-code agent @@ -15,6 +18,8 @@ run_limits: turn_timeout: 1200 initial_prompt: | + Build the flow inside a solution of the same name. + Create a UiPath Flow project named "CountLettersLowCode" that uses the existing CountLetters low-code agent to count the number of r's in 'arrow' and return the answer. The agent already exists — discover it and wire it diff --git a/tests/tasks/uipath-maestro-flow/single_node/openmeteo_weather/openmeteo_weather.yaml b/tests/tasks/uipath-maestro-flow/single_node/openmeteo_weather/openmeteo_weather.yaml index f44cef504c..4a1d1b45ea 100644 --- a/tests/tasks/uipath-maestro-flow/single_node/openmeteo_weather/openmeteo_weather.yaml +++ b/tests/tasks/uipath-maestro-flow/single_node/openmeteo_weather/openmeteo_weather.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-openmeteo-weather description: > End-to-end: build a Flow whose process fetches the CURRENT weather in Bellevue @@ -24,13 +27,15 @@ run_limits: turn_timeout: 1200 initial_prompt: | + Build the flow inside a solution of the same name. + Create a UiPath Flow project named "OpenMeteoWeather". The process must fetch the CURRENT weather in Bellevue using the Open-Meteo connector, then surface the current temperature as a flow output variable. Use the Open-Meteo Integration Service connector. Validate the flow. - The task is not complete until `uip maestro flow validate` passes. + The task is not complete until the flow validates. Do NOT substitute a mock or a manual HTTP request for the connector. diff --git a/tests/tasks/uipath-maestro-flow/single_node/outlook_trigger_inbox/check_outlook_trigger_inbox.py b/tests/tasks/uipath-maestro-flow/single_node/outlook_trigger_inbox/check_outlook_trigger_inbox.py index a385bdaae7..1517a6fda0 100644 --- a/tests/tasks/uipath-maestro-flow/single_node/outlook_trigger_inbox/check_outlook_trigger_inbox.py +++ b/tests/tasks/uipath-maestro-flow/single_node/outlook_trigger_inbox/check_outlook_trigger_inbox.py @@ -4,6 +4,8 @@ Two checks: check_trigger_node Structural — flow contains the email-received trigger. + check_folder_binding Structural — trigger binds both an Outlook connection + and a non-empty MailFolder reference. check_folder_id_fresh Regression — parentFolderId written into the flow is a live MailFolder ID on the currently-bound Outlook connection. Catches "agent resolved-but-reused a @@ -19,11 +21,13 @@ (which may itself be a name). Only counts and lengths. """ -import glob import json -import os import subprocess import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) +from _shared.flow_check import find_flow_file # noqa: E402 CONNECTOR_KEY = "uipath-microsoft-outlook365" TRIGGER_TYPE_MARKER = "uipath.connector.trigger.uipath-microsoft-outlook365.email-received" @@ -118,11 +122,9 @@ def _uip_resources_run(tail_args: list[str]) -> dict: def _read_flow() -> tuple[dict, str]: - flows = glob.glob("**/OutlookTriggerInbox*.flow", recursive=True) - if not flows: - sys.exit("FAIL: no OutlookTriggerInbox*.flow found under cwd") - with open(flows[0]) as f: - return json.load(f), flows[0] + path = find_flow_file(flow_glob="OutlookTriggerInbox*.flow") + with open(path) as f: + return json.load(f), path def _find_test_folder_key() -> str: @@ -192,6 +194,19 @@ def check_trigger_node(): print("OK: Outlook email-received trigger node present") +def check_folder_binding(): + flow, _ = _read_flow() + trigger = _find_trigger_node(flow) + detail = trigger.get("inputs", {}).get("detail", {}) or {} + connection_id = detail.get("connectionId") + parent_folder_id = (detail.get("eventParameters") or {}).get("parentFolderId") + if not connection_id: + sys.exit("FAIL: trigger.inputs.detail.connectionId is missing") + if not parent_folder_id: + sys.exit("FAIL: trigger.inputs.detail.eventParameters.parentFolderId is missing") + print("OK: Outlook connection and MailFolder reference are both bound") + + # ── subcommand: check_folder_id_fresh ────────────────────────────────── def check_folder_id_fresh(): flow, _ = _read_flow() @@ -256,6 +271,7 @@ def check_folder_id_fresh(): DISPATCH = { "check_trigger_node": check_trigger_node, + "check_folder_binding": check_folder_binding, "check_folder_id_fresh": check_folder_id_fresh, } diff --git a/tests/tasks/uipath-maestro-flow/single_node/outlook_trigger_inbox/outlook_trigger_inbox.yaml b/tests/tasks/uipath-maestro-flow/single_node/outlook_trigger_inbox/outlook_trigger_inbox.yaml index 567936b1b4..8291e84eda 100644 --- a/tests/tasks/uipath-maestro-flow/single_node/outlook_trigger_inbox/outlook_trigger_inbox.yaml +++ b/tests/tasks/uipath-maestro-flow/single_node/outlook_trigger_inbox/outlook_trigger_inbox.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-outlook-trigger-inbox description: > Regression test for PR #348: verifies the agent freshly resolves the @@ -79,29 +82,11 @@ success_criteria: weight: 0 pass_threshold: 0 - - type: command_executed - description: "Agent ran the MailFolder resolve against the current --connection-id (PR #348 regression). Outcome is proven by check_folder_id_fresh below, not here" - tool_name: "Bash" - # `[\s\S]` matches across newlines — agents often split `uip` commands with - # backslash line-continuation, which `.*` would miss. - # Verb alternation tolerates both the post-rename verb ("run", current) - # and the legacy "execute" verb that older CLIs in the sandbox PATH may - # still expose (MST-9674). - command_pattern: 'uip\s+is\s+resources\s+(?:run|execute)\s+list[\s\S]+?MailFolder[\s\S]+?--connection-id' - # Narrows, does NOT close, the hole seen 2026-09-03 (lookup 403'd, agent - # wrote the display name "Inbox" into the id field, criterion still scored - # 1.0). `require_success` reads the *wrapper's* exit status - # (codex_agent.py: `result_status = "success" if exit_code == 0`), so a - # batched `bash -lc` script whose MailFolder lookup fails but whose last - # command exits 0 still counts — the same limitation - # tests/tasks/uipath-review/_shared/check_review_cli_provenance.py:13-15 - # documents. It does close the unbatched case, which is the shape the - # 2026-09-03 run took. Ground truth for whether the resolve SUCCEEDED is - # the `check_folder_id_fresh` criterion below: it re-resolves MailFolder on - # the trigger's bound connection and requires the written parentFolderId to - # be a live id there, which no skipped or failed resolve can produce. - require_success: true - min_count: 1 + - type: run_command + description: "Flow carries a MailFolder reference on its bound Outlook connection" + command: "python3 $TASK_DIR/check_outlook_trigger_inbox.py check_folder_binding" + timeout: 30 + expected_exit_code: 0 weight: 2.5 pass_threshold: 1.0 diff --git a/tests/tasks/uipath-maestro-flow/single_node/outlook_trigger_inbox/test_check_outlook_trigger_inbox.py b/tests/tasks/uipath-maestro-flow/single_node/outlook_trigger_inbox/test_check_outlook_trigger_inbox.py index eeffb8063f..4a1b560567 100644 --- a/tests/tasks/uipath-maestro-flow/single_node/outlook_trigger_inbox/test_check_outlook_trigger_inbox.py +++ b/tests/tasks/uipath-maestro-flow/single_node/outlook_trigger_inbox/test_check_outlook_trigger_inbox.py @@ -131,6 +131,32 @@ def fake_run(_args, **_kwargs): checker.check_folder_id_fresh() # must NOT raise (id resolves on the connection) +def test_folder_binding_requires_connection_and_reference(monkeypatch) -> None: + checker = _load_checker() + monkeypatch.setattr( + checker, + "_read_flow", + lambda: ( + { + "nodes": [ + { + "type": checker.TRIGGER_TYPE_MARKER, + "inputs": { + "detail": { + "connectionId": "connection-1", + "eventParameters": {"parentFolderId": "mail-folder-1"}, + } + }, + } + ] + }, + "OutlookTriggerInbox.flow", + ), + ) + + checker.check_folder_binding() + + def test_folder_id_mismatch_fails_with_pr348_signature(monkeypatch) -> None: """If the flow's parentFolderId is not in the live ID set, the checker fails with the PR #348 regression marker (independent of the verb).""" diff --git a/tests/tasks/uipath-maestro-flow/single_node/outlook_waitfor_email/check_outlook_waitfor_email.py b/tests/tasks/uipath-maestro-flow/single_node/outlook_waitfor_email/check_outlook_waitfor_email.py index 0d6f7b397d..b797f13c72 100644 --- a/tests/tasks/uipath-maestro-flow/single_node/outlook_waitfor_email/check_outlook_waitfor_email.py +++ b/tests/tasks/uipath-maestro-flow/single_node/outlook_waitfor_email/check_outlook_waitfor_email.py @@ -24,9 +24,12 @@ MST-8802) cannot satisfy the check on its own. """ -import glob import json import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) +from _shared.flow_check import find_flow_file # noqa: E402 EVENT_MARKERS = ( "uipath.connector.event", @@ -71,10 +74,7 @@ def _filter_trees(detail: dict): def _read_flow() -> dict: - flows = glob.glob("**/OutlookWaitForEmail*.flow", recursive=True) - if not flows: - _fail("no OutlookWaitForEmail*.flow found under cwd") - with open(flows[0]) as f: + with open(find_flow_file(flow_glob="OutlookWaitForEmail*.flow")) as f: return json.load(f) diff --git a/tests/tasks/uipath-maestro-flow/single_node/outlook_waitfor_email/outlook_waitfor_email.yaml b/tests/tasks/uipath-maestro-flow/single_node/outlook_waitfor_email/outlook_waitfor_email.yaml index 760e75f43b..05852962da 100644 --- a/tests/tasks/uipath-maestro-flow/single_node/outlook_waitfor_email/outlook_waitfor_email.yaml +++ b/tests/tasks/uipath-maestro-flow/single_node/outlook_waitfor_email/outlook_waitfor_email.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-outlook-waitfor-email description: > Build-and-validate: a Flow with a manual start trigger, a mid-flow @@ -39,7 +42,7 @@ initial_prompt: | is received in the **Inbox** folder whose SUBJECT CONTAINS the string "TestWaitFor". Then End the flow. - Validate the flow. Do not debug the flow. + Validate the flow. This run is headless. No user is present and nobody will answer a question or grant an approval, so do not ask, do not pause, and do not wait for input. diff --git a/tests/tasks/uipath-maestro-flow/single_node/rpa/rpa.yaml b/tests/tasks/uipath-maestro-flow/single_node/rpa/rpa.yaml index 1a72b3cfa7..10a9dd7c71 100644 --- a/tests/tasks/uipath-maestro-flow/single_node/rpa/rpa.yaml +++ b/tests/tasks/uipath-maestro-flow/single_node/rpa/rpa.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-rpa description: > Create a UiPath Flow that uses the HelpDeskLookup RPA workflow to retrieve @@ -11,6 +14,8 @@ run_limits: task_timeout: 2700 # 1380s grading + a turn; inherited 1200 could not fit grading initial_prompt: | + Build the flow inside a solution of the same name. + Create a UiPath Flow project named "HelpDeskTicketTitle" that uses the HelpDeskLookup RPA workflow to retrieve the title for ticket 123 and return it as an output. diff --git a/tests/tasks/uipath-maestro-flow/single_node/subflow/subflow.yaml b/tests/tasks/uipath-maestro-flow/single_node/subflow/subflow.yaml index a26a12b1c8..5c6fdf97f7 100644 --- a/tests/tasks/uipath-maestro-flow/single_node/subflow/subflow.yaml +++ b/tests/tasks/uipath-maestro-flow/single_node/subflow/subflow.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-subflow description: > Create a UiPath Flow that uses a Subflow node to encapsulate string-reversal @@ -12,6 +15,8 @@ run_limits: turn_timeout: 1200 initial_prompt: | + Build the flow inside a solution of the same name. + Create a UiPath Flow project named "SubflowDemo" that takes a string input called "text" and reverses it. The string reversal logic should be encapsulated inside a subflow rather than done directly in the main flow. diff --git a/tests/tasks/uipath-maestro-flow/single_node/switch/switch.yaml b/tests/tasks/uipath-maestro-flow/single_node/switch/switch.yaml index 04f02ca617..86e9f7457b 100644 --- a/tests/tasks/uipath-maestro-flow/single_node/switch/switch.yaml +++ b/tests/tasks/uipath-maestro-flow/single_node/switch/switch.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-switch description: > Create a UiPath Flow that takes a quarter number (1-4) and uses a Switch node @@ -10,6 +13,8 @@ run_limits: turn_timeout: 1200 initial_prompt: | + Build the flow inside a solution of the same name. + Create a UiPath Flow project named "SeasonLookup" that takes a quarter number (1, 2, 3, or 4) as input and returns the corresponding season name: - 1 -> "Spring" diff --git a/tests/tasks/uipath-maestro-flow/single_node/terminate/terminate.yaml b/tests/tasks/uipath-maestro-flow/single_node/terminate/terminate.yaml index 1f058d0724..66ba80c4eb 100644 --- a/tests/tasks/uipath-maestro-flow/single_node/terminate/terminate.yaml +++ b/tests/tasks/uipath-maestro-flow/single_node/terminate/terminate.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-terminate description: > Create a UiPath Flow with two parallel branches from the trigger. One branch @@ -12,6 +15,8 @@ run_limits: turn_timeout: 1200 initial_prompt: | + Build the flow inside a solution of the same name. + Create a UiPath Flow project named "TerminateParallel" with two parallel branches from the trigger: diff --git a/tests/tasks/uipath-maestro-flow/single_node/test_transform_output_envelopes.py b/tests/tasks/uipath-maestro-flow/single_node/test_transform_output_envelopes.py new file mode 100644 index 0000000000..9e8ff2ec35 --- /dev/null +++ b/tests/tasks/uipath-maestro-flow/single_node/test_transform_output_envelopes.py @@ -0,0 +1,108 @@ +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path + +import pytest + +SINGLE_NODE = Path(__file__).resolve().parent + + +@pytest.mark.parametrize( + ("task", "flow_name", "node_type", "inputs"), + [ + ( + "transform_map", + "TransformMapDemo.flow", + "core.action.transform.map", + { + "collection": "$vars.people", + "operations": [ + { + "type": "map", + "config": { + "mappings": [{"field": "name", "transformation": "uppercase"}] + }, + } + ], + }, + ), + ( + "transform_filter", + "TransformFilterDemo.flow", + "core.action.transform.filter", + { + "collection": "$vars.items", + "operations": [ + { + "type": "filter", + "config": { + "filters": [ + {"field": "amount", "condition": "greater_equal", "value": 100} + ] + }, + } + ], + }, + ), + ( + "transform_group_by", + "TransformGroupByDemo.flow", + "core.action.transform.group-by", + { + "collection": "$vars.employees", + "operations": [ + { + "type": "groupBy", + "config": { + "groupByField": "department", + "aggregations": [{"operation": "count", "alias": "headcount"}], + }, + } + ], + }, + ), + ], +) +def test_transform_checkers_accept_sdk_literal_output_envelopes( + tmp_path: Path, task: str, flow_name: str, node_type: str, inputs: dict +) -> None: + flow = { + "nodes": [ + { + "id": "transform", + "type": node_type, + "typeVersion": "1.0.0", + "inputs": inputs, + "outputs": { + "output": { + "source": { + "type": "literal", + "expression": "=result.response", + "fieldType": "string", + } + }, + "error": { + "source": { + "type": "literal", + "expression": "=Error", + "fieldType": "string", + } + }, + }, + } + ] + } + (tmp_path / flow_name).write_text(json.dumps(flow)) + + result = subprocess.run( + [sys.executable, str(SINGLE_NODE / task / f"check_{task}_flow.py")], + cwd=tmp_path, + capture_output=True, + text=True, + check=False, + ) + + assert result.returncode == 0, result.stdout + result.stderr diff --git a/tests/tasks/uipath-maestro-flow/single_node/transform_filter/check_transform_filter_flow.py b/tests/tasks/uipath-maestro-flow/single_node/transform_filter/check_transform_filter_flow.py index 1faffe0118..ca123e2e63 100644 --- a/tests/tasks/uipath-maestro-flow/single_node/transform_filter/check_transform_filter_flow.py +++ b/tests/tasks/uipath-maestro-flow/single_node/transform_filter/check_transform_filter_flow.py @@ -23,11 +23,15 @@ NOT pin a specific version). """ -import glob import json import sys +from pathlib import Path from typing import NoReturn +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) +from _shared.advisory_flow_utils import unwrap # noqa: E402 +from _shared.flow_check import find_flow_file # noqa: E402 + NODE_TYPE = "core.action.transform.filter" EXPECTED_OUTPUT_SOURCE = "=result.response" EXPECTED_ERROR_SOURCE = "=Error" @@ -55,10 +59,7 @@ def _fail(msg: str) -> NoReturn: def _read_flow() -> dict: - flows = glob.glob("**/TransformFilterDemo*.flow", recursive=True) - if not flows: - _fail("no TransformFilterDemo*.flow found under cwd") - with open(flows[0]) as f: + with open(find_flow_file(flow_glob="TransformFilterDemo*.flow")) as f: return json.load(f) @@ -169,12 +170,12 @@ def _check_operations(inputs: dict) -> None: def _check_output_sources(node: dict) -> None: outputs = node.get("outputs") or {} - out_src = ((outputs.get("output")) or {}).get("source") + out_src = unwrap(((outputs.get("output")) or {}).get("source")) if out_src != EXPECTED_OUTPUT_SOURCE: _fail( f"outputs.output.source={out_src!r}; must be {EXPECTED_OUTPUT_SOURCE!r}" ) - err_src = ((outputs.get("error")) or {}).get("source") + err_src = unwrap(((outputs.get("error")) or {}).get("source")) if err_src != EXPECTED_ERROR_SOURCE: _fail( f"outputs.error.source={err_src!r}; must be {EXPECTED_ERROR_SOURCE!r}" diff --git a/tests/tasks/uipath-maestro-flow/single_node/transform_filter/transform_filter.yaml b/tests/tasks/uipath-maestro-flow/single_node/transform_filter/transform_filter.yaml index f435e4adfe..1520113754 100644 --- a/tests/tasks/uipath-maestro-flow/single_node/transform_filter/transform_filter.yaml +++ b/tests/tasks/uipath-maestro-flow/single_node/transform_filter/transform_filter.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-transform-filter description: > Create a UiPath Flow that uses a dedicated `core.action.transform.filter` @@ -34,12 +37,10 @@ initial_prompt: | templates. Use the exact condition name `greater_equal`. - Wire the node `outputs.output.source` to `=result.response` and `outputs.error.source` to `=Error`. - - Set `typeVersion` from the node registry for - `core.action.transform.filter` — do not hardcode a guessed value. + - Set `typeVersion` from the authoritative definition for + `core.action.transform.filter`. - Do NOT run flow debug — just validate the flow. - Before starting, follow the documented workflow steps exactly — in particular the transform plugin's filter - node shape (`operations[].type == "filter"` with `config.filters`). + Validate the flow. This run is headless. No user is present and nobody will answer a question or grant an approval, so do not ask, do not pause, and do not wait for input. diff --git a/tests/tasks/uipath-maestro-flow/single_node/transform_group_by/check_transform_group_by_flow.py b/tests/tasks/uipath-maestro-flow/single_node/transform_group_by/check_transform_group_by_flow.py index 1719206f60..ae03d8c123 100644 --- a/tests/tasks/uipath-maestro-flow/single_node/transform_group_by/check_transform_group_by_flow.py +++ b/tests/tasks/uipath-maestro-flow/single_node/transform_group_by/check_transform_group_by_flow.py @@ -21,11 +21,15 @@ deliberately NOT pinned to a specific value here). """ -import glob import json import sys +from pathlib import Path from typing import NoReturn +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) +from _shared.advisory_flow_utils import unwrap # noqa: E402 +from _shared.flow_check import find_flow_file # noqa: E402 + NODE_TYPE = "core.action.transform.group-by" EXPECTED_OUTPUT_SOURCE = "=result.response" EXPECTED_ERROR_SOURCE = "=Error" @@ -36,10 +40,7 @@ def _fail(msg: str) -> NoReturn: def _read_flow() -> dict: - flows = glob.glob("**/TransformGroupByDemo*.flow", recursive=True) - if not flows: - _fail("no TransformGroupByDemo*.flow found under cwd") - with open(flows[0]) as f: + with open(find_flow_file(flow_glob="TransformGroupByDemo*.flow")) as f: return json.load(f) @@ -114,12 +115,12 @@ def _check_operations(inputs: dict) -> None: def _check_ports(node: dict) -> None: outputs = node.get("outputs") or {} - out_src = ((outputs.get("output") or {}).get("source")) + out_src = unwrap((outputs.get("output") or {}).get("source")) if out_src != EXPECTED_OUTPUT_SOURCE: _fail( f"outputs.output.source={out_src!r}; must be {EXPECTED_OUTPUT_SOURCE!r}." ) - err_src = ((outputs.get("error") or {}).get("source")) + err_src = unwrap((outputs.get("error") or {}).get("source")) if err_src != EXPECTED_ERROR_SOURCE: _fail( f"outputs.error.source={err_src!r}; must be {EXPECTED_ERROR_SOURCE!r}." diff --git a/tests/tasks/uipath-maestro-flow/single_node/transform_group_by/transform_group_by.yaml b/tests/tasks/uipath-maestro-flow/single_node/transform_group_by/transform_group_by.yaml index 4d8ba07bd9..ee479c981a 100644 --- a/tests/tasks/uipath-maestro-flow/single_node/transform_group_by/transform_group_by.yaml +++ b/tests/tasks/uipath-maestro-flow/single_node/transform_group_by/transform_group_by.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-transform-group-by description: > Create a UiPath Flow with a single Group By transform node @@ -35,12 +38,10 @@ initial_prompt: | `average`) and a non-empty `alias` for the output field. - The node's `outputs.output.source` must be `=result.response` and `outputs.error.source` must be `=Error`. - - Set the node's `typeVersion` from the node registry for - `core.action.transform.group-by` — do NOT hardcode a guessed version. + - Set the node's `typeVersion` from the authoritative definition for + `core.action.transform.group-by`. - Validate the flow. Do NOT run flow debug — just validate. - Before starting, follow the documented workflow steps exactly — in particular, the transform plugin's Group By JSON - shape and the collection-input contract. + Validate the flow. This run is headless. No user is present and nobody will answer a question or grant an approval, so do not ask, do not pause, and do not wait for input. diff --git a/tests/tasks/uipath-maestro-flow/single_node/transform_map/check_transform_map_flow.py b/tests/tasks/uipath-maestro-flow/single_node/transform_map/check_transform_map_flow.py index 2239a03655..cc377c09e7 100644 --- a/tests/tasks/uipath-maestro-flow/single_node/transform_map/check_transform_map_flow.py +++ b/tests/tasks/uipath-maestro-flow/single_node/transform_map/check_transform_map_flow.py @@ -21,11 +21,15 @@ pin a specific version string here). """ -import glob import json import sys +from pathlib import Path from typing import NoReturn +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) +from _shared.advisory_flow_utils import unwrap # noqa: E402 +from _shared.flow_check import find_flow_file # noqa: E402 + NODE_TYPE = "core.action.transform.map" EXPECTED_OUTPUT_SOURCE = "=result.response" EXPECTED_ERROR_SOURCE = "=Error" @@ -36,10 +40,7 @@ def _fail(msg: str) -> NoReturn: def _read_flow() -> dict: - flows = glob.glob("**/TransformMapDemo*.flow", recursive=True) - if not flows: - _fail("no TransformMapDemo*.flow found under cwd") - with open(flows[0]) as f: + with open(find_flow_file(flow_glob="TransformMapDemo*.flow")) as f: return json.load(f) @@ -105,12 +106,12 @@ def _check_operations(inputs: dict) -> None: def _check_output_sources(node: dict) -> None: outputs = node.get("outputs") or {} - out_src = ((outputs.get("output")) or {}).get("source") + out_src = unwrap(((outputs.get("output")) or {}).get("source")) if out_src != EXPECTED_OUTPUT_SOURCE: _fail( f"outputs.output.source={out_src!r}; must be {EXPECTED_OUTPUT_SOURCE!r}." ) - err_src = ((outputs.get("error")) or {}).get("source") + err_src = unwrap(((outputs.get("error")) or {}).get("source")) if err_src != EXPECTED_ERROR_SOURCE: _fail( f"outputs.error.source={err_src!r}; must be {EXPECTED_ERROR_SOURCE!r}." diff --git a/tests/tasks/uipath-maestro-flow/single_node/transform_map/transform_map.yaml b/tests/tasks/uipath-maestro-flow/single_node/transform_map/transform_map.yaml index 8d79e6c4c3..b85515f068 100644 --- a/tests/tasks/uipath-maestro-flow/single_node/transform_map/transform_map.yaml +++ b/tests/tasks/uipath-maestro-flow/single_node/transform_map/transform_map.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-transform-map description: > Smoke test: agent builds a pure-OOTB UiPath Flow with a single @@ -32,13 +35,10 @@ initial_prompt: | `transformation: "uppercase"`. - The node `outputs.output.source` must be `=result.response` and `outputs.error.source` must be `=Error`. - - Set the node's `typeVersion` from the node registry for - `core.action.transform.map` — do NOT hardcode a guessed version (the - skill's workflow teaches how to read the registry). + - Set the node's `typeVersion` from the authoritative definition for + `core.action.transform.map`. - Do NOT run flow debug — just validate the flow. - Before starting, follow the documented workflow steps exactly — in particular, the transform plugin's Map JSON - shape and the `collection` path contract. + Validate the flow. This run is headless. No user is present and nobody will answer a question or grant an approval, so do not ask, do not pause, and do not wait for input. diff --git a/tests/tasks/uipath-maestro-flow/smoke/check_merge_parallel_sync_flow.py b/tests/tasks/uipath-maestro-flow/smoke/check_merge_parallel_sync_flow.py index 55f221bece..00cb1c4a0c 100644 --- a/tests/tasks/uipath-maestro-flow/smoke/check_merge_parallel_sync_flow.py +++ b/tests/tasks/uipath-maestro-flow/smoke/check_merge_parallel_sync_flow.py @@ -22,12 +22,15 @@ source node that feeds the merge itself has an incoming edge. """ -import glob import json import sys from collections import Counter +from pathlib import Path from typing import NoReturn +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from _shared.flow_check import find_flow_file # noqa: E402 + NODE_TYPE = "core.logic.merge" @@ -36,14 +39,7 @@ def _fail(msg: str) -> NoReturn: def _read_flow() -> dict: - flows = sorted(glob.glob("**/ParallelSync*.flow", recursive=True)) - if not flows: - _fail("no ParallelSync*.flow found under cwd") - # Prefer the canonical project path; fall back to the first match so the - # read is deterministic even if stray *.flow files exist. - canonical = "ParallelSync/ParallelSync/ParallelSync.flow" - path = canonical if canonical in flows else flows[0] - with open(path) as f: + with open(find_flow_file(flow_glob="ParallelSync*.flow")) as f: return json.load(f) diff --git a/tests/tasks/uipath-maestro-flow/smoke/check_scheduled_trigger_flow.py b/tests/tasks/uipath-maestro-flow/smoke/check_scheduled_trigger_flow.py index 20735c65e6..4d6e4c23bc 100644 --- a/tests/tasks/uipath-maestro-flow/smoke/check_scheduled_trigger_flow.py +++ b/tests/tasks/uipath-maestro-flow/smoke/check_scheduled_trigger_flow.py @@ -34,12 +34,15 @@ - zero `definitions[]` entries with `nodeType == "core.trigger.manual"`. """ -import glob import json import re import sys +from pathlib import Path from typing import NoReturn +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from _shared.flow_check import find_flow_file # noqa: E402 + SCHEDULED = "core.trigger.scheduled" MANUAL = "core.trigger.manual" TRIGGER_PREFIX = "core.trigger." @@ -60,10 +63,7 @@ def _fail(msg: str) -> NoReturn: def _read_flow() -> dict: - flows = glob.glob("**/ScheduledReport*.flow", recursive=True) - if not flows: - _fail("no ScheduledReport*.flow found under cwd") - with open(flows[0]) as f: + with open(find_flow_file(flow_glob="ScheduledReport*.flow")) as f: return json.load(f) diff --git a/tests/tasks/uipath-maestro-flow/smoke/init_validate.yaml b/tests/tasks/uipath-maestro-flow/smoke/init_validate.yaml index 07cd098222..4479119fc9 100644 --- a/tests/tasks/uipath-maestro-flow/smoke/init_validate.yaml +++ b/tests/tasks/uipath-maestro-flow/smoke/init_validate.yaml @@ -1,8 +1,10 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-init-validate description: > - Skill-guided evaluation: agent uses the uipath-maestro-flow skill to create - a new UiPath Flow project inside a solution and validate it. Tests whether - the skill teaches the correct solution-first workflow and CLI usage. + Skill-guided evaluation: agent creates a new UiPath Flow project and + validates it. Tests whether the skill produces a valid Flow artifact. tags: [uipath-maestro-flow, smoke, lifecycle:generate] run_limits: @@ -12,22 +14,6 @@ initial_prompt: | Create a new UiPath Flow project called "WeatherAlert" and make sure it validates successfully. - A Flow project MUST be created - inside a solution: - 1. Create the solution first. - 2. Create the Flow project inside that solution. - 3. Confirm the flow project is registered in the parent solution. - - The correct flow-file path is: - WeatherAlert/WeatherAlert/WeatherAlert.flow - - The task is NOT complete until `uip maestro flow validate` has passed for - that exact file path. - - Important: - - The `uip` CLI is already available in the environment. - - Validate locally with `uip maestro flow validate`. - This run is headless. No user is present and nobody will answer a question or grant an approval, so do not ask, do not pause, and do not wait for input. Complete the task in one pass: take the best available option and supply the @@ -41,18 +27,18 @@ initial_prompt: | the task take precedence over this paragraph. success_criteria: - type: command_executed - description: "Agent created a solution with uip solution init (or legacy new)" + description: "Advisory: live-v1 agent created a solution" tool_name: "Bash" command_pattern: '(uip|\$UIP)\s+solution\s+(init|new)' min_count: 1 weight: 1.5 pass_threshold: 0.0 # advisory: init auto-scaffolds the parent solution (CLI #2470) - - type: command_executed - description: "Agent initialized a Flow project with uip maestro flow init" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+(maestro\s+)?flow\s+init' - min_count: 1 + - type: run_command + description: "Initialized flow has the default manual trigger" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py --flow-name WeatherAlert 'core.trigger.manual'" + timeout: 30 + expected_exit_code: 0 weight: 1.5 pass_threshold: 1.0 @@ -64,10 +50,10 @@ success_criteria: weight: 1.5 pass_threshold: 1.0 - # Outcome-based check (auto-registration via `flow init` OR explicit - # `uip solution project add` both populate Projects[] in the .uipx). + # Solution registration is a live-v1 layout detail. Keep it visible without + # gating the SDK arm, whose native output is a root-level .flow artifact. - type: json_check - description: "Flow project is registered in the parent solution .uipx" + description: "Advisory: live-v1 Flow project is registered in the parent solution .uipx" path: "WeatherAlert/WeatherAlert.uipx" assertions: - expression: "length(Projects)" @@ -77,11 +63,12 @@ success_criteria: operator: "gte" expected: 1 weight: 1.0 - pass_threshold: 1.0 + pass_threshold: 0.0 - - type: file_exists - description: "Flow file was created inside the solution" - path: "WeatherAlert/WeatherAlert/WeatherAlert.flow" + - type: run_command + description: "Flow file was created in a supported authoring layout" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py --flow-name WeatherAlert" + timeout: 30 + expected_exit_code: 0 weight: 1.5 pass_threshold: 1.0 - diff --git a/tests/tasks/uipath-maestro-flow/smoke/inline_agent_robust.yaml b/tests/tasks/uipath-maestro-flow/smoke/inline_agent_robust.yaml index 0d95df39d1..d998d20e33 100644 --- a/tests/tasks/uipath-maestro-flow/smoke/inline_agent_robust.yaml +++ b/tests/tasks/uipath-maestro-flow/smoke/inline_agent_robust.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-inline-agent-robust description: > Skill-guided evaluation: agent uses the uipath-maestro-flow skill to build a @@ -12,26 +15,17 @@ run_limits: max_turns: 90 initial_prompt: | - Create a new UiPath Flow project called "EmailTriage" (inside a solution) and + Create a new UiPath Flow project called "EmailTriage" and add an inline agent node that classifies an inbound support email into a category and priority. - A Flow project MUST be created - inside a solution, so the inline agent lands at: - EmailTriage/EmailTriage//agent.json - Configure the inline agent to a production standard, not the scaffold default: - Pick an appropriate current model (do not keep the scaffold default). - Write a real system prompt for the classification task (no placeholder). - Declare a typed outputSchema (e.g. category, priority) — not a bare `content` string. - The task is NOT complete until `uip maestro flow validate` passes for - EmailTriage/EmailTriage/EmailTriage.flow. - - Important: - - The `uip` CLI is already available in the environment. - - Validate locally with `uip maestro flow validate`. + Validate the flow. This run is headless. No user is present and nobody will answer a question or grant an approval, so do not ask, do not pause, and do not wait for input. @@ -45,33 +39,35 @@ initial_prompt: | decision, assumption, and blocked step in your final response. Instructions in the task take precedence over this paragraph. success_criteria: - - type: command_executed - description: "Agent scaffolded the inline agent with uip agent init --inline-in-flow" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+agent\s+init\s+.*--inline-in-flow' - min_count: 1 + - type: run_command + description: "Inline agent sidecar was created" + command: 'python3 "$SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/check_inline_agent.py" --check exists "**/agent.json"' + timeout: 15 + expected_exit_code: 0 weight: 1.5 pass_threshold: 1.0 - - type: command_executed - description: "Agent discovered available models with uip agent model list" - tool_name: "Bash" - command_pattern: '(uip|\$UIP)\s+agent\s+model\s+list' - min_count: 1 + - type: run_command + description: "Inline agent uses an explicitly selected current model" + command: 'python3 "$SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/check_inline_agent.py" --check model "**/agent.json"' + timeout: 15 + expected_exit_code: 0 weight: 1.0 pass_threshold: 1.0 - - type: file_exists - description: "Flow file was created inside the solution" - path: "EmailTriage/EmailTriage/EmailTriage.flow" + - type: run_command + description: "Flow file was created in a supported authoring layout" + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py --flow-name EmailTriage" + timeout: 30 + expected_exit_code: 0 weight: 1.5 pass_threshold: 1.0 - - type: file_contains + - type: run_command description: "Flow contains the inline agent node type" - path: "EmailTriage/EmailTriage/EmailTriage.flow" - includes: - - '"uipath.agent.autonomous"' + command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/flow_contains.py --flow-name EmailTriage 'uipath.agent.autonomous'" + timeout: 30 + expected_exit_code: 0 weight: 2.0 pass_threshold: 1.0 @@ -79,11 +75,11 @@ success_criteria: # knowable — the shared checker globs it (excluding generated .agent-builder/) # and asserts the three production-quality properties the skill now teaches: # model overridden off gpt-4o, real (non-placeholder) system prompt, and a - # typed outputSchema beyond bare `content`. $TASK_DIR is this YAML's dir; - # _shared is its sibling. + # typed outputSchema beyond bare `content`. Use the mounted checker repository + # because preview freezes only this task directory at $TASK_DIR. - type: run_command description: "Inline agent.json: model overridden, real system prompt, typed outputSchema" - command: 'python3 "$SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/check_inline_agent.py" "EmailTriage/EmailTriage/*/agent.json"' + command: 'python3 "$SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/check_inline_agent.py" "**/agent.json"' timeout: 15 expected_exit_code: 0 weight: 3.0 diff --git a/tests/tasks/uipath-maestro-flow/smoke/merge_parallel_sync.yaml b/tests/tasks/uipath-maestro-flow/smoke/merge_parallel_sync.yaml index e1cd17fada..6c27965f9a 100644 --- a/tests/tasks/uipath-maestro-flow/smoke/merge_parallel_sync.yaml +++ b/tests/tasks/uipath-maestro-flow/smoke/merge_parallel_sync.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-merge-parallel-sync description: > Build a UiPath Flow with two parallel branches that fork from the trigger and @@ -17,12 +20,6 @@ initial_prompt: | Create a UiPath Flow project named "ParallelSync" that runs two branches in parallel and synchronizes them with a Merge node before finishing. - A Flow project MUST live inside a solution: - 1. Create the solution first (`uip solution init`). - 2. Create the Flow project inside it (`uip maestro flow init`). The correct - flow-file path is: - ParallelSync/ParallelSync/ParallelSync.flow - Build this exact fork/join topology (the default manual trigger is fine — keep it as the start node and use it as the fork): @@ -38,7 +35,7 @@ initial_prompt: | `display` object with a `label` — validate rejects a node without it. - Wire these FIVE edges. Every edge needs both `sourcePort` and `targetPort` (validate rejects an edge missing either). Use the correct - port ids per the registry/skill: + port ids from the authoritative node definitions: - trigger `output` → branchA `input` - trigger `output` → branchB `input` (this fork = trigger has 2 outgoing edges) - branchA `success` → merge `input` @@ -46,21 +43,11 @@ initial_prompt: | - merge `output` → end `input` The two edges into the merge must have `targetNodeId` = the merge node id; the edge out of the merge must have `sourceNodeId` = the merge node id. - - Set every node's `typeVersion` from the registry — do NOT hardcode guessed - versions (`uip maestro flow registry get core.logic.merge --output json`, - etc.). Append each node type's definition to `definitions[]` (the merge - definition carries `model.type: "bpmn:ParallelGateway"`). - - The task is NOT complete until `uip maestro flow validate` passes for that - exact flow-file path. - - Important: - - The `uip` CLI is already available; use `--output json` on uip commands. - - After wiring, you may run `uip maestro flow format ` to normalize - layout/variables, then validate with `uip maestro flow validate` — do NOT - run `uip maestro flow debug` (the parallel join only synchronizes in a - live engine). + - Set every node's `typeVersion` from its authoritative node definition. + Include each node type's definition in `definitions[]`; the Merge + definition carries `model.type: "bpmn:ParallelGateway"`. + Validate the flow. This run is headless. No user is present and nobody will answer a question or grant an approval, so do not ask, do not pause, and do not wait for input. Complete the task in one pass: take the best available option and supply the diff --git a/tests/tasks/uipath-maestro-flow/smoke/registry_discovery.yaml b/tests/tasks/uipath-maestro-flow/smoke/registry_discovery.yaml index 4fe39bdeaf..ef87d47738 100644 --- a/tests/tasks/uipath-maestro-flow/smoke/registry_discovery.yaml +++ b/tests/tasks/uipath-maestro-flow/smoke/registry_discovery.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-registry-discovery description: > Skill-guided evaluation: agent uses the uipath-maestro-flow skill to explore @@ -10,18 +13,10 @@ run_limits: max_turns: 28 initial_prompt: | - I want to build a UiPath Flow that makes an HTTP request and processes the - response with a script. Before building anything, explore what node types - are available in the Flow registry and find the right ones for this task. - - Important: - - The `uip` CLI is already available in the environment. - - Start by refreshing the registry cache with `uip maestro flow registry pull --output json`. - - After list/search identifies the candidates, inspect the exact schemas with - `uip maestro flow registry get core.action.http --output json` and - `uip maestro flow registry get core.action.script --output json`. - - Use `--output json` on all uip commands. - - Do not build the flow — just explore. + Explore the available Flow authoring capabilities for a flow that makes an + HTTP request and processes the response with a script. Identify the exact + HTTP and Script node types and inspect their input and output schemas. + Do not build the flow; report what you find. This run is headless. No user is present and nobody will answer a question or grant an approval, so do not ask, do not pause, and do not wait for input. @@ -35,34 +30,36 @@ initial_prompt: | decision, assumption, and blocked step in your final response. Instructions in the task take precedence over this paragraph. success_criteria: + # This task deliberately produces no artifact. Under ratified ruling D-2, + # v1-only registry telemetry remains report-only rather than gating the SDK arm. - type: command_executed - description: "Agent pulled registry data" + description: "Advisory: live-v1 agent refreshed registry data" tool_name: "Bash" command_pattern: 'uip\s+(maestro\s+)?flow\s+registry\s+pull' min_count: 1 weight: 1.0 - pass_threshold: 1.0 + pass_threshold: 0.0 - type: command_executed - description: "Agent listed or searched registry nodes" + description: "Advisory: live-v1 agent listed or searched registry nodes" tool_name: "Bash" command_pattern: 'uip\s+(maestro\s+)?flow\s+registry\s+(list|search)' min_count: 1 weight: 1.5 - pass_threshold: 1.0 + pass_threshold: 0.0 - type: command_executed - description: "Agent retrieved details on the HTTP node type via registry get" + description: "Advisory: live-v1 agent retrieved details on the HTTP node type" tool_name: "Bash" command_pattern: 'uip\s+(maestro\s+)?flow\s+registry\s+get\s+.*core\.action\.http' min_count: 1 weight: 1.5 - pass_threshold: 1.0 + pass_threshold: 0.0 - type: command_executed - description: "Agent retrieved details on the script node type via registry get" + description: "Advisory: live-v1 agent retrieved details on the Script node type" tool_name: "Bash" command_pattern: 'uip\s+(maestro\s+)?flow\s+registry\s+get\s+.*core\.action\.script' min_count: 1 weight: 1.5 - pass_threshold: 1.0 + pass_threshold: 0.0 diff --git a/tests/tasks/uipath-maestro-flow/smoke/scheduled_trigger.yaml b/tests/tasks/uipath-maestro-flow/smoke/scheduled_trigger.yaml index 9da12584a7..5a0d549e2d 100644 --- a/tests/tasks/uipath-maestro-flow/smoke/scheduled_trigger.yaml +++ b/tests/tasks/uipath-maestro-flow/smoke/scheduled_trigger.yaml @@ -1,3 +1,6 @@ +# Same-ground alignment expansion plan (Tao, 2026-08-21): loop-neutral +# v1-base prompt, outcome-graded criteria at preserved weights, and +# arm-agnostic artifact discovery. task_id: skill-flow-scheduled-trigger description: > Scaffold a UiPath Flow whose start node is a Scheduled Trigger @@ -17,38 +20,22 @@ initial_prompt: | Create a UiPath Flow project named "ScheduledReport" whose flow is started by a **scheduled trigger** that runs every hour. - A Flow project MUST live inside a solution: - 1. Create the solution first (`uip solution init`). - 2. Create the Flow project inside it (`uip maestro flow init`). The correct - flow-file path is: - ScheduledReport/ScheduledReport/ScheduledReport.flow - - `uip maestro flow init` scaffolds the flow with a default **manual** trigger - (`core.trigger.manual`). REPLACE that manual trigger with a **scheduled** - trigger so the flow's start node is a scheduled trigger: + Replace the default **manual** trigger (`core.trigger.manual`) with a + **scheduled** trigger so the flow's start node is a scheduled trigger: - The start node `type` must be `core.trigger.scheduled`. - Keep the existing `entryPointId` from the manual trigger's `inputs`. - `inputs.timerType` must be `"timeCycle"`. - `inputs.timerPreset` must be `"R/PT1H"` (every hour — an ISO 8601 repeating interval). Do NOT use `"custom"` for this smoke test. - - Set the node `typeVersion` from the registry for `core.trigger.scheduled` - — do NOT hardcode a guessed version - (`uip maestro flow registry get core.trigger.scheduled --output json`). + - Set the node `typeVersion` from its authoritative node definition. - Remove the `core.trigger.manual` node AND its `definitions[]` entry, and - append the `core.trigger.scheduled` definition from the registry get above + include the authoritative `core.trigger.scheduled` definition (it carries `model.type: "bpmn:StartEvent"` and `model.eventDefinition: "bpmn:TimerEventDefinition"`). A flow must have exactly one trigger. - The task is NOT complete until `uip maestro flow validate` passes for that - exact flow-file path. - - Important: - - The `uip` CLI is already available; use `--output json` on uip commands. - - Validate locally with `uip maestro flow validate` — do NOT run - `uip maestro flow debug` (the schedule fires only in a live engine). - + Validate the flow. This run is headless. No user is present and nobody will answer a question or grant an approval, so do not ask, do not pause, and do not wait for input. Complete the task in one pass: take the best available option and supply the diff --git a/tests/tasks/uipath-maestro-flow/voice/voice_inbound_call.yaml b/tests/tasks/uipath-maestro-flow/voice/voice_inbound_call.yaml index 22cb4636cd..5a05f96b83 100644 --- a/tests/tasks/uipath-maestro-flow/voice/voice_inbound_call.yaml +++ b/tests/tasks/uipath-maestro-flow/voice/voice_inbound_call.yaml @@ -22,9 +22,10 @@ run_limits: # that rejects the run); the outbound task grades the same pattern for the # harder reason that a debug run there places a real call. initial_prompt: | - Build a UiPath Flow project named "SupportLine" that answers incoming phone - calls to our support line, has an AI agent hold the conversation with the - caller, and hangs up when the conversation is over. + Build a UiPath Flow project named "SupportLine" inside a solution of the same + name. It answers incoming phone calls to our support line, has an AI agent + hold the conversation with the caller, and hangs up when the conversation is + over. This sandbox has no UiPath tenant and is not logged in, so do NOT run anything that needs a live connection, and do not pack, publish, or deploy. @@ -53,7 +54,9 @@ success_criteria: skill_name: "uipath-maestro-flow" expected_skill: "uipath-maestro-flow" weight: 2.0 - pass_threshold: 1.0 + # Advisory: tool telemetry differs across authoring loops; the functional + # criteria below grade whether the skill's contract was actually followed. + pass_threshold: 0 - type: run_command description: "uip maestro flow validate passes on the generated flow" diff --git a/tests/tasks/uipath-maestro-flow/voice/voice_outbound_call.yaml b/tests/tasks/uipath-maestro-flow/voice/voice_outbound_call.yaml index 6fd7a95913..e84fb678f3 100644 --- a/tests/tasks/uipath-maestro-flow/voice/voice_outbound_call.yaml +++ b/tests/tasks/uipath-maestro-flow/voice/voice_outbound_call.yaml @@ -39,8 +39,9 @@ initial_prompt: | This sandbox has no UiPath tenant and is not logged in, so: - Use placeholder E.164 numbers for the calling and destination numbers — do not look up or provision real ones. - - Do NOT run anything that needs a live connection, and do NOT pack, deploy, - or place a real call. In particular, do not run `uip maestro flow debug`. + - Keep the work fully offline: stop after local validation without packing, + deploying, or placing a real call. `uip maestro flow debug` is outside this + task because it can place a real call. The task is NOT complete until `uip maestro flow validate` passes on the generated flow.