From 36dbb2e9f1ff73e8b8adb9f31bdd1fb00ecd6b5f Mon Sep 17 00:00:00 2001 From: Bai Li Date: Mon, 31 Aug 2026 14:42:49 -0700 Subject: [PATCH 1/9] feat(ci): run coder-eval through the published action and publish an expiring evalboard link Three things, all in the Linux job of the dispatch workflow. **Dogfood the action.** `UiPath/coder_eval@v0` had no real consumers: it was exercised only by its own repo's self-test, so a bad release would surface there rather than in someone's workflow. This job now installs and invokes coder-eval through it. What blocked adoption before is fixed upstream: the suite lives in `tests/`, needs an agent extra, and for delegate-sdk needs a plugin inside the environment the CLI actually runs from, and none of that was expressible. The single 150-line run step becomes prep + invoke. `working-directory:` is illegal on a `uses:` step and a job-level `defaults.run` does not reach inside a composite action, so everything conditional resolves into plain inputs first. The prep body is the head of the old step moved rather than rewritten: each `export FOO=bar` became a line in an env-block accumulator, and the trailing `coder-eval run` is gone. Every guard is preserved and was exercised locally against the extracted script: the three `:?` model hard-failures, delegate's `-j` clamp with its notice, its per-key `~/.uipath/.auth` validation, and the non-empty-only re-export of the three `*_VAR` knobs. All twenty non-`_VAR` env entries are forwarded, including the four model variables the shell also reads, because coder-eval's docker driver passes `CODEX_MODEL` and `ANTIGRAVITY_MODEL` into task containers by default. The bracketed `-D sandbox.docker.env_passthrough_extra=[...]` override now goes through the action's `args` input, which appends one argument per line verbatim. That is the same protection the old array-and-quote gave it, for the same reason: to bash the list is a character class. **An evalboard link per run.** The run uploads to a dedicated `runs-gha` container, which expires it after 14 days, and the link lands in the job summary. Entirely best-effort: every step warns and returns 0, because a missing dashboard link must never be mistaken for a failed eval. Inert until `AZURE_EVAL_UPLOAD_CLIENT_ID` exists as a repo variable, so this behaves exactly as before until the Azure identity is provisioned. Credentials are OIDC, not a storage key: `workflow_dispatch` runs the workflow definition from whatever branch the dispatcher picks, so a key in a repo secret would let any branch delete months of nightly history on that account. **Two fixes to things already broken.** The workflow had no redaction step, unlike smoke-skills.yml, so its artifact has been shipping unredacted transcripts; it is ported into both jobs and covers every secret this job forwards, not just the three smoke-skills strips. And the Linux artifact globs were off by one directory level, uploading no experiment- or variant-level report at all, because `--run-dir /tmp/runs` makes that directory itself the run dir. `run-name` is one line and independently the highest value per character available here: all ~2,500 historical runs are titled "Run Coder Eval". Co-Authored-By: Claude Opus 5 --- .github/workflows/run-coder-eval.yml | 636 +++++++++++++++++++++++---- 1 file changed, 548 insertions(+), 88 deletions(-) diff --git a/.github/workflows/run-coder-eval.yml b/.github/workflows/run-coder-eval.yml index 3482b31485..0f8322bf6c 100644 --- a/.github/workflows/run-coder-eval.yml +++ b/.github/workflows/run-coder-eval.yml @@ -1,5 +1,11 @@ name: Run Coder Eval +# Every one of the ~2,500 historical runs is titled "Run Coder Eval", so the run +# list is a wall of identical rows: nobody can find their own dispatch, see what a +# colleague tested, or spot a duplicate. These three fields are what anyone +# actually scans for. +run-name: ${{ inputs.task_globs }} · ${{ inputs.agent }} · @${{ github.actor }} + # GH-hosted runner for coder-eval against the skills task tree. Use cases: # single task ad-hoc, a folder, or (with explicit confirmation) the full # suite that the VM cron runs nightly. Long-term replacement for the @@ -119,6 +125,7 @@ jobs: windows_count: ${{ steps.split.outputs.windows_count }} coder_eval_version: ${{ steps.ceref.outputs.version }} coder_eval_uipath_ref: ${{ steps.ceref.outputs.ref }} + image_tag: ${{ steps.ceref.outputs.image_tag }} steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 @@ -128,11 +135,16 @@ jobs: # the nightly VM also read, overridable to dry-run a candidate release) # and the coder_eval_uipath plugin ref (delegate-sdk only). Mirrors the # version resolution in daily.sh. + # The agent image tag is resolved HERE rather than in the Linux job's + # install step, because that step is now the composite action and no longer + # runs before the docker pull. Same defaulting as before: blank + # coder_eval_image_tag means the pinned coder-eval version. - name: Resolve coder-eval version + coder_eval_uipath ref id: ceref env: CE_VERSION_INPUT: ${{ inputs.coder_eval_version }} CE_UIPATH_REF_INPUT: ${{ inputs.coder_eval_uipath_ref }} + CE_IMAGE_TAG_INPUT: ${{ inputs.coder_eval_image_tag }} run: | set -euo pipefail version="$CE_VERSION_INPUT" @@ -144,8 +156,12 @@ jobs: exit 1 fi echo "coder-eval pinned to $version; coder_eval_uipath ref ${CE_UIPATH_REF_INPUT:-main}" - echo "version=$version" >> "$GITHUB_OUTPUT" - echo "ref=${CE_UIPATH_REF_INPUT:-main}" >> "$GITHUB_OUTPUT" + echo "agent image tag ${CE_IMAGE_TAG_INPUT:-$version}" + { + echo "version=$version" + echo "ref=${CE_UIPATH_REF_INPUT:-main}" + echo "image_tag=${CE_IMAGE_TAG_INPUT:-$version}" + } >> "$GITHUB_OUTPUT" - name: Resolve globs, split by tag, enforce large-run gate id: split @@ -207,6 +223,23 @@ jobs: # hiccups, etc.). timeout-minutes: 330 name: Run coder-eval (Linux) + # JOB-level, deliberately, and not workflow-level: a `permissions:` block + # makes every permission it omits `none` for every job it covers, so a + # workflow-level one would silently strip the Windows job's token too. The + # file declares none today, which is why this is purely additive here. + # id-token: write -> mint the OIDC token the evalboard upload exchanges + # for an Azure credential (no storage key in a secret) + # contents: read -> actions/checkout + permissions: + id-token: write + contents: read + # The federated credential's subject is `repo:UiPath/skills:environment: + # eval-upload`. Environment scope rather than branch scope because + # workflow_dispatch can target ANY branch and Entra subjects do not wildcard + # reliably, so a branch-scoped credential would break on the next dispatch + # from a feature branch. NOTE: adding a protection rule to this environment + # would gate EVERY dispatch of this workflow behind it. + environment: eval-upload # Disable uip CLI version-sync: a task's `uip login status` re-pins # ~/.uipath/config.json to a stale line and, via nightly.yaml's shared # ~/.uipath mount, downgrades the CLI for later tasks. Mirrors daily.sh. @@ -217,15 +250,29 @@ jobs: # The coder_eval_uipath plugin (private repo): registers the delegate-sdk # agent via the `coder_eval.plugins` entry point and carries the delegate - # docker-overlay build context. delegate-sdk ONLY (same gate as daily.sh): - # its register() monkeypatches core apply_overrides and mutates the pricing - # table, and the other agents must not depend on this private checkout - # succeeding. Needs a repo-read token — GH_PACKAGES_TOKEN is packages-only, - # so this is the one step that requires GH_PAT. persist-credentials: false - # keeps that PAT out of .git/config: this checkout doubles as the overlay - # docker build context below. + # docker-overlay build context. Its register() monkeypatches core + # apply_overrides and mutates the pricing table, so it is still INSTALLED + # for delegate-sdk only (same gate as daily.sh) and no other agent depends + # on that code path. + # + # The CHECKOUT is now wider than the install: the repo also ships + # `eval_runner/`, a separate package (`coder-eval-uipath-eval-runner`) that + # is the blob uploader behind the evalboard link. Installing that one + # touches nothing the agents use — it lands in its own uv tool environment + # and imports no coder-eval code. + # + # Non-fatal on purpose. This puts GH_PAT on the happy path of every + # dispatch where it used to be delegate-only, and a stale PAT must cost the + # evalboard link, not the eval. delegate-sdk still fails loudly: the prep + # step below asserts the checkout is present before the plugin is needed. + # + # Needs a repo-read token — GH_PACKAGES_TOKEN is packages-only, so this is + # the one step that requires GH_PAT. persist-credentials: false keeps that + # PAT out of .git/config: this checkout doubles as the overlay docker build + # context below. - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - if: inputs.agent == 'delegate-sdk' + if: inputs.agent == 'delegate-sdk' || vars.AZURE_EVAL_UPLOAD_CLIENT_ID != '' + continue-on-error: true with: repository: UiPath/coder_eval_uipath ref: ${{ needs.partition.outputs.coder_eval_uipath_ref }} @@ -239,53 +286,61 @@ jobs: - uses: astral-sh/setup-uv@38f3f104447c67c051c4a08e39b64a148898af3a # v4.2.0 - # Host coder-eval CLI at the version the partition job resolved, plus — for - # delegate-sdk only — the coder_eval_uipath plugin from the checkout above. - # ONE uv invocation so the `==` pin and the local plugin co-resolve: `uv pip - # install ` ignores the plugin's lockfile and its dependency range is - # an open `0.9.x`, so the pin is what keeps this run on the same framework - # version as smoke-skills.yml and the nightly VM. Agent extras stay - # unpinned — they ride the same resolve. The Linux docker driver still - # imports the selected agent class on the host before dispatching work, so - # the plugin and the extras must be installed here as well as in the - # container image. The pinned version doubles as the default agent image - # tag (a non-blank coder_eval_image_tag input overrides it for ad-hoc / - # unreleased images). + # coder-eval itself is no longer installed here: the composite action + # (`uses: UiPath/coder_eval@v0`, below) does it, via `uv tool install`, from + # the same `tests/.coder-eval-version` pin the partition job resolved. Its + # `extras` and `extra-packages` inputs carry what this step used to compose + # by hand — including, for delegate-sdk, the coder_eval_uipath plugin, which + # must share an environment with coder-eval for its `coder_eval.plugins` + # entry point to be discovered at all. # - # Coupling this creates, on purpose: the pin must sit INSIDE the plugin's - # declared coder-eval range (currently `>=0.9.5,<0.10.0`). If a + # The coupling that step documented is unchanged and still deliberate: the + # pin must sit INSIDE the plugin's declared coder-eval range. If a # tests/.coder-eval-version bump lands before coder_eval_uipath widens its - # range, this step fails at resolve time with uv's "requirements are - # unsatisfiable" — loudly, before any task runs, and only on delegate-sdk. - # That is the point: the alternative (resolving through the plugin's open - # range) silently drifts. Fix by widening the plugin's range, or pass an - # in-range coder_eval_version for the delegate run. - - name: Install coder-eval - env: - AGENT: ${{ inputs.agent }} - CE_VERSION: ${{ needs.partition.outputs.coder_eval_version }} - CE_IMAGE_TAG_INPUT: ${{ inputs.coder_eval_image_tag }} + # range, the install fails at resolve time with uv's "requirements are + # unsatisfiable" — loudly, before any task runs, and only on delegate-sdk, + # because `--with` requirements co-resolve with the tool's own. Fix by + # widening the plugin's range, or pass an in-range coder_eval_version. + + # `uv tool install` puts its shims in uv's tool bin dir, and both the + # action's `coder-eval` and the uploader's `eval-runner` are found only if + # that dir is on PATH. It is on PATH on the GitHub-hosted images; do not + # assume it on `uipath-ubuntu-latest`, which is a different image. + - name: Ensure the uv tool bin dir is on PATH + run: echo "$HOME/.local/bin" >> "$GITHUB_PATH" + + # The blob uploader behind the evalboard link. A SEPARATE package from the + # plugin above — coder_eval_uipath ships `coder-eval-uipath` (the plugin) at + # its root and `coder-eval-uipath-eval-runner` (this CLI) under + # eval_runner/ — so it goes in its own isolated tool environment and cannot + # perturb the one the action builds. All six of its dependencies are public + # PyPI; no private index is involved. + # + # Skipped entirely until the Azure identity exists, and non-fatal even then: + # a missing uploader costs the link, never the run. + - name: Install the evalboard uploader + if: vars.AZURE_EVAL_UPLOAD_CLIENT_ID != '' + continue-on-error: true run: | set -euo pipefail - case "$AGENT" in - codex) pkgs=("coder-eval[codex,litellm]==$CE_VERSION") ;; - antigravity) pkgs=("coder-eval[antigravity,litellm]==$CE_VERSION") ;; - delegate-sdk) pkgs=("coder-eval[litellm]==$CE_VERSION" ./coder_eval_uipath) ;; - *) pkgs=("coder-eval[litellm]==$CE_VERSION") ;; - esac - uv pip install --system "${pkgs[@]}" - python -c 'from importlib.metadata import version; print("coder-eval resolved to", version("coder-eval"))' - echo "CE_IMAGE_TAG=${CE_IMAGE_TAG_INPUT:-$CE_VERSION}" >> "$GITHUB_ENV" + if [ ! -d coder_eval_uipath/eval_runner ]; then + echo "::warning::coder_eval_uipath checkout missing — skipping the evalboard upload for this run" + exit 0 + fi + uv tool install ./coder_eval_uipath/eval_runner + eval-runner --help > /dev/null # Pull the agent image from GHCR (never built from source); the skills # image extends it below. Tag defaults to the pinned coder-eval version - # (CE_IMAGE_TAG from the install step) but can be overridden + # (resolved by the partition job) but can be overridden # (coder_eval_image_tag) for ad-hoc / unreleased images. # Needs read:packages only — GH_PACKAGES_TOKEN, never the repo-write PAT. # This workflow is `workflow_dispatch`-triggered, so it is not exposed to # the fork/branch-push reader that motivates the split in smoke-skills.yml # and smoke-rpa-skills.yml; the narrower token is least-privilege here. - name: Pull coder-eval-agent image + env: + CE_IMAGE_TAG: ${{ needs.partition.outputs.image_tag }} run: | echo "${{ secrets.GH_PACKAGES_TOKEN || secrets.GH_PAT }}" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin docker pull "ghcr.io/uipath/coder-eval-agent:${CE_IMAGE_TAG}" @@ -296,6 +351,7 @@ jobs: env: DOCKER_BUILDKIT: "1" NPM_AUTH_TOKEN: ${{ secrets.GH_PACKAGES_TOKEN || secrets.GH_PAT }} + CE_IMAGE_TAG: ${{ needs.partition.outputs.image_tag }} run: | docker build \ --build-arg CODER_EVAL_IMAGE="ghcr.io/uipath/coder-eval-agent:${CE_IMAGE_TAG}" \ @@ -359,9 +415,24 @@ jobs: - name: Stage _shared grader helpers run: bash tests/scripts/stage_shared.sh - # TASK_GLOBS here is the partition-resolved list of *file paths* (not - # globs). Word-splitting is intentional — do not quote. - - name: Run coder-eval + # ================================================================== + # The run is a two-step split: this step COMPUTES, the composite action + # below EXECUTES. It exists because `working-directory:` is illegal on a + # `uses:` step and a job-level `defaults.run` does not reach inside a + # composite action, so everything conditional has to be resolved into + # plain inputs first. + # + # The body is the head of the old single `Run coder-eval` step, moved + # rather than rewritten: same guards, same order, same messages. The two + # mechanical changes are that each `export FOO=bar` became a line in the + # env-block accumulator, and the trailing `coder-eval run ...` is gone. + # Review it as a diff against that step, not as new code. + # + # TASK_GLOBS is the partition-resolved list of *file paths* (not globs) and + # is handed straight to the action's `tasks` input, which word-splits it. + # ================================================================== + - name: Prepare the coder-eval invocation + id: prep env: SKILLS_REPO_PATH: ${{ github.workspace }} API_BACKEND: bedrock @@ -404,25 +475,68 @@ jobs: DELEGATE_STDIO_LOG_MAX_CHARS_VAR: ${{ vars.DELEGATE_STDIO_LOG_MAX_CHARS }} USE_LLM_GATEWAY_FOR_FIREWORKS_VAR: ${{ vars.USE_LLM_GATEWAY_FOR_FIREWORKS }} working-directory: tests - id: eval run: | + set -uo pipefail + # Guard the empty case (e.g. a future non-dispatch trigger): the # `parallelism` input default (4) only applies on workflow_dispatch. j="${TASK_PARALLELISM:-4}" - # Agent selection. claude is the default; codex authenticates via - # CODEX_API_KEY/CODEX_BASE_URL, antigravity via GEMINI_API_KEY (SDK baked into - # the agent image), delegate-sdk via the ROPC user token minted above. The - # model always comes from the harness's own variable above, with the - # agent_model input overriding any of them for a one-off run. Passed - # explicitly because the experiment YAML pins no model (it is shared by all - # harnesses). Driver is unchanged (docker here). - delegate_overrides=() + + # Environment forwarded to the coder-eval process via the action's + # `env` input, which exports these for that process only — never into + # $GITHUB_ENV, so nothing bleeds into later steps. Built from a name + # list so it cannot drift from the `env:` block above: compare the two + # lists, not 20 hand-written lines. + # + # This is EVERY non-_VAR entry of that block, including the ones only the + # shell below reads, so the environment coder-eval sees is unchanged by + # the split. Not merely defensive: coder-eval's docker driver forwards + # CODEX_MODEL and ANTIGRAVITY_MODEL into task containers by default + # (the passthrough list in models/sandbox.py), where they select the + # model should agent.model ever be unset — so dropping them because the + # shell also reads them would be a real change. An empty value forwards + # as empty, which is what the `env:` block already produced for an unset + # repo variable. Every value must be single-line; all of these are. + env_lines=() + for name in SKILLS_REPO_PATH API_BACKEND AWS_BEARER_TOKEN_BEDROCK AWS_REGION \ + BEDROCK_MODEL ANTHROPIC_API_KEY E2E_PROCESS_KEY E2E_LONG_PROCESS_KEY \ + TASK_GLOBS TASK_COUNT TASK_PARALLELISM AGENT AGENT_MODEL \ + CLAUDE_CODE_MODEL CODEX_MODEL ANTIGRAVITY_MODEL DELEGATE_MODEL \ + CODEX_API_KEY CODEX_BASE_URL GEMINI_API_KEY; do + env_lines+=("$name=${!name}") + done + + # Args appended to `coder-eval run`, ONE PER LINE. The action appends + # each verbatim with no word splitting and no pathname expansion, which + # is exactly what the delegate `-D` override below needs: to bash the + # `[...]` list is a character class, and an unquoted expansion silently + # collapses the whole flag to `...extra=` if a file of that name + # exists in the tests/ cwd. + args=(-e experiments/nightly.yaml) + + # Agent selection. codex authenticates via CODEX_API_KEY/CODEX_BASE_URL, + # antigravity via GEMINI_API_KEY (SDK baked into the agent image), + # delegate-sdk via the ROPC user token minted above. The model always + # comes from the harness's own variable above, with the agent_model input + # overriding any of them for a one-off run. Passed explicitly because the + # experiment YAML pins no model (it is shared by all harnesses). Driver + # is unchanged (docker here). + # + # `extras` are the agent's own host-side SDK plus the litellm judge + # transport nightly.yaml's checker_context routes through. They go into + # the install requirement, not a second install: the action's tool + # environment shadows every other coder-eval on PATH, so an extra added + # beside it would never be imported by the CLI that runs. + extras="litellm" + extra_packages="" if [ "$AGENT" = "codex" ]; then model="${AGENT_MODEL:-${CODEX_MODEL:?unset — set the CODEX_MODEL repo variable}}" - agent_flags="--type codex --model $model" + args+=(--type codex) + extras="codex,litellm" elif [ "$AGENT" = "antigravity" ]; then model="${AGENT_MODEL:-${ANTIGRAVITY_MODEL:?unset — set the ANTIGRAVITY_MODEL repo variable}}" - agent_flags="--type antigravity --model $model" + args+=(--type antigravity) + extras="antigravity,litellm" elif [ "$AGENT" = "delegate-sdk" ]; then # Default kimi-k2-7-code (hyphenated id — the delegate host converts # it to the backend's underscored form); the optional DELEGATE_MODEL @@ -430,7 +544,20 @@ jobs: # claude default is rejected by the alpha Delegate backend, so a # model must always be passed. model="${AGENT_MODEL:-${DELEGATE_MODEL:-kimi-k2-7-code}}" - agent_flags="--type delegate-sdk --model $model" + args+=(--type delegate-sdk) + # The plugin must resolve into the SAME environment as coder-eval for + # its `coder_eval.plugins` entry point to be discovered; that is what + # `extra-packages` (uv tool install --with) does, and why the pin and + # the plugin still co-resolve in one invocation. Path is relative to + # the action's working-directory (tests/). + extra_packages="../coder_eval_uipath" + # That checkout is best-effort now (it also feeds the evalboard + # uploader), so assert it here: for delegate-sdk it is mandatory, and + # this fails with a readable message instead of a uv resolve error. + if [ ! -d ../coder_eval_uipath ]; then + echo "::error::coder_eval_uipath checkout is missing — delegate-sdk needs it for the plugin. Check the GH_PAT secret." + exit 1 + fi # Alpha backend rate limits: the reference pipeline caps delegate-sdk # at -j 6 (eval_runner cli.py HARNESS_CONCURRENCY — at -j 20 retry # backoff blew task timeouts into TIMEOUT waves, build 12599458). @@ -443,8 +570,8 @@ jobs: fi # Cloud env slug for the Delegate backend + first-response stall # recovery budget (mirrors daily.sh in coder_eval_uipath). - export DELEGATE_SDK_ENV="${DELEGATE_SDK_ENV:-alpha}" - export DELEGATE_STALL_TIMEOUT_S="${DELEGATE_STALL_TIMEOUT_S:-240}" + env_lines+=("DELEGATE_SDK_ENV=${DELEGATE_SDK_ENV:-alpha}") + env_lines+=("DELEGATE_STALL_TIMEOUT_S=${DELEGATE_STALL_TIMEOUT_S:-240}") # Delegate auth: reuse the ROPC USER token refresh-auth.sh wrote — # the Delegate backend rejects client_credentials tokens. The host # reads auth + org/tenant identity (GUIDs AND slugs — slugs build the @@ -465,7 +592,14 @@ jobs: echo "::error::no $key in $AUTH_FILE — the ROPC login did not complete" exit 1 fi - export "$out=$val" + # refresh-auth.sh already ::add-mask::ed the token at mint time and + # `chmod -R o+rwX ~/.uipath` leaves it readable job-wide, so routing + # it through a step output adds no exposure. Re-masking is one line + # and survives anyone reordering those steps. + if [ "$out" = "AUTH_TOKEN" ]; then + echo "::add-mask::$val" + fi + env_lines+=("$out=$val") done # Re-export the debugging knobs above only when the repo variable is # actually set, so a blank one leaves the host on its own default. @@ -473,7 +607,7 @@ jobs: USE_LLM_GATEWAY_FOR_FIREWORKS; do src="${knob}_VAR" if [ -n "${!src:-}" ]; then - export "$knob=${!src}" + env_lines+=("$knob=${!src}") echo "delegate knob: $knob=${!src}" fi done @@ -483,7 +617,7 @@ jobs: # PROVIDED the file stays fresh. This workflow mints the token once # (no refresher loop like the nightly's), so runs longer than ~1h can # still hit token expiry on late tasks. - export DELEGATE_AUTH_TOKEN_FILE="/.uipath/.auth:$AUTH_FILE" + env_lines+=("DELEGATE_AUTH_TOKEN_FILE=/.uipath/.auth:$AUTH_FILE") # Docker forwards only an env allowlist into the container; append # the delegate auth/env vars to nightly.yaml's own list (MergeField # append-semantics). Same list as run.py in coder_eval_uipath: @@ -491,28 +625,67 @@ jobs: # the environment provides one, and it is simply unset (harmless) # here. DELEGATE_STDIO_PATH is deliberately NOT forwarded: the # overlay image bakes its own, and the host path would not resolve - # in-container. Built as an ARRAY and quoted at the call site: to bash - # the `[...]` list is a glob character class, so an unquoted expansion - # silently collapses the whole flag to `...extra=` if a file of - # that name ever exists in the tests/ cwd. Unlikely, but an array - # costs nothing and removes the class of bug outright. - + # in-container. delegate_env="AUTH_TOKEN,DELEGATE_AUTH_TOKEN_FILE,TENANT_ID,ORG_ID,USER_ID" delegate_env="$delegate_env,ORG_LOGICAL_NAME,TENANT_NAME,DELEGATE_SDK_ENV" delegate_env="$delegate_env,DELEGATE_STALL_TIMEOUT_S,USE_LLM_GATEWAY_FOR_FIREWORKS" delegate_env="$delegate_env,DELEGATE_STDIO_VERBOSE,DELEGATE_STDIO_LOG_MAX_CHARS" - delegate_overrides=(-D "sandbox.docker.env_passthrough_extra=[$delegate_env]") + args+=(-D "sandbox.docker.env_passthrough_extra=[$delegate_env]") else model="${AGENT_MODEL:-${CLAUDE_CODE_MODEL:?unset — set the CLAUDE_CODE_MODEL repo variable}}" - agent_flags="--model $model" fi - echo "Running: coder-eval run $TASK_GLOBS -e experiments/nightly.yaml $agent_flags ${delegate_overrides[*]} -j $j ($TASK_COUNT task files)" - coder-eval run $TASK_GLOBS \ - -e experiments/nightly.yaml \ - $agent_flags \ - "${delegate_overrides[@]}" \ - -j "$j" -v \ - --run-dir /tmp/runs + + # After the clamp, so a delegate run reports the -j it will actually use. + args+=(-j "$j" -v) + + echo "Running: coder-eval run $TASK_GLOBS --model $model ${args[*]} ($TASK_COUNT task files)" + + # Multi-line step outputs need the heredoc form. The delimiter is + # randomised because a value that happened to equal it, alone on a + # line, would truncate the block. + delim="ghaout$RANDOM$RANDOM$RANDOM" + { + echo "model=$model" + echo "extras=$extras" + echo "extra_packages=$extra_packages" + echo "args<<$delim" + printf '%s\n' "${args[@]}" + echo "$delim" + echo "env_block<<$delim" + printf '%s\n' "${env_lines[@]}" + echo "$delim" + } >> "$GITHUB_OUTPUT" + + # The published composite action, consumed as a real dependent rather than + # only by its own self-test. It installs coder-eval at the resolved pin + # (plus `extras` / `extra-packages`) and runs it. + # + # `minimum-task-score` is deliberately NOT set: the gate here is "every row + # is SUCCESS" (the verdict step below), which is a different predicate from + # a weighted_score floor. Swapping gate semantics inside a refactor whose + # premise is behaviour preservation would make a regression indistinguishable + # from the intended change. + # + # `step-summary` is off for the same reason it cannot simply be turned on: + # the action writes run.md to the job summary immediately after the run, + # which is BEFORE any redaction step can execute. The summary is appended + # below instead, after redaction. Revisit once redaction moves upstream into + # the action. + - name: Run coder-eval + id: eval + uses: UiPath/coder_eval@v0 + with: + version: ${{ needs.partition.outputs.coder_eval_version }} + working-directory: tests + tasks: ${{ needs.partition.outputs.linux_globs }} + model: ${{ steps.prep.outputs.model }} + extras: ${{ steps.prep.outputs.extras }} + extra-packages: ${{ steps.prep.outputs.extra_packages }} + args: ${{ steps.prep.outputs.args }} + env: ${{ steps.prep.outputs.env_block }} + run-dir: /tmp/runs + junit-path: /tmp/coder-eval-junit.xml + step-summary: "false" - name: Fix permissions on /tmp/runs if: always() @@ -525,21 +698,108 @@ jobs: run: | find /tmp/runs -type d \( -name .venv -o -name node_modules \) -path "*/artifacts/*" -exec rm -rf {} + || true + # Ported from smoke-skills.yml, which has had this since the reports + # started carrying agent transcripts. This workflow never had it, so its + # artifact has been shipping unredacted transcripts all along. Runs BEFORE + # anything that publishes the tree — the artifact upload, the job summary, + # and the evalboard blob — and AFTER the cleanup above, which removes tens + # of thousands of files this walk would otherwise scan for nothing. + - name: Redact secrets from the eval report + if: always() + env: + AWS_BEARER_TOKEN_BEDROCK: ${{ secrets.AWS_BEARER_TOKEN_BEDROCK }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + CODEX_API_KEY: ${{ secrets.CODEX_API_KEY }} + GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} + E2E_PROCESS_KEY: ${{ secrets.E2E_PROCESS_KEY }} + E2E_LONG_PROCESS_KEY: ${{ secrets.E2E_LONG_PROCESS_KEY }} + UIPATH_ROPC_CLIENT_SECRET: ${{ secrets.UIPATH_ROPC_CLIENT_SECRET }} + UIPATH_BOT_PASSWORD: ${{ secrets.UIPATH_BOT_PASSWORD }} + run: | + python - <<'PY' + import os + import pathlib + + # Every secret this job forwards into the run, not just the three + # smoke-skills.yml strips. It matters more here than there: this report + # is uploaded twice (a GitHub artifact and, when the identity is wired, + # an evalboard blob) and its run.md is echoed into the job summary. + names = ( + "AWS_BEARER_TOKEN_BEDROCK", + "ANTHROPIC_API_KEY", + "CODEX_API_KEY", + "GEMINI_API_KEY", + "E2E_PROCESS_KEY", + "E2E_LONG_PROCESS_KEY", + "UIPATH_ROPC_CLIENT_SECRET", + "UIPATH_BOT_PASSWORD", + # The ROPC user token, which the delegate host and the in-container + # uip CLI both use. Read from the file rather than the environment: + # this workflow mints it at runtime, so no secret carries it. + "UIPATH_CLI_AUTH_TOKEN", + ) + values = [os.environ[n] for n in names if len(os.environ.get(n, "")) >= 8] + + auth = pathlib.Path(os.path.expanduser("~/.uipath/.auth")) + if auth.is_file(): + for line in auth.read_text(encoding="utf-8", errors="ignore").splitlines(): + # Token only. Redacting the org/tenant SLUGS as well would + # garble every report that legitimately names the tenant. + if line.startswith("UIPATH_ACCESS_TOKEN=") and len(line) > 28: + values.append(line.split("=", 1)[1]) + + if not values: + raise SystemExit(0) + + suffixes = {".html", ".json", ".log", ".md", ".xml"} + for path in pathlib.Path("/tmp/runs").rglob("*"): + if not path.is_file() or path.suffix not in suffixes: + continue + text = path.read_text(encoding="utf-8", errors="ignore") + redacted = text + for value in values: + redacted = redacted.replace(value, "***") + if redacted != text: + path.write_text(redacted, encoding="utf-8") + PY + + # The action's own `step-summary` cannot be used for this: it writes run.md + # the instant the run finishes, which is before the redaction above can + # execute. Same output, correct order. Same 1 MB cap the action applies — + # GitHub rejects a job summary over 1 MiB outright, losing the verdict too. + - name: Append the run report to the job summary + if: always() + run: | + set -uo pipefail + if [ -f /tmp/runs/run.md ]; then + head -c 1000000 /tmp/runs/run.md >> "$GITHUB_STEP_SUMMARY" + else + echo "_No run.md produced._" >> "$GITHUB_STEP_SUMMARY" + fi + - name: Upload eval report if: always() uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: coder-eval-linux-${{ github.run_id }} - # ** so the pattern matches the replicate-index segment that - # coder_eval adds to per-task run dirs (/tmp/runs/////). + # These were off by one directory level and so uploaded NO + # experiment-level or variant-level report, only the per-task files the + # `**` patterns caught. `--run-dir /tmp/runs` makes /tmp/runs itself the + # run dir: reports land at /tmp/runs/{run,experiment}.* and + # /tmp/runs//variant.*, and per-task dirs at + # /tmp/runs//// (hence `**`, for the replicate + # segment). The Windows job's copies are correct as written because it + # passes no --run-dir and so gets a timestamped subdirectory. path: | - /tmp/runs/*/experiment.html - /tmp/runs/*/experiment.md - /tmp/runs/*/experiment.json - /tmp/runs/*/experiment.log - /tmp/runs/*/*/variant.html - /tmp/runs/*/*/variant.md - /tmp/runs/*/*/variant.json + /tmp/runs/run.json + /tmp/runs/run.md + /tmp/runs/experiment.html + /tmp/runs/experiment.md + /tmp/runs/experiment.json + /tmp/runs/experiment.log + /tmp/runs/*/variant.html + /tmp/runs/*/variant.md + /tmp/runs/*/variant.json /tmp/runs/**/task.html /tmp/runs/**/task.json /tmp/runs/**/task.log @@ -566,6 +826,132 @@ jobs: sys.exit(0 if rows and ok == len(rows) else 1) PY + # ================================================================== + # Evalboard publish. Entirely additive and entirely best-effort: every + # step here warns and returns 0 rather than reddening a run, because a + # missing dashboard link must never be mistaken for a failed eval. + # + # Inert until `AZURE_EVAL_UPLOAD_CLIENT_ID` is set as a repo variable, so + # this merges and behaves exactly as before while the Azure identity is + # still being provisioned. + # ================================================================== + + # Requested straight from the Actions OIDC endpoint rather than through + # azure/login, which would additionally need the `az` CLI present on this + # runner image (a self-hosted one, so do not assume it). Writing the token + # to a file and pointing AZURE_FEDERATED_TOKEN_FILE at it is all + # DefaultAzureCredential needs to resolve via WorkloadIdentityCredential. + # No storage account key is involved anywhere: workflow_dispatch runs the + # workflow definition from whatever branch the dispatcher picks, so a key in + # a repo secret would let any branch read or delete months of nightly + # history on the same account. + - name: Mint an Azure OIDC token + if: always() && vars.AZURE_EVAL_UPLOAD_CLIENT_ID != '' + continue-on-error: true + run: | + set -uo pipefail + if [ -z "${ACTIONS_ID_TOKEN_REQUEST_URL:-}" ]; then + echo "::warning::no OIDC request URL in the environment (id-token permission?) — no evalboard link for this run" + exit 0 + fi + # api://AzureADTokenExchange is the audience an Entra federated + # credential validates against. + if ! resp="$(curl -sSf \ + -H "Authorization: bearer $ACTIONS_ID_TOKEN_REQUEST_TOKEN" \ + "${ACTIONS_ID_TOKEN_REQUEST_URL}&audience=api://AzureADTokenExchange")"; then + echo "::warning::OIDC token request failed — no evalboard link for this run" + exit 0 + fi + # Never echoed: the response body is a bearer assertion. + printf '%s' "$resp" | python3 -c 'import json,sys; sys.stdout.write(json.load(sys.stdin)["value"])' \ + > "$RUNNER_TEMP/azure-oidc-token" + chmod 600 "$RUNNER_TEMP/azure-oidc-token" + if [ ! -s "$RUNNER_TEMP/azure-oidc-token" ]; then + echo "::warning::OIDC response carried no token — no evalboard link for this run" + rm -f "$RUNNER_TEMP/azure-oidc-token" + exit 0 + fi + echo "OIDC token minted; subject is repo:$GITHUB_REPOSITORY:environment:eval-upload" + + - name: Publish the run to evalboard + if: always() && vars.AZURE_EVAL_UPLOAD_CLIENT_ID != '' + continue-on-error: true + env: + # Client and tenant ids are identifiers, not secrets, so they live as + # repo variables — and the absence of the client id is what gates every + # step in this block. + AZURE_CLIENT_ID: ${{ vars.AZURE_EVAL_UPLOAD_CLIENT_ID }} + AZURE_TENANT_ID: ${{ vars.AZURE_EVAL_UPLOAD_TENANT_ID }} + AZURE_FEDERATED_TOKEN_FILE: ${{ runner.temp }}/azure-oidc-token + # AZURE_STORAGE_KEY is deliberately left UNSET: an empty key is what + # makes eval-runner fall through to DefaultAzureCredential. + AZURE_STORAGE_ACCOUNT: ${{ vars.AZURE_EVAL_UPLOAD_STORAGE_ACCOUNT || 'coderevaltests' }} + AZURE_BLOB_CONTAINER: ${{ vars.AZURE_EVAL_UPLOAD_CONTAINER || 'runs-gha' }} + EVALBOARD_BASE: ${{ vars.EVALBOARD_BASE_URL || 'https://coder-evalboard.uipath-dev.com' }} + AGENT: ${{ inputs.agent }} + TASK_COUNT: ${{ needs.partition.outputs.linux_count }} + TASK_GLOBS_INPUT: ${{ inputs.task_globs }} + run: | + set -uo pipefail + if ! command -v eval-runner > /dev/null 2>&1; then + echo "::warning::eval-runner is not installed — no evalboard link for this run" + exit 0 + fi + if [ ! -f "$AZURE_FEDERATED_TOKEN_FILE" ]; then + echo "::warning::no Azure OIDC token on disk — no evalboard link for this run" + exit 0 + fi + if [ ! -f /tmp/runs/run.json ]; then + echo "::warning::/tmp/runs/run.json is missing — nothing to publish" + exit 0 + fi + + # `eval-runner upload` derives the blob id from the run directory's + # NAME, so the rename is how the id is chosen. Timestamp first to keep + # runs chronologically sortable, then the GitHub run id and attempt: + # uniqueness is the point, since two dispatches landing in the same + # second would otherwise share an id and silently overwrite each other. + # Safe to move because this is the last step to touch /tmp/runs — the + # artifact upload and the verdict have both already run. + # All-hyphen after the timestamp: a `.` before the attempt number reads + # as a file extension in the URL path. + blob_id="$(date -u +%Y-%m-%d_%H-%M-%S)-gha-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + mv /tmp/runs "/tmp/$blob_id" || { + echo "::warning::could not stage /tmp/runs for upload — no evalboard link for this run" + exit 0 + } + + run_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" + if ! eval-runner upload "/tmp/$blob_id" \ + --adhoc \ + --title "gha #${GITHUB_RUN_NUMBER}: ${AGENT} x ${TASK_COUNT} linux task(s) by @${GITHUB_ACTOR} - ${TASK_GLOBS_INPUT}" \ + --description "$run_url"; then + echo "::warning::evalboard upload failed — see the log above. The GitHub artifact is unaffected." + exit 0 + fi + + # ?src=gha is required, not cosmetic: sourceById COERCES an unknown or + # absent src to the default source instead of throwing, so a link with + # it stripped 404s against the nightly container with no explanation. + url="${EVALBOARD_BASE}/runs/${blob_id}?src=gha" + # Azure evaluates lifecycle rules once a day and can lag a day or two + # past the threshold, so the date is a floor rather than a promise. + expires="$(date -u -d '+14 days' +%Y-%m-%d 2> /dev/null || true)" + if [ -n "$expires" ]; then + when="on or after \`$expires\`" + else + when="about 14 days from now" + fi + { + echo "" + echo "### Evalboard" + echo "" + echo "[$url]($url)" + echo "" + echo "Covers the **Linux** tasks of this dispatch only. Deleted $when by the 14-day lifecycle rule on the \`$AZURE_BLOB_CONTAINER\` container." + } >> "$GITHUB_STEP_SUMMARY" + echo "Evalboard: $url" + run-windows: needs: partition if: needs.partition.outputs.windows_count != '0' @@ -1018,6 +1404,66 @@ jobs: tail -n 100 "$log" || true fi + # Same redaction as the Linux job, on this job's own run tree. Ported to + # BOTH jobs deliberately: leaving it Linux-only would mean the Windows + # artifact kept shipping unredacted transcripts, which is the status quo but + # becomes a visible inconsistency once one job strips them. + - name: Redact secrets from the eval report + if: always() + shell: bash + env: + AWS_BEARER_TOKEN_BEDROCK: ${{ secrets.AWS_BEARER_TOKEN_BEDROCK }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + CODEX_API_KEY: ${{ secrets.CODEX_API_KEY }} + GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} + E2E_PROCESS_KEY: ${{ secrets.E2E_PROCESS_KEY }} + E2E_LONG_PROCESS_KEY: ${{ secrets.E2E_LONG_PROCESS_KEY }} + UIPATH_ROPC_CLIENT_SECRET: ${{ secrets.UIPATH_ROPC_CLIENT_SECRET }} + UIPATH_BOT_PASSWORD: ${{ secrets.UIPATH_BOT_PASSWORD }} + # This job mints its own ROPC token to a workspace file rather than + # ~/.uipath/.auth (that path is the uip CLI's own browser-OAuth login). + DELEGATE_AUTH_FILE: ${{ github.workspace }}/.delegate-sdk-auth + run: | + python - <<'PY' + import os + import pathlib + + names = ( + "AWS_BEARER_TOKEN_BEDROCK", + "ANTHROPIC_API_KEY", + "CODEX_API_KEY", + "GEMINI_API_KEY", + "E2E_PROCESS_KEY", + "E2E_LONG_PROCESS_KEY", + "UIPATH_ROPC_CLIENT_SECRET", + "UIPATH_BOT_PASSWORD", + "UIPATH_CLI_AUTH_TOKEN", + ) + values = [os.environ[n] for n in names if len(os.environ.get(n, "")) >= 8] + + auth = pathlib.Path(os.environ.get("DELEGATE_AUTH_FILE", "")) + if auth.name and auth.is_file(): + for line in auth.read_text(encoding="utf-8", errors="ignore").splitlines(): + # Token only — redacting the org/tenant SLUGS would garble every + # report that legitimately names the tenant. + if line.startswith("UIPATH_ACCESS_TOKEN=") and len(line) > 28: + values.append(line.split("=", 1)[1]) + + if not values: + raise SystemExit(0) + + suffixes = {".html", ".json", ".log", ".md", ".xml"} + for path in pathlib.Path("tests/runs").rglob("*"): + if not path.is_file() or path.suffix not in suffixes: + continue + text = path.read_text(encoding="utf-8", errors="ignore") + redacted = text + for value in values: + redacted = redacted.replace(value, "***") + if redacted != text: + path.write_text(redacted, encoding="utf-8") + PY + - name: Upload eval report if: always() uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 @@ -1057,3 +1503,17 @@ jobs: print(f"\n{ok}/{len(rows)} PASS (windows)") sys.exit(0 if rows and ok == len(rows) else 1) PY + + # A single dispatch can start both jobs, and only the Linux one publishes an + # evalboard link. Said out loud so its absence here reads as "not + # applicable" rather than as an upload that failed. + # + # Its own step, deliberately, so the verdict above stays byte-identical: an + # echo appended after that `python | tee` pipeline would become the step's + # exit status and silently turn every red Windows run green. + - name: Note the absent evalboard link + if: always() + shell: bash + run: | + echo "" >> "$GITHUB_STEP_SUMMARY" + echo "_Windows tasks are reported through the \`coder-eval-windows-${{ github.run_id }}\` artifact only. The evalboard link, if any, covers the Linux half of this dispatch._" >> "$GITHUB_STEP_SUMMARY" From 92ae8e4fa8ca117fa5c3a98b49b2a8b6de7e9d4f Mon Sep 17 00:00:00 2001 From: Bai Li Date: Tue, 1 Sep 2026 10:51:08 -0700 Subject: [PATCH 2/9] refactor(ci): authenticate the evalboard upload with the existing storage credential The evalboard upload minted a GitHub OIDC token and exchanged it for an Azure credential, which needed a new Entra app registration, a federated credential scoped to a GitHub environment, and a container role assignment before it could work at all. The ADO pipelines already authenticate to this same storage account with `AZURE_STORAGE_KEY` / `AZURE_STORAGE_ACCOUNT`, and eval-runner passes that value straight to `BlobServiceClient`, so reusing it removes every one of those provisioning steps: two repo settings and nothing else. The Azure SDK sniffs the credential string, so the secret holds either the account key or a container-scoped SAS token with no change here. The SAS is preferable, since an account key is authority over the whole account including the nightly-history container, and this workflow runs its definition from whatever branch the dispatcher picks. Drops `id-token: write`, the `eval-upload` environment and the token-minting step. The gate moves to a job-env boolean because the `secrets` context is unavailable in any `if:`; deriving a boolean rather than exporting the secret keeps the credential in the upload step alone, matching how ADO scopes it. The container name is now hardcoded, since the unlisted evalboard source and the 14-day expiry rule both depend on ad-hoc runs never landing in `runs`. Co-Authored-By: Claude Opus 5 --- .github/workflows/run-coder-eval.yml | 102 +++++++++------------------ 1 file changed, 34 insertions(+), 68 deletions(-) diff --git a/.github/workflows/run-coder-eval.yml b/.github/workflows/run-coder-eval.yml index 0f8322bf6c..8c73e527a2 100644 --- a/.github/workflows/run-coder-eval.yml +++ b/.github/workflows/run-coder-eval.yml @@ -227,24 +227,20 @@ jobs: # makes every permission it omits `none` for every job it covers, so a # workflow-level one would silently strip the Windows job's token too. The # file declares none today, which is why this is purely additive here. - # id-token: write -> mint the OIDC token the evalboard upload exchanges - # for an Azure credential (no storage key in a secret) # contents: read -> actions/checkout permissions: - id-token: write contents: read - # The federated credential's subject is `repo:UiPath/skills:environment: - # eval-upload`. Environment scope rather than branch scope because - # workflow_dispatch can target ANY branch and Entra subjects do not wildcard - # reliably, so a branch-scoped credential would break on the next dispatch - # from a feature branch. NOTE: adding a protection rule to this environment - # would gate EVERY dispatch of this workflow behind it. - environment: eval-upload # Disable uip CLI version-sync: a task's `uip login status` re-pins # ~/.uipath/config.json to a stale line and, via nightly.yaml's shared # ~/.uipath mount, downgrades the CLI for later tasks. Mirrors daily.sh. env: UIPATH_CLI_DISABLE_VERSION_SYNC: "1" + # Whether this dispatch can publish an evalboard link, as a plain boolean + # string. The `secrets` context is NOT available in any `if:`, so the gate + # has to be laundered through the job env like this; deriving a boolean + # rather than exporting the secret itself keeps the credential out of every + # step but the one that uploads. + EVALBOARD_UPLOAD_ENABLED: ${{ secrets.AZURE_STORAGE_KEY != '' }} steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 @@ -271,7 +267,7 @@ jobs: # PAT out of .git/config: this checkout doubles as the overlay docker build # context below. - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - if: inputs.agent == 'delegate-sdk' || vars.AZURE_EVAL_UPLOAD_CLIENT_ID != '' + if: inputs.agent == 'delegate-sdk' || env.EVALBOARD_UPLOAD_ENABLED == 'true' continue-on-error: true with: repository: UiPath/coder_eval_uipath @@ -316,10 +312,10 @@ jobs: # perturb the one the action builds. All six of its dependencies are public # PyPI; no private index is involved. # - # Skipped entirely until the Azure identity exists, and non-fatal even then: + # Skipped entirely until AZURE_STORAGE_KEY is set, and non-fatal even then: # a missing uploader costs the link, never the run. - name: Install the evalboard uploader - if: vars.AZURE_EVAL_UPLOAD_CLIENT_ID != '' + if: env.EVALBOARD_UPLOAD_ENABLED == 'true' continue-on-error: true run: | set -euo pipefail @@ -831,62 +827,36 @@ jobs: # step here warns and returns 0 rather than reddening a run, because a # missing dashboard link must never be mistaken for a failed eval. # - # Inert until `AZURE_EVAL_UPLOAD_CLIENT_ID` is set as a repo variable, so - # this merges and behaves exactly as before while the Azure identity is - # still being provisioned. + # Inert until `AZURE_STORAGE_KEY` is set as a repo secret, so this merges + # and behaves exactly as before while the credential is being provisioned. + # + # Authenticated with the SAME credential the ADO pipelines already use for + # this storage account (`AZURE_STORAGE_KEY` / `AZURE_STORAGE_ACCOUNT` out of + # `coder-eval-athena-{secrets,config}`), rather than a second, GitHub-only + # Entra app + federated credential. eval-runner passes the value straight to + # `BlobServiceClient(credential=...)`, and the Azure SDK sniffs the string, + # so the secret can hold either the account key or a container-scoped SAS + # token with no change here. Prefer the SAS: an account key is authority + # over the WHOLE account, including the `runs` container holding months of + # nightly history, and this workflow runs its definition from whatever + # branch the dispatcher picks. # ================================================================== - # Requested straight from the Actions OIDC endpoint rather than through - # azure/login, which would additionally need the `az` CLI present on this - # runner image (a self-hosted one, so do not assume it). Writing the token - # to a file and pointing AZURE_FEDERATED_TOKEN_FILE at it is all - # DefaultAzureCredential needs to resolve via WorkloadIdentityCredential. - # No storage account key is involved anywhere: workflow_dispatch runs the - # workflow definition from whatever branch the dispatcher picks, so a key in - # a repo secret would let any branch read or delete months of nightly - # history on the same account. - - name: Mint an Azure OIDC token - if: always() && vars.AZURE_EVAL_UPLOAD_CLIENT_ID != '' - continue-on-error: true - run: | - set -uo pipefail - if [ -z "${ACTIONS_ID_TOKEN_REQUEST_URL:-}" ]; then - echo "::warning::no OIDC request URL in the environment (id-token permission?) — no evalboard link for this run" - exit 0 - fi - # api://AzureADTokenExchange is the audience an Entra federated - # credential validates against. - if ! resp="$(curl -sSf \ - -H "Authorization: bearer $ACTIONS_ID_TOKEN_REQUEST_TOKEN" \ - "${ACTIONS_ID_TOKEN_REQUEST_URL}&audience=api://AzureADTokenExchange")"; then - echo "::warning::OIDC token request failed — no evalboard link for this run" - exit 0 - fi - # Never echoed: the response body is a bearer assertion. - printf '%s' "$resp" | python3 -c 'import json,sys; sys.stdout.write(json.load(sys.stdin)["value"])' \ - > "$RUNNER_TEMP/azure-oidc-token" - chmod 600 "$RUNNER_TEMP/azure-oidc-token" - if [ ! -s "$RUNNER_TEMP/azure-oidc-token" ]; then - echo "::warning::OIDC response carried no token — no evalboard link for this run" - rm -f "$RUNNER_TEMP/azure-oidc-token" - exit 0 - fi - echo "OIDC token minted; subject is repo:$GITHUB_REPOSITORY:environment:eval-upload" - - name: Publish the run to evalboard - if: always() && vars.AZURE_EVAL_UPLOAD_CLIENT_ID != '' + if: always() && env.EVALBOARD_UPLOAD_ENABLED == 'true' continue-on-error: true env: - # Client and tenant ids are identifiers, not secrets, so they live as - # repo variables — and the absence of the client id is what gates every - # step in this block. - AZURE_CLIENT_ID: ${{ vars.AZURE_EVAL_UPLOAD_CLIENT_ID }} - AZURE_TENANT_ID: ${{ vars.AZURE_EVAL_UPLOAD_TENANT_ID }} - AZURE_FEDERATED_TOKEN_FILE: ${{ runner.temp }}/azure-oidc-token - # AZURE_STORAGE_KEY is deliberately left UNSET: an empty key is what - # makes eval-runner fall through to DefaultAzureCredential. - AZURE_STORAGE_ACCOUNT: ${{ vars.AZURE_EVAL_UPLOAD_STORAGE_ACCOUNT || 'coderevaltests' }} - AZURE_BLOB_CONTAINER: ${{ vars.AZURE_EVAL_UPLOAD_CONTAINER || 'runs-gha' }} + # The only step in the job that sees the credential. Mirrors how the ADO + # pipelines scope it ("upload in its own step so AZURE_STORAGE_KEY is + # never in the eval step's env"): a task-driven agent with shell access + # runs in those other steps. + AZURE_STORAGE_KEY: ${{ secrets.AZURE_STORAGE_KEY }} + AZURE_STORAGE_ACCOUNT: ${{ vars.AZURE_STORAGE_ACCOUNT || 'coderevaltests' }} + # Hardcoded, with no variable override on purpose. Everything about this + # feature — the unlisted evalboard source, the 14-day expiry rule — is + # predicated on ad-hoc runs never landing in `runs`, and ADO already uses + # the name `AZURE_BLOB_CONTAINER` for `runs` itself. + AZURE_BLOB_CONTAINER: runs-gha EVALBOARD_BASE: ${{ vars.EVALBOARD_BASE_URL || 'https://coder-evalboard.uipath-dev.com' }} AGENT: ${{ inputs.agent }} TASK_COUNT: ${{ needs.partition.outputs.linux_count }} @@ -897,10 +867,6 @@ jobs: echo "::warning::eval-runner is not installed — no evalboard link for this run" exit 0 fi - if [ ! -f "$AZURE_FEDERATED_TOKEN_FILE" ]; then - echo "::warning::no Azure OIDC token on disk — no evalboard link for this run" - exit 0 - fi if [ ! -f /tmp/runs/run.json ]; then echo "::warning::/tmp/runs/run.json is missing — nothing to publish" exit 0 From 98a274b99d2a6ea47394a1cc629e5de3063ec625 Mon Sep 17 00:00:00 2001 From: Bai Li Date: Tue, 1 Sep 2026 11:40:28 -0700 Subject: [PATCH 3/9] refactor(ci): pass the whole coder-eval command line through the action's args Follows the action's input simplification: it no longer promotes any of `coder-eval run`'s flags to named inputs, so the task globs and `--model` join the `-e`, `--type`, `-j`, `-v` and `-D` entries in the single `args` block the prep step already composed. The `model` step output goes away with them. Task globs are word-split when appended (TASK_GLOBS is a space-separated list from the partition job) but never pathname-expanded, which is an improvement on what the action used to do with them: the CLI expands them itself, so `**` works without `globstar` and a glob matching nothing exits 1 rather than reaching the CLI as a literal path. `step-summary: "false"` and `junit-path` are gone. The action writes no job summary at all now and reports `run-md-path` instead, so the append can happen after redaction, which is the order this workflow needs and previously had to work around. The append reads that output with the literal path as a fallback, because a composite action's outputs may not propagate from a step that exited non-zero and this step runs on failure. The JUnit report moves to /tmp/runs/junit.xml, derived from run-dir; nothing consumed it at the old path. Co-Authored-By: Claude Opus 5 --- .github/workflows/run-coder-eval.yml | 53 ++++++++++++++++------------ 1 file changed, 31 insertions(+), 22 deletions(-) diff --git a/.github/workflows/run-coder-eval.yml b/.github/workflows/run-coder-eval.yml index 8c73e527a2..da51b083d8 100644 --- a/.github/workflows/run-coder-eval.yml +++ b/.github/workflows/run-coder-eval.yml @@ -632,16 +632,24 @@ jobs: fi # After the clamp, so a delegate run reports the -j it will actually use. - args+=(-j "$j" -v) + args+=(-j "$j" -v --model "$model") - echo "Running: coder-eval run $TASK_GLOBS --model $model ${args[*]} ($TASK_COUNT task files)" + # Task globs are `args` entries like everything else: the action + # promotes none of the CLI's flags to named inputs, so there is no + # `tasks:` or `model:` to pass separately. Word-split on purpose here + # (TASK_GLOBS is a space-separated list from the partition job) so each + # glob becomes its own argv entry; NOT pathname-expanded, because the + # CLI expands them itself and handles `**` without globstar. + # shellcheck disable=SC2206 + args+=($TASK_GLOBS) + + echo "Running: coder-eval run ${args[*]} ($TASK_COUNT task files)" # Multi-line step outputs need the heredoc form. The delimiter is # randomised because a value that happened to equal it, alone on a # line, would truncate the block. delim="ghaout$RANDOM$RANDOM$RANDOM" { - echo "model=$model" echo "extras=$extras" echo "extra_packages=$extra_packages" echo "args<<$delim" @@ -656,32 +664,27 @@ jobs: # only by its own self-test. It installs coder-eval at the resolved pin # (plus `extras` / `extra-packages`) and runs it. # - # `minimum-task-score` is deliberately NOT set: the gate here is "every row - # is SUCCESS" (the verdict step below), which is a different predicate from - # a weighted_score floor. Swapping gate semantics inside a refactor whose - # premise is behaviour preservation would make a regression indistinguishable - # from the intended change. + # Everything the CLI takes — the task globs, --model, --type, -j, the -D + # overrides — is in the single `args` block the prep step composed. The + # action promotes none of the CLI's flags to named inputs, which is also why + # there is nothing here to turn the job-summary write off: the action does + # not write one. It is appended below instead, after redaction, which is the + # order this workflow needs and could not previously get. # - # `step-summary` is off for the same reason it cannot simply be turned on: - # the action writes run.md to the job summary immediately after the run, - # which is BEFORE any redaction step can execute. The summary is appended - # below instead, after redaction. Revisit once redaction moves upstream into - # the action. + # The JUnit report lands at /tmp/runs/junit.xml (derived from run-dir). + # Nothing here consumes it; it travels with the run into the artifact and + # the evalboard blob. - name: Run coder-eval id: eval uses: UiPath/coder_eval@v0 with: version: ${{ needs.partition.outputs.coder_eval_version }} working-directory: tests - tasks: ${{ needs.partition.outputs.linux_globs }} - model: ${{ steps.prep.outputs.model }} extras: ${{ steps.prep.outputs.extras }} extra-packages: ${{ steps.prep.outputs.extra_packages }} args: ${{ steps.prep.outputs.args }} env: ${{ steps.prep.outputs.env_block }} run-dir: /tmp/runs - junit-path: /tmp/coder-eval-junit.xml - step-summary: "false" - name: Fix permissions on /tmp/runs if: always() @@ -759,16 +762,22 @@ jobs: path.write_text(redacted, encoding="utf-8") PY - # The action's own `step-summary` cannot be used for this: it writes run.md - # the instant the run finishes, which is before the redaction above can - # execute. Same output, correct order. Same 1 MB cap the action applies — + # The action reports where run.md is and leaves the writing to the consumer, + # which is what lets this land AFTER the redaction above. Capped at 1 MB: # GitHub rejects a job summary over 1 MiB outright, losing the verdict too. + # + # Read from the action's `run-md-path` output rather than rebuilding the + # path, with the literal as a fallback: a composite action's outputs may not + # propagate from a step that exited non-zero, and this step runs on failure. - name: Append the run report to the job summary if: always() + env: + RUN_MD: ${{ steps.eval.outputs.run-md-path }} run: | set -uo pipefail - if [ -f /tmp/runs/run.md ]; then - head -c 1000000 /tmp/runs/run.md >> "$GITHUB_STEP_SUMMARY" + run_md="${RUN_MD:-/tmp/runs/run.md}" + if [ -f "$run_md" ]; then + head -c 1000000 "$run_md" >> "$GITHUB_STEP_SUMMARY" else echo "_No run.md produced._" >> "$GITHUB_STEP_SUMMARY" fi From 2a283dc91bde630fdf5dc81f4a25ebc2858726b2 Mon Sep 17 00:00:00 2001 From: Bai Li Date: Tue, 1 Sep 2026 14:07:01 -0700 Subject: [PATCH 4/9] test(ci): temporarily pin the action to the PR-147 head for an e2e dry run MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The `v0` tag still serves the ten-input action, so this workflow's `with:` block would have `args`, `extras`, `extra-packages` and `working-directory` silently dropped — GitHub ignores an input the referenced tag does not declare. Pinning the PR head validates the whole path before anything merges and before Release moves `v0`. REVERT TO @v0 BEFORE MERGING. Co-Authored-By: Claude Opus 5 --- .github/workflows/run-coder-eval.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/run-coder-eval.yml b/.github/workflows/run-coder-eval.yml index da51b083d8..e73d40f9f1 100644 --- a/.github/workflows/run-coder-eval.yml +++ b/.github/workflows/run-coder-eval.yml @@ -676,7 +676,11 @@ jobs: # the evalboard blob. - name: Run coder-eval id: eval - uses: UiPath/coder_eval@v0 + # TEMPORARY: pinned to the PR-147 head so this workflow can be validated + # against the eight-input action BEFORE that PR merges and Release moves + # `v0`. Revert to `@v0` before merging — the v0 tag still serves the old + # ten-input action, which would silently ignore every input below. + uses: UiPath/coder_eval@a747ce353111beadd51a6370b9e8e2dba88f4133 # bai/action-inputs-and-gha-source with: version: ${{ needs.partition.outputs.coder_eval_version }} working-directory: tests From d207e9c655442674af303680c9b1a881323db878 Mon Sep 17 00:00:00 2001 From: Bai Li Date: Tue, 1 Sep 2026 14:09:19 -0700 Subject: [PATCH 5/9] docs(ci): trim the comment bulk around the action split Comments only. Fifteen blocks cut to their contract, 118 lines out for 58 in, so the file sits at its existing 33% comment density instead of above it. Also drops a stale claim: the two-step-split banner said TASK_GLOBS "is handed straight to the action's `tasks` input, which word-splits it". There is no `tasks` input, and `args` entries are passed verbatim. Co-Authored-By: Claude Opus 5 --- .github/workflows/run-coder-eval.yml | 176 +++++++++------------------ 1 file changed, 58 insertions(+), 118 deletions(-) diff --git a/.github/workflows/run-coder-eval.yml b/.github/workflows/run-coder-eval.yml index e73d40f9f1..580454184d 100644 --- a/.github/workflows/run-coder-eval.yml +++ b/.github/workflows/run-coder-eval.yml @@ -1,9 +1,7 @@ name: Run Coder Eval -# Every one of the ~2,500 historical runs is titled "Run Coder Eval", so the run -# list is a wall of identical rows: nobody can find their own dispatch, see what a -# colleague tested, or spot a duplicate. These three fields are what anyone -# actually scans for. +# Every historical run is titled "Run Coder Eval", so the run list is a wall of +# identical rows. These three fields are what anyone actually scans for. run-name: ${{ inputs.task_globs }} · ${{ inputs.agent }} · @${{ github.actor }} # GH-hosted runner for coder-eval against the skills task tree. Use cases: @@ -135,10 +133,8 @@ jobs: # the nightly VM also read, overridable to dry-run a candidate release) # and the coder_eval_uipath plugin ref (delegate-sdk only). Mirrors the # version resolution in daily.sh. - # The agent image tag is resolved HERE rather than in the Linux job's - # install step, because that step is now the composite action and no longer - # runs before the docker pull. Same defaulting as before: blank - # coder_eval_image_tag means the pinned coder-eval version. + # The image tag is resolved here, not in the Linux job: that step is now the + # composite action and no longer runs before the docker pull. - name: Resolve coder-eval version + coder_eval_uipath ref id: ceref env: @@ -223,11 +219,9 @@ jobs: # hiccups, etc.). timeout-minutes: 330 name: Run coder-eval (Linux) - # JOB-level, deliberately, and not workflow-level: a `permissions:` block - # makes every permission it omits `none` for every job it covers, so a - # workflow-level one would silently strip the Windows job's token too. The - # file declares none today, which is why this is purely additive here. - # contents: read -> actions/checkout + # JOB-level, not workflow-level: a `permissions:` block makes every omitted + # permission `none` for every job it covers, so a workflow-level one would + # silently strip the Windows job's token. `contents: read` is for checkout. permissions: contents: read # Disable uip CLI version-sync: a task's `uip login status` re-pins @@ -235,11 +229,9 @@ jobs: # ~/.uipath mount, downgrades the CLI for later tasks. Mirrors daily.sh. env: UIPATH_CLI_DISABLE_VERSION_SYNC: "1" - # Whether this dispatch can publish an evalboard link, as a plain boolean - # string. The `secrets` context is NOT available in any `if:`, so the gate - # has to be laundered through the job env like this; deriving a boolean - # rather than exporting the secret itself keeps the credential out of every - # step but the one that uploads. + # The `secrets` context is NOT available in any `if:`, so the gate is + # laundered through the job env. A boolean, not the secret, so the + # credential stays in the one step that uploads. EVALBOARD_UPLOAD_ENABLED: ${{ secrets.AZURE_STORAGE_KEY != '' }} steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 @@ -282,38 +274,24 @@ jobs: - uses: astral-sh/setup-uv@38f3f104447c67c051c4a08e39b64a148898af3a # v4.2.0 - # coder-eval itself is no longer installed here: the composite action - # (`uses: UiPath/coder_eval@v0`, below) does it, via `uv tool install`, from - # the same `tests/.coder-eval-version` pin the partition job resolved. Its - # `extras` and `extra-packages` inputs carry what this step used to compose - # by hand — including, for delegate-sdk, the coder_eval_uipath plugin, which - # must share an environment with coder-eval for its `coder_eval.plugins` - # entry point to be discovered at all. - # - # The coupling that step documented is unchanged and still deliberate: the - # pin must sit INSIDE the plugin's declared coder-eval range. If a - # tests/.coder-eval-version bump lands before coder_eval_uipath widens its - # range, the install fails at resolve time with uv's "requirements are - # unsatisfiable" — loudly, before any task runs, and only on delegate-sdk, - # because `--with` requirements co-resolve with the tool's own. Fix by - # widening the plugin's range, or pass an in-range coder_eval_version. - - # `uv tool install` puts its shims in uv's tool bin dir, and both the - # action's `coder-eval` and the uploader's `eval-runner` are found only if - # that dir is on PATH. It is on PATH on the GitHub-hosted images; do not - # assume it on `uipath-ubuntu-latest`, which is a different image. + # coder-eval is installed by the composite action below, not here, via + # `uv tool install` at the pin the partition job resolved; its `extras` and + # `extra-packages` inputs carry what this step used to compose by hand. + # The old coupling holds: the pin must sit inside the plugin's declared + # coder-eval range, or the install fails at resolve time (delegate-sdk + # only) because `--with` requirements co-resolve with the tool's own. + + # Both the action's `coder-eval` and the uploader's `eval-runner` are uv + # tool shims, found only if uv's tool bin dir is on PATH. It is on the + # GitHub-hosted images; do not assume it on `uipath-ubuntu-latest`. - name: Ensure the uv tool bin dir is on PATH run: echo "$HOME/.local/bin" >> "$GITHUB_PATH" - # The blob uploader behind the evalboard link. A SEPARATE package from the - # plugin above — coder_eval_uipath ships `coder-eval-uipath` (the plugin) at - # its root and `coder-eval-uipath-eval-runner` (this CLI) under - # eval_runner/ — so it goes in its own isolated tool environment and cannot - # perturb the one the action builds. All six of its dependencies are public - # PyPI; no private index is involved. - # - # Skipped entirely until AZURE_STORAGE_KEY is set, and non-fatal even then: - # a missing uploader costs the link, never the run. + # The blob uploader behind the evalboard link — a SEPARATE package from the + # plugin above (`coder-eval-uipath-eval-runner`, under eval_runner/), so it + # gets its own tool environment and cannot perturb the action's. Skipped + # until AZURE_STORAGE_KEY is set, and non-fatal: a missing uploader costs + # the link, never the run. - name: Install the evalboard uploader if: env.EVALBOARD_UPLOAD_ENABLED == 'true' continue-on-error: true @@ -411,22 +389,12 @@ jobs: - name: Stage _shared grader helpers run: bash tests/scripts/stage_shared.sh - # ================================================================== - # The run is a two-step split: this step COMPUTES, the composite action - # below EXECUTES. It exists because `working-directory:` is illegal on a - # `uses:` step and a job-level `defaults.run` does not reach inside a - # composite action, so everything conditional has to be resolved into - # plain inputs first. - # - # The body is the head of the old single `Run coder-eval` step, moved - # rather than rewritten: same guards, same order, same messages. The two - # mechanical changes are that each `export FOO=bar` became a line in the - # env-block accumulator, and the trailing `coder-eval run ...` is gone. - # Review it as a diff against that step, not as new code. - # - # TASK_GLOBS is the partition-resolved list of *file paths* (not globs) and - # is handed straight to the action's `tasks` input, which word-splits it. - # ================================================================== + # Two-step split: this step COMPUTES, the composite action below EXECUTES. + # `working-directory:` is illegal on a `uses:` step and a job-level + # `defaults.run` does not reach inside a composite, so everything + # conditional has to be resolved into plain inputs first. The body is the + # head of the old single `Run coder-eval` step, moved rather than + # rewritten — review it as a diff against that step. - name: Prepare the coder-eval invocation id: prep env: @@ -478,21 +446,12 @@ jobs: # `parallelism` input default (4) only applies on workflow_dispatch. j="${TASK_PARALLELISM:-4}" - # Environment forwarded to the coder-eval process via the action's - # `env` input, which exports these for that process only — never into - # $GITHUB_ENV, so nothing bleeds into later steps. Built from a name - # list so it cannot drift from the `env:` block above: compare the two - # lists, not 20 hand-written lines. - # - # This is EVERY non-_VAR entry of that block, including the ones only the - # shell below reads, so the environment coder-eval sees is unchanged by - # the split. Not merely defensive: coder-eval's docker driver forwards - # CODEX_MODEL and ANTIGRAVITY_MODEL into task containers by default - # (the passthrough list in models/sandbox.py), where they select the - # model should agent.model ever be unset — so dropping them because the - # shell also reads them would be a real change. An empty value forwards - # as empty, which is what the `env:` block already produced for an unset - # repo variable. Every value must be single-line; all of these are. + # Forwarded to coder-eval via the action's `env` input, which exports + # these for that process only. A name list, so it cannot drift from the + # `env:` block above. Deliberately EVERY non-_VAR entry of that block, + # including ones only this shell reads: the docker driver forwards + # CODEX_MODEL / ANTIGRAVITY_MODEL into task containers, where they pick + # the model if agent.model is unset. Values must be single-line. env_lines=() for name in SKILLS_REPO_PATH API_BACKEND AWS_BEARER_TOKEN_BEDROCK AWS_REGION \ BEDROCK_MODEL ANTHROPIC_API_KEY E2E_PROCESS_KEY E2E_LONG_PROCESS_KEY \ @@ -502,12 +461,9 @@ jobs: env_lines+=("$name=${!name}") done - # Args appended to `coder-eval run`, ONE PER LINE. The action appends - # each verbatim with no word splitting and no pathname expansion, which - # is exactly what the delegate `-D` override below needs: to bash the - # `[...]` list is a character class, and an unquoted expansion silently - # collapses the whole flag to `...extra=` if a file of that name - # exists in the tests/ cwd. + # Args appended to `coder-eval run`, ONE PER LINE, each verbatim — no + # word splitting, no pathname expansion. That is what the delegate `-D` + # override needs: to bash its `[...]` list is a character class. args=(-e experiments/nightly.yaml) # Agent selection. codex authenticates via CODEX_API_KEY/CODEX_BASE_URL, @@ -518,11 +474,10 @@ jobs: # experiment YAML pins no model (it is shared by all harnesses). Driver # is unchanged (docker here). # - # `extras` are the agent's own host-side SDK plus the litellm judge - # transport nightly.yaml's checker_context routes through. They go into - # the install requirement, not a second install: the action's tool - # environment shadows every other coder-eval on PATH, so an extra added - # beside it would never be imported by the CLI that runs. + # `extras` are the agent's host-side SDK plus the litellm judge + # transport. They go into the install requirement, not a second + # install: the action's tool environment shadows every other + # coder-eval on PATH. extras="litellm" extra_packages="" if [ "$AGENT" = "codex" ]; then @@ -541,11 +496,9 @@ jobs: # model must always be passed. model="${AGENT_MODEL:-${DELEGATE_MODEL:-kimi-k2-7-code}}" args+=(--type delegate-sdk) - # The plugin must resolve into the SAME environment as coder-eval for - # its `coder_eval.plugins` entry point to be discovered; that is what - # `extra-packages` (uv tool install --with) does, and why the pin and - # the plugin still co-resolve in one invocation. Path is relative to - # the action's working-directory (tests/). + # The plugin must share coder-eval's environment for its + # `coder_eval.plugins` entry point to be found, which is what + # `extra-packages` does. Path is relative to tests/. extra_packages="../coder_eval_uipath" # That checkout is best-effort now (it also feeds the evalboard # uploader), so assert it here: for delegate-sdk it is mandatory, and @@ -588,10 +541,8 @@ jobs: echo "::error::no $key in $AUTH_FILE — the ROPC login did not complete" exit 1 fi - # refresh-auth.sh already ::add-mask::ed the token at mint time and - # `chmod -R o+rwX ~/.uipath` leaves it readable job-wide, so routing - # it through a step output adds no exposure. Re-masking is one line - # and survives anyone reordering those steps. + # Already masked at mint time, and the file is readable job-wide, + # so a step output adds no exposure. Re-masking survives a reorder. if [ "$out" = "AUTH_TOKEN" ]; then echo "::add-mask::$val" fi @@ -660,25 +611,15 @@ jobs: echo "$delim" } >> "$GITHUB_OUTPUT" - # The published composite action, consumed as a real dependent rather than - # only by its own self-test. It installs coder-eval at the resolved pin - # (plus `extras` / `extra-packages`) and runs it. - # - # Everything the CLI takes — the task globs, --model, --type, -j, the -D - # overrides — is in the single `args` block the prep step composed. The - # action promotes none of the CLI's flags to named inputs, which is also why - # there is nothing here to turn the job-summary write off: the action does - # not write one. It is appended below instead, after redaction, which is the - # order this workflow needs and could not previously get. - # - # The JUnit report lands at /tmp/runs/junit.xml (derived from run-dir). - # Nothing here consumes it; it travels with the run into the artifact and - # the evalboard blob. + # The published composite action, consumed as a real dependent. Everything + # the CLI takes is in the single `args` block the prep step composed: the + # action promotes none of its flags to named inputs. That is also why there + # is no job-summary switch here — the action writes none, so the report is + # appended below, after redaction. JUnit lands at /tmp/runs/junit.xml and + # travels with the run into the artifact and the blob. - name: Run coder-eval id: eval - # TEMPORARY: pinned to the PR-147 head so this workflow can be validated - # against the eight-input action BEFORE that PR merges and Release moves - # `v0`. Revert to `@v0` before merging — the v0 tag still serves the old + # TEMPORARY — REVERT TO @v0 BEFORE MERGING. `v0` still serves the old # ten-input action, which would silently ignore every input below. uses: UiPath/coder_eval@a747ce353111beadd51a6370b9e8e2dba88f4133 # bai/action-inputs-and-gha-source with: @@ -909,9 +850,8 @@ jobs: exit 0 fi - # ?src=gha is required, not cosmetic: sourceById COERCES an unknown or - # absent src to the default source instead of throwing, so a link with - # it stripped 404s against the nightly container with no explanation. + # ?src=gha is required: sourceById COERCES an absent src to the default + # source, so a stripped link 404s against the nightly container. url="${EVALBOARD_BASE}/runs/${blob_id}?src=gha" # Azure evaluates lifecycle rules once a day and can lag a day or two # past the threshold, so the date is a floor rather than a promise. From 1b800ef1f6d9bb0a4a0489b5173981791a074dc9 Mon Sep 17 00:00:00 2001 From: Bai Li Date: Tue, 1 Sep 2026 14:25:40 -0700 Subject: [PATCH 6/9] fix(ci): put the evalboard link at the top of the job summary The link was appended after run.md, which is roughly 40 lines of tables, so it landed below the fold and was easy to miss. Prepend rather than reorder the steps: the evalboard step moves /tmp/runs, which both the artifact upload and the verdict step read, so it has to stay last. $GITHUB_STEP_SUMMARY is an ordinary file, so the block is written to a temp file with the existing summary appended, then truncate-and-rewritten in place to keep the inode the runner tracks. Co-Authored-By: Claude Opus 5 --- .github/workflows/run-coder-eval.yml | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run-coder-eval.yml b/.github/workflows/run-coder-eval.yml index 580454184d..8288a0d1f8 100644 --- a/.github/workflows/run-coder-eval.yml +++ b/.github/workflows/run-coder-eval.yml @@ -861,14 +861,23 @@ jobs: else when="about 14 days from now" fi + # Prepended, not appended: run.md is ~40 lines of tables, so an appended + # link lands below the fold. Moving this step earlier is not an option, + # because it MOVES /tmp/runs, which the artifact upload and the verdict + # both read. $GITHUB_STEP_SUMMARY is an ordinary file; truncate-and-rewrite + # rather than `mv` so the runner keeps tracking the same inode. + summary_tmp="$(mktemp)" { - echo "" echo "### Evalboard" echo "" echo "[$url]($url)" echo "" echo "Covers the **Linux** tasks of this dispatch only. Deleted $when by the 14-day lifecycle rule on the \`$AZURE_BLOB_CONTAINER\` container." - } >> "$GITHUB_STEP_SUMMARY" + echo "" + cat "$GITHUB_STEP_SUMMARY" 2> /dev/null || true + } > "$summary_tmp" + cat "$summary_tmp" > "$GITHUB_STEP_SUMMARY" + rm -f "$summary_tmp" echo "Evalboard: $url" run-windows: From df068eed53725d4bbbe6c67b6fb14b717f907788 Mon Sep 17 00:00:00 2001 From: Bai Li Date: Tue, 1 Sep 2026 15:11:25 -0700 Subject: [PATCH 7/9] fix(ci): publish evalboard before the report so the link renders first The previous attempt prepended to $GITHUB_STEP_SUMMARY inside the evalboard step. That cannot work: the variable points at a SEPARATE file per step, and GitHub builds the job summary by concatenating them in step order, so the step read its own empty file and its block still rendered last. Verified on run 33564007553, where the link stayed at the bottom. Move the publish above the step that appends run.md, which is the only way to control position, and copy the run directory instead of moving it so the artifact upload and the verdict still find /tmp/runs. Also emit a ::notice::, which renders above the job list on the run page. Keep "Run Coder Eval" as the run-name prefix so the run list stays scannable by workflow as well as by glob, agent and actor. Co-Authored-By: Claude Opus 5 --- .github/workflows/run-coder-eval.yml | 173 ++++++++++++++------------- 1 file changed, 88 insertions(+), 85 deletions(-) diff --git a/.github/workflows/run-coder-eval.yml b/.github/workflows/run-coder-eval.yml index 8288a0d1f8..9c7a112fe6 100644 --- a/.github/workflows/run-coder-eval.yml +++ b/.github/workflows/run-coder-eval.yml @@ -1,8 +1,9 @@ name: Run Coder Eval # Every historical run is titled "Run Coder Eval", so the run list is a wall of -# identical rows. These three fields are what anyone actually scans for. -run-name: ${{ inputs.task_globs }} · ${{ inputs.agent }} · @${{ github.actor }} +# identical rows. Keep that prefix (it is how the workflow is recognised) and +# append the three fields anyone actually scans for. +run-name: Run Coder Eval · ${{ inputs.task_globs }} · ${{ inputs.agent }} · @${{ github.actor }} # GH-hosted runner for coder-eval against the skills task tree. Use cases: # single task ad-hoc, a folder, or (with explicit confirmation) the full @@ -706,76 +707,6 @@ jobs: if redacted != text: path.write_text(redacted, encoding="utf-8") PY - - # The action reports where run.md is and leaves the writing to the consumer, - # which is what lets this land AFTER the redaction above. Capped at 1 MB: - # GitHub rejects a job summary over 1 MiB outright, losing the verdict too. - # - # Read from the action's `run-md-path` output rather than rebuilding the - # path, with the literal as a fallback: a composite action's outputs may not - # propagate from a step that exited non-zero, and this step runs on failure. - - name: Append the run report to the job summary - if: always() - env: - RUN_MD: ${{ steps.eval.outputs.run-md-path }} - run: | - set -uo pipefail - run_md="${RUN_MD:-/tmp/runs/run.md}" - if [ -f "$run_md" ]; then - head -c 1000000 "$run_md" >> "$GITHUB_STEP_SUMMARY" - else - echo "_No run.md produced._" >> "$GITHUB_STEP_SUMMARY" - fi - - - name: Upload eval report - if: always() - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 - with: - name: coder-eval-linux-${{ github.run_id }} - # These were off by one directory level and so uploaded NO - # experiment-level or variant-level report, only the per-task files the - # `**` patterns caught. `--run-dir /tmp/runs` makes /tmp/runs itself the - # run dir: reports land at /tmp/runs/{run,experiment}.* and - # /tmp/runs//variant.*, and per-task dirs at - # /tmp/runs//// (hence `**`, for the replicate - # segment). The Windows job's copies are correct as written because it - # passes no --run-dir and so gets a timestamped subdirectory. - path: | - /tmp/runs/run.json - /tmp/runs/run.md - /tmp/runs/experiment.html - /tmp/runs/experiment.md - /tmp/runs/experiment.json - /tmp/runs/experiment.log - /tmp/runs/*/variant.html - /tmp/runs/*/variant.md - /tmp/runs/*/variant.json - /tmp/runs/**/task.html - /tmp/runs/**/task.json - /tmp/runs/**/task.log - /tmp/runs/**/artifacts - if-no-files-found: warn - retention-days: 14 - - - name: Plaintext summary + verdict - if: always() - run: | - set -uo pipefail - run_dir=$(ls -td /tmp/runs/*/ 2>/dev/null | head -n 1 || true) - if [ -z "$run_dir" ]; then - echo "FAIL — no run directory produced" | tee -a "$GITHUB_STEP_SUMMARY" - exit 1 - fi - python - "$run_dir" <<'PY' | tee -a "$GITHUB_STEP_SUMMARY" - import json, pathlib, sys - rows = [json.loads(p.read_text(encoding='utf-8')) for p in sorted(pathlib.Path(sys.argv[1]).rglob('task.json'))] - for r in rows: - print(f"{r.get('task_id','?')}: {r.get('final_status','?')}") - ok = sum(1 for r in rows if r.get('final_status') == 'SUCCESS') - print(f"\n{ok}/{len(rows)} PASS (linux)") - sys.exit(0 if rows and ok == len(rows) else 1) - PY - # ================================================================== # Evalboard publish. Entirely additive and entirely best-effort: every # step here warns and returns 0 rather than reddening a run, because a @@ -831,12 +762,14 @@ jobs: # runs chronologically sortable, then the GitHub run id and attempt: # uniqueness is the point, since two dispatches landing in the same # second would otherwise share an id and silently overwrite each other. - # Safe to move because this is the last step to touch /tmp/runs — the - # artifact upload and the verdict have both already run. + # A COPY, not a move: this step now runs BEFORE the artifact upload and + # the verdict, which both read /tmp/runs, so the original has to survive. + # The tree is already stripped of .venv/node_modules by the cleanup step + # above, so the copy is cheap. # All-hyphen after the timestamp: a `.` before the attempt number reads # as a file extension in the URL path. blob_id="$(date -u +%Y-%m-%d_%H-%M-%S)-gha-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" - mv /tmp/runs "/tmp/$blob_id" || { + cp -a /tmp/runs "/tmp/$blob_id" || { echo "::warning::could not stage /tmp/runs for upload — no evalboard link for this run" exit 0 } @@ -861,12 +794,12 @@ jobs: else when="about 14 days from now" fi - # Prepended, not appended: run.md is ~40 lines of tables, so an appended - # link lands below the fold. Moving this step earlier is not an option, - # because it MOVES /tmp/runs, which the artifact upload and the verdict - # both read. $GITHUB_STEP_SUMMARY is an ordinary file; truncate-and-rewrite - # rather than `mv` so the runner keeps tracking the same inode. - summary_tmp="$(mktemp)" + # $GITHUB_STEP_SUMMARY is a SEPARATE file per step, and GitHub renders the + # job summary by concatenating them in step order. So the only way to put + # this block first is for the step itself to run before the one that + # appends run.md, which is why the publish moved above it and copies the + # run dir rather than moving it. Rewriting the file in place cannot work: + # a step only ever sees its own. { echo "### Evalboard" echo "" @@ -874,12 +807,82 @@ jobs: echo "" echo "Covers the **Linux** tasks of this dispatch only. Deleted $when by the 14-day lifecycle rule on the \`$AZURE_BLOB_CONTAINER\` container." echo "" - cat "$GITHUB_STEP_SUMMARY" 2> /dev/null || true - } > "$summary_tmp" - cat "$summary_tmp" > "$GITHUB_STEP_SUMMARY" - rm -f "$summary_tmp" + } >> "$GITHUB_STEP_SUMMARY" + # Annotations render at the top of the run page, above the job list, so + # the link is reachable without opening the summary at all. + echo "::notice title=Evalboard::$url" echo "Evalboard: $url" + + # The action reports where run.md is and leaves the writing to the consumer, + # which is what lets this land AFTER the redaction above. Capped at 1 MB: + # GitHub rejects a job summary over 1 MiB outright, losing the verdict too. + # + # Read from the action's `run-md-path` output rather than rebuilding the + # path, with the literal as a fallback: a composite action's outputs may not + # propagate from a step that exited non-zero, and this step runs on failure. + - name: Append the run report to the job summary + if: always() + env: + RUN_MD: ${{ steps.eval.outputs.run-md-path }} + run: | + set -uo pipefail + run_md="${RUN_MD:-/tmp/runs/run.md}" + if [ -f "$run_md" ]; then + head -c 1000000 "$run_md" >> "$GITHUB_STEP_SUMMARY" + else + echo "_No run.md produced._" >> "$GITHUB_STEP_SUMMARY" + fi + + - name: Upload eval report + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: coder-eval-linux-${{ github.run_id }} + # These were off by one directory level and so uploaded NO + # experiment-level or variant-level report, only the per-task files the + # `**` patterns caught. `--run-dir /tmp/runs` makes /tmp/runs itself the + # run dir: reports land at /tmp/runs/{run,experiment}.* and + # /tmp/runs//variant.*, and per-task dirs at + # /tmp/runs//// (hence `**`, for the replicate + # segment). The Windows job's copies are correct as written because it + # passes no --run-dir and so gets a timestamped subdirectory. + path: | + /tmp/runs/run.json + /tmp/runs/run.md + /tmp/runs/experiment.html + /tmp/runs/experiment.md + /tmp/runs/experiment.json + /tmp/runs/experiment.log + /tmp/runs/*/variant.html + /tmp/runs/*/variant.md + /tmp/runs/*/variant.json + /tmp/runs/**/task.html + /tmp/runs/**/task.json + /tmp/runs/**/task.log + /tmp/runs/**/artifacts + if-no-files-found: warn + retention-days: 14 + + - name: Plaintext summary + verdict + if: always() + run: | + set -uo pipefail + run_dir=$(ls -td /tmp/runs/*/ 2>/dev/null | head -n 1 || true) + if [ -z "$run_dir" ]; then + echo "FAIL — no run directory produced" | tee -a "$GITHUB_STEP_SUMMARY" + exit 1 + fi + python - "$run_dir" <<'PY' | tee -a "$GITHUB_STEP_SUMMARY" + import json, pathlib, sys + rows = [json.loads(p.read_text(encoding='utf-8')) for p in sorted(pathlib.Path(sys.argv[1]).rglob('task.json'))] + for r in rows: + print(f"{r.get('task_id','?')}: {r.get('final_status','?')}") + ok = sum(1 for r in rows if r.get('final_status') == 'SUCCESS') + print(f"\n{ok}/{len(rows)} PASS (linux)") + sys.exit(0 if rows and ok == len(rows) else 1) + PY + run-windows: needs: partition if: needs.partition.outputs.windows_count != '0' From 63b780eabf7b184a3c2b5b65bf96683383924f15 Mon Sep 17 00:00:00 2001 From: Bai Li Date: Tue, 1 Sep 2026 15:41:52 -0700 Subject: [PATCH 8/9] fix(ci): grade the run directory, not its newest variant `--run-dir /tmp/runs` makes /tmp/runs the run dir itself, so its subdirectories are VARIANTS. The verdict step selected one with `ls -td /tmp/runs/*/ | head -n 1`, which is correct only while nightly.yaml declares a single variant: a second one would silently grade a single arm and could report a red run green. This is the same off-by-one the artifact globs in this branch already fix, one step lower. Co-Authored-By: Claude Opus 5 --- .github/workflows/run-coder-eval.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run-coder-eval.yml b/.github/workflows/run-coder-eval.yml index 9c7a112fe6..0737c963ae 100644 --- a/.github/workflows/run-coder-eval.yml +++ b/.github/workflows/run-coder-eval.yml @@ -868,8 +868,13 @@ jobs: if: always() run: | set -uo pipefail - run_dir=$(ls -td /tmp/runs/*/ 2>/dev/null | head -n 1 || true) - if [ -z "$run_dir" ]; then + # /tmp/runs IS the run dir (--run-dir above), so its subdirectories are + # VARIANTS. `ls -td */ | head -n 1` picked one of them: correct only + # while nightly.yaml declares a single variant, and a second variant + # would silently grade one arm and could turn a red run green. Same + # off-by-one the artifact globs carry a fix for. + run_dir=/tmp/runs + if [ ! -d "$run_dir" ] || [ -z "$(ls -A "$run_dir" 2>/dev/null)" ]; then echo "FAIL — no run directory produced" | tee -a "$GITHUB_STEP_SUMMARY" exit 1 fi From ea72164548fca753f0f95bb495292eb22a313cb9 Mon Sep 17 00:00:00 2001 From: Bai Li Date: Tue, 1 Sep 2026 16:03:13 -0700 Subject: [PATCH 9/9] ci(coder-eval): consume the published action at @v0 The temporary SHA pin pointed at the PR branch of coder_eval#147, which is now merged and released as 0.11.6 with v0 promoted to it, so the eight-input surface this workflow passes is what v0 serves. Co-Authored-By: Claude Opus 5 --- .github/workflows/run-coder-eval.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/run-coder-eval.yml b/.github/workflows/run-coder-eval.yml index 0737c963ae..df40b5bb73 100644 --- a/.github/workflows/run-coder-eval.yml +++ b/.github/workflows/run-coder-eval.yml @@ -620,9 +620,12 @@ jobs: # travels with the run into the artifact and the blob. - name: Run coder-eval id: eval - # TEMPORARY — REVERT TO @v0 BEFORE MERGING. `v0` still serves the old - # ten-input action, which would silently ignore every input below. - uses: UiPath/coder_eval@a747ce353111beadd51a6370b9e8e2dba88f4133 # bai/action-inputs-and-gha-source + # A moving major tag, not a SHA like the third-party actions above. + # First-party and in this org: `v0` is the contract coder_eval's release + # promotes, and pinning a SHA here would freeze the input surface while + # `version:` below keeps moving, which is the mismatch that makes GitHub + # silently ignore inputs. Breaking input changes land as `v1`. + uses: UiPath/coder_eval@v0 with: version: ${{ needs.partition.outputs.coder_eval_version }} working-directory: tests