chore(coder-eval): bump to 0.11.1 and migrate to directory-only references

.github/workflows/activation-gate.yml

@@ -26,7 +26,7 @@ permissions:
 # tests/.coder-eval-version still resolves — see smoke-skills.yml for the full
 # rationale. Workflow-level so any job added later inherits it.
 env:
-  SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS: coder-eval
+  SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS: coder-eval,anthropic
 
 jobs:
   detect:

.github/workflows/run-coder-eval.yml

@@ -100,7 +100,7 @@ on:
 # for the full rationale. Workflow-level so both the Linux and Windows
 # install steps, and any job added later, inherit it.
 env:
-  SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS: coder-eval
+  SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS: coder-eval,anthropic
 
 jobs:
   partition:
@@ -337,7 +337,21 @@ jobs:
           CLIENT_SECRET:            ${{ secrets.UIPATH_ROPC_CLIENT_SECRET }}
           CE_USERNAME:              ${{ secrets.UIPATH_BOT_USERNAME }}
           CE_PASSWORD:              ${{ secrets.UIPATH_BOT_PASSWORD }}
-        run: bash .github/scripts/refresh-auth.sh
+        run: |
+          bash .github/scripts/refresh-auth.sh
+          # coder-eval 0.11+ drops DAC_OVERRIDE/DAC_READ_SEARCH from task
+          # containers (reference anti-cheat), so container-root can no longer
+          # read the host-owned ~/.uipath extra_mount (mounted by
+          # experiments/nightly.yaml) by capability. Widen it recursively via
+          # the "other" bits so the mounted .auth file (mode 600) is readable —
+          # mirrors coder-eval's own grant_container_access.
+          chmod -R o+rwX "$HOME/.uipath"
+
+      # coder-eval 0.11 isolates $TASK_DIR to the leaf task folder, so graders
+      # can no longer import the group-level _shared/ helper by walking up the
+      # tree. Co-locate a copy of each group's _shared into its task dirs first.
+      - name: Stage _shared grader helpers
+        run: bash tests/scripts/stage_shared.sh
 
       # TASK_GLOBS here is the partition-resolved list of *file paths* (not
       # globs). Word-splitting is intentional — do not quote.

.github/workflows/smoke-rpa-skills.yml

@@ -19,7 +19,7 @@ on:
 # tests/.coder-eval-version still resolves — see smoke-skills.yml for the full
 # rationale. Workflow-level so any job added later inherits it.
 env:
-  SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS: coder-eval
+  SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS: coder-eval,anthropic
 
 jobs:
   detect:

.github/workflows/smoke-skills.yml

@@ -24,8 +24,14 @@ on:
 # fails. coder_eval is a first-party internal package from the Azure Artifacts
 # feed, not public-registry supply-chain surface, so exclude it from the age
 # check only. Every other safe-chain check still applies.
+#
+# `anthropic` is excluded for the same reason, transitively: coder-eval >=0.11.1
+# pins anthropic>=1.0.0 (a fresh major), which the age gate suppresses — leaving
+# the resolver with only anthropic<=0.123.0 and "requirements are unsatisfiable"
+# until 1.0.0 ages out. The Anthropic SDK is the official first-party client, so
+# exempting it from the age check (only) is the same low-risk call as coder-eval.
 env:
-  SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS: coder-eval
+  SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS: coder-eval,anthropic
 
 jobs:
   detect:
@@ -401,6 +407,13 @@ jobs:
           set -euo pipefail
           bash .github/scripts/refresh-auth.sh
 
+          # coder-eval 0.11+ drops DAC_OVERRIDE/DAC_READ_SEARCH from task
+          # containers (reference anti-cheat), so container-root can no longer
+          # read the host-owned ~/.uipath extra_mount by capability. Widen it
+          # recursively via the "other" bits so the mounted .auth file (mode
+          # 600) is readable — mirrors coder-eval's own grant_container_access.
+          chmod -R o+rwX "$HOME/.uipath"
+
           # Fail once at setup, with a direct error, rather than letting every
           # guardrail task discover an unusable auth context independently.
           docker run --rm \
@@ -421,6 +434,12 @@ jobs:
           AGENT_MODEL: ${{ vars.CLAUDE_CODE_MODEL }}
         run: ': "${AGENT_MODEL:?unset — set the CLAUDE_CODE_MODEL repo variable}"'
 
+      # coder-eval 0.11 isolates $TASK_DIR to the leaf task folder, so graders
+      # can no longer import the group-level _shared/ helper by walking up the
+      # tree. Co-locate a copy of each group's _shared into its task dirs first.
+      - name: Stage _shared grader helpers
+        run: bash tests/scripts/stage_shared.sh
+
       - name: Run smoke tests
         env:
           SKILLS_REPO_PATH: ${{ github.workspace }}

tests/.coder-eval-version

@@ -1 +1 @@
-0.10.2
+0.11.1

tests/scripts/stage_shared.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+# Co-locate each skill group's `_shared/` helper package into every task dir
+# whose grader imports it, so `from _shared ...` resolves under coder-eval 0.11+.
+#
+# Why this exists: coder-eval 0.11 stages $TASK_DIR as an ISOLATED copy of just
+# the leaf task folder (/work/task_dir) — the reference anti-cheat deliberately
+# no longer exposes the task dir's parent. But ~215 graders import the
+# group-level `tests/tasks/<group>/_shared/` package by walking UP out of the
+# task dir (`sys.path.insert(0, .../../..)`), which now points at nothing in the
+# container -> ModuleNotFoundError.
+#
+# Fix, with no grader changes: copy the group's `_shared/` INTO the task dir so
+# it travels inside the /work/task_dir copy. `python3 $TASK_DIR/check.py` puts
+# the script's own dir on sys.path[0], so a co-located `_shared/` is importable
+# regardless of the grader's (now-inert) up-walk. A symlink would NOT work —
+# coder-eval's copytree ignores symlinks.
+#
+# Run on the CI checkout BEFORE `coder-eval run`. Idempotent. Ephemeral: it
+# mutates the checked-out tree, not the committed repo.
+set -euo pipefail
+
+cd "$(git rev-parse --show-toplevel)"
+
+count=0
+while IFS= read -r f; do
+  dir=$(dirname "$f")
+  # Never recurse into a _shared package itself.
+  case "$dir" in */_shared | */_shared/*) continue ;; esac
+  # group = tests/tasks/<group>
+  group=$(printf '%s\n' "$f" | sed -E 's#^(tests/tasks/[^/]+)/.*#\1#')
+  src="$group/_shared"
+  [ -d "$src" ] || continue          # group has no _shared -> nothing to stage
+  [ -e "$dir/_shared" ] && continue  # already present (idempotent / hand-authored)
+  cp -R "$src" "$dir/_shared"
+  count=$((count + 1))
+done < <(git grep -lE 'from _shared|import _shared' -- 'tests/tasks/**/*.py')
+
+echo "stage_shared: co-located _shared into ${count} task dir(s)"

tests/tasks/uipath-troubleshoot/CLAUDE.md

@@ -146,7 +146,8 @@ A single scenario (leaf) under its group folder:
 ```
 tests/tasks/uipath-troubleshoot/<group>/<scenario-name>/
 ├── task.yaml                    # tags, mock_path_dirs, llm_judge criteria
-├── RESOLUTION.md                # ground truth for the LLM judge
+├── reference/                   # directory-only reference (coder-eval >=0.11); never shown to the agent
+│   └── RESOLUTION.md            # ground truth for the LLM judge
 ├── data/                        # short dir names keep Windows paths under MAX_PATH (260)
 │   └── m/
 │       └── r/
@@ -285,7 +286,7 @@ Every scenario MUST include exactly TWO criteria, and ONLY these two:
 
 Concrete failure mode this rule prevents: an agent that reaches the correct root cause via `jobs logs` (skipping `jobs get`) is graded FAILURE solely because a `command_executed` rule required `jobs get d5fed611`. The conclusion was right; the path was different. Brittle.
 
-Anything task-specific the test needs to verify goes in `RESOLUTION.md`. The judge reads it via `include_reference` and compares.
+Anything task-specific the test needs to verify goes in `RESOLUTION.md`. The judge reads it via the `files: [$REFERENCE_DIR/RESOLUTION.md]` entry and compares.
 
 #### Judge configuration — single canonical shape for every task
 
@@ -298,7 +299,9 @@ Every `llm_judge` criterion across all troubleshoot tasks uses the **same** prom
   description: "Agent's diagnosis matches RESOLUTION.md"
   weight: 3.0
   pass_threshold: 0.7
-  include_reference: true
+  include_reference: false
+  files:
+    - $REFERENCE_DIR/RESOLUTION.md
   include_dialog: true
   include_agent_output: true
   prompt: |
@@ -316,9 +319,9 @@ Every `llm_judge` criterion across all troubleshoot tasks uses the **same** prom
     Return JSON: {"score": <float>, "rationale": "<one sentence>"}
 ```
 
-**What the judge sees** (all three flags MUST be `true`):
+**What the judge sees:**
 
-- `include_reference: true` — passes `RESOLUTION.md` (the file named under `reference:` at the task root)
+- `include_reference: false` + `files: [$REFERENCE_DIR/RESOLUTION.md]` — passes **only** `reference/RESOLUTION.md`, not the whole `reference:` directory. (`include_reference: true` would inline the entire directory as a separate block, so anything else added to `reference/` later would silently reach the judge — and it would duplicate the file if combined with this `files:` entry.)
 - `include_dialog: true` — passes the full user<->agent dialog, every turn
 - `include_agent_output: true` — passes the agent's final user-facing response
 
@@ -334,7 +337,7 @@ That is **all** the context the judge gets. The contract: agent's diagnosis (whe
 
 **Why:** the skill's internal state (`notes.md`, `raw/`) is bookkeeping for the investigation, not a deliverable. Multiple playbooks may legitimately match a signal; the fast path legitimately stops at the first branch its evidence confirms (see SKILL.md § Verification checklist). Grading on internal-state shape punishes correct skill behavior. What the agent **presents** is the contract.
 
-**Where task-specific guidance lives:** `RESOLUTION.md`. If the judge needs to know that a specific fix must name "AlterIfDisabled = True" or that "Test Heals pool" is wrong — that goes in `RESOLUTION.md` as the authoritative root cause + fix. The judge reads it via `include_reference` and grades against it.
+**Where task-specific guidance lives:** `RESOLUTION.md`. If the judge needs to know that a specific fix must name "AlterIfDisabled = True" or that "Test Heals pool" is wrong — that goes in `RESOLUTION.md` as the authoritative root cause + fix. The judge reads it via the `files: [$REFERENCE_DIR/RESOLUTION.md]` entry and grades against it.
 
 **Lean default for every new scenario:**
 
@@ -350,7 +353,9 @@ success_criteria:
     description: "Agent's diagnosis matches RESOLUTION.md"
     weight: 3.0
     pass_threshold: 0.7
-    include_reference: true
+    include_reference: false
+    files:
+      - $REFERENCE_DIR/RESOLUTION.md
     include_dialog: true
     include_agent_output: true
     prompt: |

tests/tasks/uipath-troubleshoot/_shared/scripts/generate_scenario.py

@@ -15,8 +15,8 @@
 (e.g. `InputArguments={}`), no hypothesis labels. Keep those descriptions
 procedural ("Agent fetches the job's execution details."). The exception and
 stack trace belong ONLY in the recorded `jobs get` / `jobs logs` payloads -
-that is the evidence the agent must diagnose from. `RESOLUTION.md` lives at the
-task root and is NOT staged; `README.md` lives in `data/m/r/`, which IS staged
+that is the evidence the agent must diagnose from. `RESOLUTION.md` lives under
+`reference/` and is NOT staged; `README.md` lives in `data/m/r/`, which IS staged
 but is deleted from the sandbox by `m/seal` before the agent starts. Both may
 therefore describe the root cause freely.
 
@@ -132,7 +132,7 @@ def _find_repo_root() -> Path:
     fail_on_error: true
 
 reference:
-  file: RESOLUTION.md
+  directory: reference
 
 initial_prompt: |
 {initial_prompt_indented}
@@ -148,7 +148,9 @@ def _find_repo_root() -> Path:
     description: "Agent's diagnosis matches RESOLUTION.md"
     weight: 3.0
     pass_threshold: 0.7
-    include_reference: true
+    include_reference: false
+    files:
+      - $REFERENCE_DIR/RESOLUTION.md
     include_agent_output: true
     prompt: |
       Grade the agent's final answer against the attached RESOLUTION.md.
@@ -727,7 +729,7 @@ def render_dry_run(plan: dict) -> str:
     out.append("Files that would be written:")
     base = plan["output_dir"]
     out.append(f"  {base / 'task.yaml'}                             ({len(plan['task_yaml'])} bytes)")
-    out.append(f"  {base / 'RESOLUTION.md'}                         ({len(plan['resolution_md'])} bytes)")
+    out.append(f"  {base / 'reference' / 'RESOLUTION.md'}           ({len(plan['resolution_md'])} bytes)")
     out.append(f"  {base / 'data' / 'm' / 'r' / 'README.md'}         ({len(plan['readme_md'])} bytes)")
     out.append(f"  {base / 'data' / 'm' / 'r' / 'manifest.json'}")
     out.append(f"  {base / 'data' / 'm' / 'r' / '<hash>.json'} x {len(plan['fixtures'])}")
@@ -745,7 +747,12 @@ def apply_plan(plan: dict) -> None:
     base.mkdir(parents=True, exist_ok=True)
 
     (base / "task.yaml").write_text(plan["task_yaml"], encoding="utf-8")
-    (base / "RESOLUTION.md").write_text(plan["resolution_md"], encoding="utf-8")
+    # RESOLUTION.md lives under reference/ so coder-eval can permission-gate the
+    # whole directory as a unit (directory-only references, coder-eval >=0.11);
+    # it is the llm_judge ground truth and is never staged/shown to the agent.
+    reference_dir = base / "reference"
+    reference_dir.mkdir(parents=True, exist_ok=True)
+    (reference_dir / "RESOLUTION.md").write_text(plan["resolution_md"], encoding="utf-8")
 
     fixtures_dir = base / "data" / "m" / "r"
     fixtures_dir.mkdir(parents=True, exist_ok=True)

tests/tasks/uipath-troubleshoot/activity-packages/classic-browserscope-efail/task.yaml

@@ -36,7 +36,7 @@ pre_run:
     fail_on_error: true
 
 reference:
-  file: RESOLUTION.md
+  directory: reference
 
 initial_prompt: |
   Why did my last job from folder Finance fail?
@@ -52,7 +52,9 @@ success_criteria:
     description: "Agent's diagnosis matches RESOLUTION.md"
     weight: 3.0
     pass_threshold: 0.7
-    include_reference: true
+    include_reference: false
+    files:
+      - $REFERENCE_DIR/RESOLUTION.md
     include_dialog: true
     include_agent_output: true
     prompt: |

tests/tasks/uipath-troubleshoot/activity-packages/classic-click-noop-simulate/task.yaml

@@ -38,7 +38,7 @@ pre_run:
     fail_on_error: true
 
 reference:
-  file: RESOLUTION.md
+  directory: reference
 
 initial_prompt: |
   My ClassicClaimsSubmit job in folder Finance finished Successful last night,
@@ -56,7 +56,9 @@ success_criteria:
     description: "Agent's diagnosis matches RESOLUTION.md"
     weight: 3.0
     pass_threshold: 0.7
-    include_reference: true
+    include_reference: false
+    files:
+      - $REFERENCE_DIR/RESOLUTION.md
     include_dialog: true
     include_agent_output: true
     prompt: |

tests/tasks/uipath-troubleshoot/activity-packages/classic-invoke-arg-mismatch/task.yaml

@@ -35,7 +35,7 @@ pre_run:
     fail_on_error: true
 
 reference:
-  file: RESOLUTION.md
+  directory: reference
 
 initial_prompt: |
   My last job from folder Shared faulted on an Invoke Workflow File activity.
@@ -52,7 +52,9 @@ success_criteria:
     description: "Agent's diagnosis matches RESOLUTION.md"
     weight: 3.0
     pass_threshold: 0.7
-    include_reference: true
+    include_reference: false
+    files:
+      - $REFERENCE_DIR/RESOLUTION.md
     include_dialog: true
     include_agent_output: true
     prompt: |

tests/tasks/uipath-troubleshoot/activity-packages/classic-invoke-cache-error-7/task.yaml

@@ -35,7 +35,7 @@ pre_run:
     fail_on_error: true
 
 reference:
-  file: RESOLUTION.md
+  directory: reference
 
 initial_prompt: |
   I can't run or debug this project in Studio anymore. The Output panel throws:
@@ -57,7 +57,9 @@ success_criteria:
     description: "Agent's diagnosis matches RESOLUTION.md"
     weight: 3.0
     pass_threshold: 0.7
-    include_reference: true
+    include_reference: false
+    files:
+      - $REFERENCE_DIR/RESOLUTION.md
     include_dialog: true
     include_agent_output: true
     prompt: |

tests/tasks/uipath-troubleshoot/activity-packages/classic-invoke-child-throws/task.yaml

@@ -35,7 +35,7 @@ pre_run:
     fail_on_error: true
 
 reference:
-  file: RESOLUTION.md
+  directory: reference
 
 initial_prompt: |
   My last job from folder Shared faulted. It runs an Invoke Workflow File and
@@ -52,7 +52,9 @@ success_criteria:
     description: "Agent's diagnosis matches RESOLUTION.md"
     weight: 3.0
     pass_threshold: 0.7
-    include_reference: true
+    include_reference: false
+    files:
+      - $REFERENCE_DIR/RESOLUTION.md
     include_dialog: true
     include_agent_output: true
     prompt: |

tests/tasks/uipath-troubleshoot/activity-packages/classic-invoke-ignored-file/task.yaml

@@ -35,7 +35,7 @@ pre_run:
     fail_on_error: true
 
 reference:
-  file: RESOLUTION.md
+  directory: reference
 
 initial_prompt: |
   My process runs fine in Studio on my machine, but the last job from folder
@@ -53,7 +53,9 @@ success_criteria:
     description: "Agent's diagnosis matches RESOLUTION.md"
     weight: 3.0
     pass_threshold: 0.7
-    include_reference: true
+    include_reference: false
+    files:
+      - $REFERENCE_DIR/RESOLUTION.md
     include_dialog: true
     include_agent_output: true
     prompt: |

tests/tasks/uipath-troubleshoot/activity-packages/classic-invoke-persistence/task.yaml

@@ -35,7 +35,7 @@ pre_run:
     fail_on_error: true
 
 reference:
-  file: RESOLUTION.md
+  directory: reference
 
 initial_prompt: |
   My last job from folder Shared faulted on an Invoke Workflow File activity.
@@ -52,7 +52,9 @@ success_criteria:
     description: "Agent's diagnosis matches RESOLUTION.md"
     weight: 3.0
     pass_threshold: 0.7
-    include_reference: true
+    include_reference: false
+    files:
+      - $REFERENCE_DIR/RESOLUTION.md
     include_dialog: true
     include_agent_output: true
     prompt: |