From bb94ee783c41cd21ca54b149c958ecc6d8fe6114 Mon Sep 17 00:00:00 2001 From: Bai Li Date: Wed, 23 Sep 2026 14:54:45 -0700 Subject: [PATCH 1/2] chore(tests): bump coder-eval to 0.12.7 Picks up the command_executed fix that searches the whole command instead of only its first 2000 characters, plus the Opus 5.5 rate card. Co-Authored-By: Claude Opus 5.5 --- tests/.coder-eval-version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/.coder-eval-version b/tests/.coder-eval-version index e01e0ddd8e..e2e3067ddc 100644 --- a/tests/.coder-eval-version +++ b/tests/.coder-eval-version @@ -1 +1 @@ -0.12.4 +0.12.7 From 3b3ca8346528e5133400b2e5bcd78d415ef9b558 Mon Sep 17 00:00:00 2001 From: Bai Li Date: Wed, 23 Sep 2026 14:54:46 -0700 Subject: [PATCH 2/2] test(activation): re-baseline the activation gate on claude-opus-5-5 Measured each skill's full positive set on claude-opus-5-5 at max_turns: 1, the gate's own method, for the move of the claude-code evals from Sonnet 5 to Opus 5.5. Co-Authored-By: Claude Opus 5.5 --- tests/scripts/activation_gate.py | 34 +++++++++++--------------------- 1 file changed, 12 insertions(+), 22 deletions(-) diff --git a/tests/scripts/activation_gate.py b/tests/scripts/activation_gate.py index 6d53552155..194ebe80d8 100755 --- a/tests/scripts/activation_gate.py +++ b/tests/scripts/activation_gate.py @@ -27,8 +27,8 @@ # simulated user), which must not move with the agent under test. AGENT_MODEL = os.environ.get("AGENT_MODEL", "").strip() -# Rounded recall.yes baseline (in %) per skill, measured 2026-06-17 over each -# skill's FULL positive set on claude-sonnet-4-6 via Bedrock at max_turns: 1 — +# Rounded recall.yes baseline (in %) per skill, measured 2026-09-23 over each +# skill's FULL positive set on claude-opus-5-5 via Bedrock at max_turns: 1 — # the same model and full-set measurement the gate itself runs. The gate task # pins run_limits.max_turns: 1 (task layer overrides the experiment's 3 via # per-key field-merge) so baseline and gate stay directly comparable; at 3 @@ -45,30 +45,20 @@ "uipath-mcp-servers": 100, "uipath-tasks": 100, "uipath-human-in-the-loop": 100, - "uipath-rpa": 100, + "uipath-rpa": 90, "uipath-test": 100, "uipath-platform": 100, "uipath-maestro-flow": 95, - "uipath-maestro-bpmn": 95, - "uipath-admin": 95, - "uipath-review": 95, - # uipath-planner re-measured 2026-08-07 on the current gate model - # (claude-sonnet-5). The prior 95% figure predates the #2132 model - # retarget, and no PR between the retarget and this measurement changed - # planner frontmatter, so the gate never ran on the new model. Measured - # recall over the full positive set: main's own unchanged frontmatter - # 65.9% and 59.3% (two dispatches: actions/runs/31219477420, - # actions/runs/31220789082); the planner-sole-sdd-author branch 59.3%, - # 52.7%, 51.6%. Run-to-run spread is ~7pp on this skill's ambiguous - # positives, so 60 sits between the two arms' means; DROP_PP absorbs the - # spread. Re-baseline again after the next full activation run. - "uipath-planner": 60, + "uipath-maestro-bpmn": 100, + "uipath-admin": 100, + "uipath-review": 90, + "uipath-planner": 95, "uipath-coded-apps": 90, - "uipath-solution": 90, - "uipath-agents": 90, - "uipath-maestro-case": 90, - "uipath-api-workflow": 90, - "uipath-functions": 95, + "uipath-solution": 100, + "uipath-agents": 100, + "uipath-maestro-case": 95, + "uipath-api-workflow": 100, + "uipath-functions": 100, } DROP_PP = 10