fix(criteria): search the whole command in command_executed, not just the first 2000 chars #737

Workflow file for this run

.github/workflows/pr-checks.yml at 83a4965

	name: PR Quality Checks

	on:
	pull_request:
	branches: [main, develop]
	merge_group:
	push:
	branches: [main, develop]
	workflow_dispatch:
	inputs:
	delegate_uipath_env:
	description: "UiPath cloud environment to ROPC-login the Delegate live job against"
	required: false
	type: choice
	default: "alpha"
	options:
	- "alpha"
	- "staging"
	- "production"

	# Cancel outdated runs when new commits are pushed
	concurrency:
	group: ${{ github.workflow }}-${{ github.ref }}
	cancel-in-progress: true

	# Least-privilege default. No job in this workflow writes to the repo via
	# GITHUB_TOKEN — actions/upload-artifact and actions/cache use their own APIs.
	permissions:
	contents: read

	# Hard-disable usage telemetry for the ENTIRE workflow. The app ships a baked-in
	# default Application Insights connection string (so a fresh install reports with
	# no config), which means CI's real `coder-eval run` invocations would otherwise
	# emit to the production telemetry resource and pollute dashboards. The in-process
	# pytest autouse guard can't reach those subprocesses, so disable at the env level
	# here — TELEMETRY_ENABLED is the single canonical disable gate.
	env:
	TELEMETRY_ENABLED: "false"
	# The `uipath-*` pool enforces a minimum package-age safe-chain check on installs.
	# Workflow-level so every installing job inherits it; per-job copies are how some
	# jobs previously ended up with no exclusions at all. The literal is the operative
	# value — no secret of that name exists at repo or org level, so the bare `secrets.`
	# reference this replaced resolved to an empty list. (Image builds carry their own
	# list in docker/Dockerfile; deliberately not the same set.)
	SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS: ${{ secrets.SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS \|\| 'openai-codex-cli-bin,openai-codex' }}

	jobs:
	quality-gate:
	name: Quality Gate (Format, Lint, Type, Test, Security)
	# Fork PRs go to stock GitHub-hosted runners: this job runs the PR's own
	# `uv.lock` build hooks and test files, and the repo is public, so untrusted code
	# must not land on the shared pool image. Everything else uses the pool.
	runs-on: ${{ (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository) && 'ubuntu-latest' \|\| 'uipath-ubuntu-latest' }}
	timeout-minutes: 10

	# A dummy Anthropic key so CI tests that construct a client don't fail on a
	# missing key.
	env:
	ANTHROPIC_API_KEY: "sk-ant-test-dummy-key-for-ci-tests-only"

	steps:
	- name: Checkout code
	uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

	- name: Set up Python 3.13
	uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
	with:
	python-version: "3.13"

	- name: Cache dependencies
	uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
	with:
	# ``.venv`` is deliberately NOT cached: its interpreter symlinks
	# point at the exact hosted-Python patch, so a patch bump (e.g.
	# 3.13.13 -> 3.13.14) leaves a dangling ``.venv/bin/python`` and
	# every ``.venv/bin/*`` fails with exit 127 (cannot execute).
	# ``uv sync --frozen`` rehydrates ``.venv`` from the cached wheels.
	path: \|
	~/.cache/uv
	~/.cache/pip
	.pytest_cache
	.ruff_cache
	key: ${{ runner.os }}-py3.13-${{ hashFiles('pyproject.toml', 'uv.lock') }}
	restore-keys: \|
	${{ runner.os }}-py3.13-

	- name: Install uv
	run: \|
	python -m pip install --upgrade "pip>=26.2" # PYSEC-2026-3721 fix floor
	pip install uv

	- name: Install project dependencies (hash-verified from uv.lock)
	# --extra harbor: pyright below type-checks src/coder_eval/harbor/agent.py
	# against harbor's real types, not a scoped ignore.
	run: uv sync --frozen --extra dev --extra uipath --extra codex --extra litellm --extra harbor

	# PHASE 1: Fast checks (fail early)
	- name: Check code formatting (ruff format)
	run: .venv/bin/ruff format --check src/ tests/

	- name: Lint code (ruff check)
	run: .venv/bin/ruff check src/ tests/

	- name: Custom architectural lint (CE001+)
	run: .venv/bin/pytest tests/test_custom_lint.py -v --tb=short --no-header -p no:warnings

	- name: Prose budget (docstring/comment ratchet)
	run: .venv/bin/python -m tests.lint.prose_budget

	# PHASE 2: Type checking
	- name: Type check with pyright
	run: .venv/bin/pyright

	# The CE036 contract engine lives under tests/, which [tool.pyright] excludes
	# -- and `exclude` beats both a CLI file arg and an `include` entry, so it can
	# only be reached through a config of its own, derived from [tool.pyright] so
	# the two passes cannot drift. Mirrors `make typecheck`.
	- name: Type check the CE036 contract engine
	run: \|
	.venv/bin/python -m tests.lint.pyright_config .pyright-tests.json
	.venv/bin/pyright -p .pyright-tests.json

	# PHASE 3: Security scanning
	- name: Security - Dependency vulnerabilities (pip-audit)
	run: .venv/bin/pip-audit --desc --skip-editable --ignore-vuln CVE-2026-4539 --ignore-vuln CVE-2026-3219 --ignore-vuln PYSEC-2025-183 # pygments 2.19.2 ReDoS + pip 26.0.1 tar/ZIP ambiguity + pyjwt 2.12.1 weak-encryption (disputed by supplier; key length is application-chosen); no fixes available on PyPI yet — revisit quarterly

	- name: Security - OSV vulnerability scan (osv-scanner)
	# Complements pip-audit: pip-audit queries the PyPI advisory DB;
	# osv-scanner queries OSV.dev which aggregates GitHub Security
	# Advisories and multiple ecosystems' DBs. Running both reduces the
	# chance an advisory slips through.
	#
	# Scope: only coder_eval's own ``uv.lock`` — the same surface
	# pip-audit covers. Sibling lockfiles in this repo
	# (``evalboard/pnpm-lock.yaml``, template node_modules) are
	# intentionally out of scope here; widen scope in a follow-up once
	# those packages have a maintainer-owned bump cadence.
	#
	# Binary pinned by version + sha256 against the upstream release
	# (https://github.com/google/osv-scanner/releases) so a registry
	# compromise can't swap the binary under us.
	env:
	OSV_VERSION: "v2.3.8"
	OSV_SHA256: "bc98e15319ed0d515e3f9235287ba53cdc5535d576d24fd573978ecfe9ab92dc"
	run: \|
	set -euo pipefail
	# RUNNER_TEMP is per-job-isolated on hosted runners and properly-configured
	# self-hosted runners — avoids TOCTOU on shared /tmp between concurrent jobs.
	OSV_BIN="${RUNNER_TEMP}/osv-scanner"
	curl --fail --silent --show-error --location \
	"https://github.com/google/osv-scanner/releases/download/${OSV_VERSION}/osv-scanner_linux_amd64" \
	--output "${OSV_BIN}"
	echo "${OSV_SHA256} ${OSV_BIN}" \| sha256sum -c -
	chmod +x "${OSV_BIN}"
	"${OSV_BIN}" --config=osv-scanner.toml --lockfile=uv.lock

	- name: Security - Code analysis (bandit)
	run: .venv/bin/bandit -r src/ -ll --format json -o bandit-report.json

	- name: Upload security reports
	if: always()
	uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
	with:
	name: security-reports
	path: \|
	bandit-report.json
	retention-days: 30

	# PHASE 4: Test suite with coverage (hermetic; live tests run in the E2E job)
	- name: Run test suite with coverage
	run: \|
	.venv/bin/pytest tests/ -v \
	-m "not live and not lint" \
	--cov=coder_eval \
	--cov-report=term-missing \
	--cov-report=xml \
	--cov-fail-under=80

	- name: Upload coverage report
	uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
	with:
	name: coverage-report
	path: coverage.xml
	retention-days: 30

	# Summary
	- name: Summary
	if: always()
	run: \|
	echo "✅ Quality gate complete!"
	echo "📊 All checks passed: formatting, linting, types, security, tests"

	evalboard:
	# The dashboard's own gate. `evalboard/` ships ~460 vitest assertions, and
	# until this job existed NOTHING ran them: not a workflow, not a Makefile
	# target, not a pre-commit hook. The pricing guard was consequently red on
	# `main` for weeks while a 3x-wrong Opus rate and five unpriced in-use models
	# shipped to the board. An unrun assertion is documentation, not enforcement.
	#
	# Rate-table drift is no longer this job's concern: lib/pricing.generated.ts
	# is GENERATED from src/coder_eval/pricing.py, and CE065 in `quality-gate`
	# fails a reprice that was not regenerated. What runs here is the CONSUMPTION
	# half (pricing-generated.test.ts) — a generated file that is missing, empty
	# or narrow fails the board's own build.
	#
	# Deliberately NOT path-filtered. `paths:` is workflow-scoped in GitHub
	# Actions, and a skipped required check blocks a PR rather than passing it —
	# so the filter buys nothing and costs a merge-blocking pending status.
	name: Evalboard (Types, Tests, Build)
	# Fork-PR carve-out — see `quality-gate`. `pnpm install --frozen-lockfile` runs
	# the PR's own lockfile install scripts, same untrusted-code class.
	runs-on: ${{ (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository) && 'ubuntu-latest' \|\| 'uipath-ubuntu-latest' }}
	timeout-minutes: 15
	steps:
	- name: Checkout code
	uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

	- name: Set up Node.js 20
	uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
	with:
	node-version: "20"

	- name: Enable pnpm
	# The version comes from evalboard/package.json's `packageManager` field,
	# so corepack pins it without a second place to keep in sync.
	run: corepack enable

	- name: Install dependencies (lockfile-pinned)
	working-directory: evalboard
	run: pnpm install --frozen-lockfile

	- name: Verify (tsc --noEmit && vitest run && next build)
	working-directory: evalboard
	run: pnpm verify

	plugin-validate:
	# Proves the Claude Code plugin marketplace is installable and that the suite
	# `check-skill` scaffolds is real: the manifests pass strict validation, and the
	# bundled activation template both schema-validates and expands to one task per
	# dataset row. Needs no credentials — nothing here invokes a model.
	name: Claude Code Plugin (manifests + offline scaffold)
	runs-on: ubuntu-latest
	timeout-minutes: 10
	steps:
	- name: Checkout code
	uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

	# `claude plugin validate` ships in the Claude Code npm package.
	- name: Set up Node.js 20
	uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
	with:
	node-version: "20"

	- name: Install Claude CLI
	run: npm install -g @anthropic-ai/claude-code

	- name: Validate plugin manifest (strict)
	run: claude plugin validate ./plugins/coder-eval --strict

	- name: Validate marketplace manifest (strict)
	run: claude plugin validate . --strict

	- name: Install uv
	uses: astral-sh/setup-uv@38f3f104447c67c051c4a08e39b64a148898af3a # v4.2.0

	# Runs OUTSIDE the source tree, the way a user's copy of the template does:
	# no experiments/, no tasks/, no coder_eval checkout on the path. `plan` is a
	# schema check only (it exits 0 even when dataset.paths names a nonexistent
	# file), so the row-count assert goes through expand_dataset — otherwise this
	# step would pass even if activation-rows.jsonl were never copied.
	# Reproduce locally with: SCRATCH=$(mktemp -d) VENV=$(mktemp -d)/venv
	- name: Scaffold assert (no source tree)
	run: \|
	set -euo pipefail
	SCRATCH="$RUNNER_TEMP/scratch"
	VENV="$RUNNER_TEMP/venv"
	mkdir -p "$SCRATCH"
	cp plugins/coder-eval/reference/templates/activation.yaml "$SCRATCH/"
	cp plugins/coder-eval/reference/templates/activation-rows.jsonl "$SCRATCH/"
	# A venv (not `uv tool install`) because the expansion assert needs
	# `coder_eval` importable, not just the `coder-eval` CLI on PATH.
	uv venv "$VENV"
	VIRTUAL_ENV="$VENV" uv pip install .
	cd "$SCRATCH"
	"$VENV/bin/coder-eval" plan activation.yaml
	"$VENV/bin/python" - <<'PY'
	from pathlib import Path
	from coder_eval.orchestration.task_loader import expand_dataset, load_task
	task, _ = load_task(Path("activation.yaml"))
	rows = expand_dataset(task, Path("."))
	assert len(rows) == 6, f"expected 6 row-tasks, got {len(rows)}"
	labels = {c.expected_skill for t in rows for c in t.success_criteria}
	assert labels == {"my-skill", ""}, labels
	print(f"ok: {len(rows)} row-tasks")
	PY

	no-uipath-extra:
	# Proves that `pip install coder-eval` (without the optional `[uipath]`
	# extra) yields a working framework: imports succeed, the criterion
	# registry validates, and the uipath-specific code paths fail with a
	# clear hint instead of an import error.
	name: No-Extra Install (uipath optional)
	# Fork-PR carve-out — see the comment on `quality-gate` above.
	runs-on: ${{ (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository) && 'ubuntu-latest' \|\| 'uipath-ubuntu-latest' }}
	timeout-minutes: 5
	steps:
	- name: Checkout code
	uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

	- name: Set up Python 3.13
	uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
	with:
	python-version: "3.13"

	- name: Install uv
	run: \|
	python -m pip install --upgrade "pip>=26.2" # PYSEC-2026-3721 fix floor
	pip install uv

	- name: Install project without [uipath] extra
	run: \|
	uv venv .venv --clear
	# Only the `dev` extra is requested — `[uipath]` is deliberately omitted.
	uv pip install --python .venv/bin/python -e ".[dev]"

	- name: Verify uipath SDK is NOT installed
	run: \|
	set -e
	! .venv/bin/python -c "import uipath" 2>/dev/null
	echo "✓ uipath absent as expected"

	- name: Verify framework imports succeed without the extra
	run: \|
	.venv/bin/python -c "
	import coder_eval
	import coder_eval.models
	from coder_eval.criteria import init_criteria, CriterionRegistry
	init_criteria(validate=True)
	assert 'llm_judge' in CriterionRegistry.list_types()
	assert 'uipath_eval' in CriterionRegistry.list_types()
	print('✓ framework imports + criterion registry OK')
	"

	- name: Run optional-dependency test suite
	run: .venv/bin/pytest tests/test_optional_dependencies.py -v --no-header

	windows-smoke:
	name: Windows Smoke Test
	runs-on: uipath-windows-latest
	# 15min headroom: the smoke task itself completes in ~7min, but the
	# actions/cache post-step on Windows is slow when ``.venv`` is large.
	# We also exclude ``.venv`` from the cached paths (uv re-creates it
	# fast from the cached ``uv`` download dir), so 15min is comfortable
	# with margin for transient network slowness on the dep install.
	timeout-minutes: 15
	# Skip on fork PRs where secrets aren't available (matches e2e-smoke).
	if: github.event_name != 'pull_request' \|\| github.event.pull_request.head.repo.full_name == github.repository
	defaults:
	run:
	shell: bash

	env:
	# Job-level dummy key keeps the unit-test step deterministic and isolated
	# from real-API leakage. The e2e step below overrides to Bedrock at step scope.
	ANTHROPIC_API_KEY: "sk-ant-test-dummy-key-for-ci-tests-only"
	# Bedrock secrets at job scope so the e2e step can engage BedrockRoute.
	# See e2e-smoke for rationale on routing smoke through Bedrock.
	AWS_BEARER_TOKEN_BEDROCK: ${{ secrets.AWS_BEARER_TOKEN_BEDROCK }}
	AWS_REGION: ${{ secrets.AWS_REGION }}
	BEDROCK_MODEL: ${{ secrets.BEDROCK_MODEL }}

	steps:
	- name: Checkout code
	uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

	- name: Set up Python 3.13
	uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
	with:
	python-version: "3.13"

	- name: Cache dependencies
	uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
	with:
	# ``.venv`` is deliberately excluded: tarring/untarring tens of
	# thousands of small wheel files on Windows is what was tripping
	# the 10-minute job timeout. Caching only the uv/pip download
	# caches keeps ``uv sync --frozen`` fast (it hydrates ``.venv``
	# from the wheel cache without re-downloading) at a fraction of
	# the compress/decompress cost.
	path: \|
	~/AppData/Local/uv/cache
	~/AppData/Local/pip/Cache
	.pytest_cache
	.ruff_cache
	key: ${{ runner.os }}-py3.13-${{ hashFiles('pyproject.toml', 'uv.lock') }}
	restore-keys: \|
	${{ runner.os }}-py3.13-

	- name: Install uv
	run: \|
	python -m pip install --upgrade "pip>=26.2" # PYSEC-2026-3721 fix floor
	pip install uv

	- name: Install project dependencies (hash-verified from uv.lock)
	# --extra harbor: pyright below type-checks src/coder_eval/harbor/agent.py
	# against harbor's real types, not a scoped ignore.
	run: uv sync --frozen --extra dev --extra uipath --extra codex --extra litellm --extra harbor

	- name: Check code formatting (ruff format)
	run: .venv/Scripts/ruff format --check src/ tests/

	- name: Lint code (ruff check)
	run: .venv/Scripts/ruff check src/ tests/

	- name: Type check with pyright
	run: .venv/Scripts/pyright

	- name: Type check the CE036 contract engine
	run: \|
	.venv/Scripts/python -m tests.lint.pyright_config .pyright-tests.json
	.venv/Scripts/pyright -p .pyright-tests.json

	- name: Run test suite
	run: .venv/Scripts/pytest tests/ -v -m "not live and not lint"

	- name: Set up Node.js 20
	uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
	with:
	node-version: "20"

	- name: Install Claude CLI (Windows)
	run: npm install -g @anthropic-ai/claude-code

	- name: Verify Claude CLI on PATH (Windows)
	# Fail-fast diagnostic: if `claude` isn't resolvable here, the next
	# step's failure surfaces inside the SDK rather than at the CLI layer,
	# which is harder to triage. Cheap insurance.
	run: claude --version

	- name: Run hello_date smoke task
	env:
	# Step-level override engages BedrockRoute; job-level dummy
	# ANTHROPIC_API_KEY stays in scope for the unit-test step above.
	API_BACKEND: "bedrock"
	# No --model: under API_BACKEND=bedrock the agent layer would prepend
	# eu.anthropic./us.anthropic. to a bare alias like
	# claude-haiku-4-5-20251001 and produce an inference-profile id Bedrock
	# rejects with 400. Falling back to BEDROCK_MODEL (which is already a
	# valid Bedrock profile id) is the same pattern live-tests uses.
	run: .venv/Scripts/coder-eval run tasks/hello_date.yaml --run-dir runs/win-smoke

	- name: Verify hello_date result
	shell: bash
	run: \|
	F=runs/win-smoke/experiment.json
	test -f "$F" \|\| { echo "$F missing"; ls -la runs/ 2>/dev/null; exit 1; }
	SUCCEEDED=$(jq '[.variant_aggregates[].tasks_succeeded] \| add' "$F")
	FAILED=$(jq '[.variant_aggregates[].tasks_failed] \| add' "$F")
	echo "windows hello_date: succeeded=$SUCCEEDED failed=$FAILED"
	test "$SUCCEEDED" = "1" \|\| { echo "Expected 1 success on Windows hello_date, got $SUCCEEDED"; exit 1; }
	test "$FAILED" = "0" \|\| { echo "Windows hello_date failed"; exit 1; }

	- name: Upload Windows smoke artifacts on failure
	if: failure()
	uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
	with:
	name: windows-smoke-runs
	path: runs/win-smoke/
	retention-days: 7

	e2e-smoke:
	name: E2E Smoke Tests (Real API)
	runs-on: uipath-ubuntu-latest
	# 15 (was 10): the bucket now includes anti_cheat_reference, a driver: docker
	# task that spins its own container on top of the two image builds this job
	# already does. Headroom, not an expected duration.
	timeout-minutes: 15
	# Skip on fork PRs where secrets aren't available
	if: github.event_name != 'pull_request' \|\| github.event.pull_request.head.repo.full_name == github.repository

	env:
	# Route smoke runs through Bedrock to keep Anthropic-credit spend off the
	# critical path. ClaudeCodeAgent's BedrockRoute is exercised end-to-end
	# by the live-tests BedrockRoute step, so the same path is used here.
	# The DirectRoute path is still tested every PR by live-tests' DirectRoute
	# step, so smoke coverage of the direct-API code path is not lost.
	API_BACKEND: "bedrock"
	AWS_BEARER_TOKEN_BEDROCK: ${{ secrets.AWS_BEARER_TOKEN_BEDROCK }}
	AWS_REGION: ${{ secrets.AWS_REGION }}
	BEDROCK_MODEL: ${{ secrets.BEDROCK_MODEL }}
	# Grades smoke_system_one_judge. Not a Bedrock credential: the System One
	# judge calls TypeSafe directly and ignores the run's API backend. A
	# missing key escalates that task to ERROR rather than failing a criterion,
	# so the preflight step below fails loudly instead.
	TYPESAFE_API_KEY: ${{ secrets.TYPESAFE_API_KEY }}
	# tasks_run for --tags smoke-pass. 9 task files (hello_date, dataset_example,
	# smoke_llm_judge, smoke_agent_judge, smoke_system_one_judge, byod_smoke_test,
	# agentless_smoke_test, anti_cheat_reference, record_cli_responses);
	# dataset_example fans out to 2 inline rows, so 10 sub-tasks. If you
	# add/remove a smoke-pass task or change the dataset row count, bump these.
	#
	# anti_cheat_reference lives in a SUBDIRECTORY, which `tasks/*.yaml` does not
	# match — the smoke-pass step names its path explicitly. Keep that in sync.
	EXPECTED_SMOKE_PASS_RUN: "10"
	EXPECTED_SMOKE_PASS_SUCCEEDED: "10"
	# smoke-fail bucket: three tasks expected to fail.
	# 1. smoke_negative_path: file_contains criterion is unsatisfiable
	# (sentinel-string regression detection for success-checker).
	# 2. smoke_task_timeout: a `sleep 300` Bash call vs task_timeout: 30
	# (regression detection for the orchestrator's task_timeout watchdog).
	# 3. smoke_budget_exceeded: max_input_tokens: 1 (unsatisfiable) — guards
	# run_limits enforcement (regression detection: a disabled budget gate
	# would let the task pass criteria and land in tasks_succeeded).
	EXPECTED_SMOKE_FAIL_RUN: "3"
	EXPECTED_SMOKE_FAIL_FAILED: "3"
	# smoke-variants bucket: one task fanned out across 2 experiment variants
	# (experiments/smoke_variants.yaml) → tasks_run = 1 * 2.
	EXPECTED_SMOKE_VARIANTS_RUN: "2"
	EXPECTED_SMOKE_VARIANTS_SUCCEEDED: "2"

	steps:
	- name: Checkout code
	uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

	- name: Set up Python 3.13
	uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
	with:
	python-version: "3.13"

	- name: Set up Node.js 20
	uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
	with:
	node-version: "20"

	- name: Install Claude CLI
	run: npm install -g @anthropic-ai/claude-code

	- name: Cache dependencies
	uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
	with:
	# ``.venv`` excluded: a hosted-Python patch bump orphans its
	# interpreter symlink (dangling ``.venv/bin/python`` -> exit 127).
	# ``uv sync --frozen`` rehydrates it from the cached wheels.
	path: \|
	~/.cache/uv
	~/.cache/pip
	key: ${{ runner.os }}-py3.13-e2e-${{ hashFiles('pyproject.toml', 'uv.lock') }}
	restore-keys: \|
	${{ runner.os }}-py3.13-e2e-
	${{ runner.os }}-py3.13-

	- name: Install uv
	run: \|
	python -m pip install --upgrade "pip>=26.2" # PYSEC-2026-3721 fix floor
	pip install uv

	- name: Install project dependencies (hash-verified from uv.lock)
	run: uv sync --frozen --extra dev --extra uipath --extra codex

	- name: Build coder-eval-agent base Docker image
	run: make docker-image

	- name: Build BYOD template Docker image
	run: docker build -t byod-custom-image:0.1.0 templates/byod_smoke_test/

	# All three smoke steps below omit --model. Under API_BACKEND=bedrock,
	# the agent layer prepends eu.anthropic./us.anthropic. to a bare alias
	# like claude-haiku-4-5-20251001 and produces an inference-profile id
	# that Bedrock rejects with 400 (no such cross-region profile). Falling
	# back to BEDROCK_MODEL — a valid pre-formatted Bedrock profile id — is
	# the same pattern live-tests uses (see test_claude_settings_enforcement_live._model_for_env).
	# `tasks/*.yaml` is NOT recursive, so subdirectory tasks are listed
	# explicitly. anti_cheat_reference is the adversarial probe that the agent
	# cannot read the reference solution during its turn; it needs the
	# coder-eval-agent image built above (it is a driver: docker task).
	# record_cli_responses is the record_cli per-invocation-response probe and
	# is also driver: docker, so it needs that same image; it is flat in
	# tasks/, so the glob already matches it.
	- name: Verify smoke secrets present
	# smoke_system_one_judge grades through TypeSafe. Without the key the
	# criterion raises JudgeInfrastructureError and the task lands in
	# tasks_errored, which reads as "the harness broke" rather than "the
	# secret is missing". Fail here, where the message says which.
	run: \|
	: "${TYPESAFE_API_KEY:?TYPESAFE_API_KEY missing — needed by smoke_system_one_judge}"
	echo "All smoke secrets present."

	- name: Run smoke-pass bucket (expect all to succeed)
	run: \|
	.venv/bin/coder-eval run tasks/.yaml tasks/anti_cheat_reference/.yaml \
	--tags smoke-pass \
	--run-dir runs/ci-smoke-pass

	- name: Run smoke-fail bucket (expect failure detection)
	run: \|
	# Inverted exit semantics: coder-eval exits 1 when a task fails. We want that.
	if .venv/bin/coder-eval run tasks/*.yaml \
	--tags smoke-fail \
	--run-dir runs/ci-smoke-fail; then
	echo "FATAL: smoke-fail task succeeded — failure detection appears broken."
	exit 1
	fi

	- name: Run smoke-variants bucket (multi-variant resolver)
	run: \|
	.venv/bin/coder-eval run tasks/*.yaml \
	--tags smoke-variants \
	--experiment experiments/smoke_variants.yaml \
	--run-dir runs/ci-smoke-variants

	- name: Verify smoke-pass bucket
	run: \|
	F=runs/ci-smoke-pass/experiment.json
	test -f "$F" \|\| { echo "$F missing"; ls -la runs/; exit 1; }
	RUN=$(jq '[.variant_aggregates[].tasks_run] \| add' "$F")
	SUCCEEDED=$(jq '[.variant_aggregates[].tasks_succeeded] \| add' "$F")
	FAILED=$(jq '[.variant_aggregates[].tasks_failed] \| add' "$F")
	ERRORED=$(jq '[.variant_aggregates[].tasks_error] \| add' "$F")
	echo "smoke-pass: run=$RUN succeeded=$SUCCEEDED failed=$FAILED errored=$ERRORED"
	test "$RUN" = "$EXPECTED_SMOKE_PASS_RUN" \|\| { echo "smoke-pass tasks_run drift: expected $EXPECTED_SMOKE_PASS_RUN, got $RUN"; exit 1; }
	test "$SUCCEEDED" = "$EXPECTED_SMOKE_PASS_SUCCEEDED" \|\| { echo "smoke-pass succeeded drift: expected $EXPECTED_SMOKE_PASS_SUCCEEDED, got $SUCCEEDED"; exit 1; }
	test "$FAILED" = "0" \|\| { echo "smoke-pass had unexpected failures"; exit 1; }
	test "$ERRORED" = "0" \|\| { echo "smoke-pass had errors"; exit 1; }

	# The four wall-clock buckets (head + generation + UNION(tool) + tail)
	# must account for each turn's own duration. This is the TWO-SIDED gate:
	# the committed golden sensor only catches an OVERSHOOT, so a bucket that
	# claims LESS time than it should — the defect class this area keeps
	# producing — passes every test in the suite. It needs live task.json
	# files, which the smoke-pass run above already leaves on disk.
	#
	# COVERS CLAUDE-CODE ONLY: experiments/default.yaml sets type: claude-code,
	# so every turn here is that harness. The other four are covered by
	# tests/test_timing_identity_contract.py, which is ms-exact but synthetic.
	- name: Verify timing residual (claude-code only)
	run: \|
	.venv/bin/python scripts/timing/decompose_run.py \
	$(find runs/ci-smoke-pass -name task.json) --max-residual-pct 5

	- name: Verify smoke-fail bucket
	run: \|
	F=runs/ci-smoke-fail/experiment.json
	test -f "$F" \|\| { echo "$F missing"; ls -la runs/; exit 1; }
	RUN=$(jq '[.variant_aggregates[].tasks_run] \| add' "$F")
	FAILED=$(jq '[.variant_aggregates[].tasks_failed] \| add' "$F")
	ERRORED=$(jq '[.variant_aggregates[].tasks_error] \| add' "$F")
	echo "smoke-fail: run=$RUN failed=$FAILED errored=$ERRORED"
	test "$RUN" = "$EXPECTED_SMOKE_FAIL_RUN" \|\| { echo "smoke-fail tasks_run drift: expected $EXPECTED_SMOKE_FAIL_RUN, got $RUN"; exit 1; }
	test "$FAILED" = "$EXPECTED_SMOKE_FAIL_FAILED" \|\| { echo "smoke-fail expected $EXPECTED_SMOKE_FAIL_FAILED failure(s), got $FAILED — failure detection may be broken"; exit 1; }
	test "$ERRORED" = "0" \|\| { echo "smoke-fail unexpectedly errored (not the same as failing)"; exit 1; }

	- name: Verify smoke-variants bucket
	run: \|
	F=runs/ci-smoke-variants/experiment.json
	test -f "$F" \|\| { echo "$F missing"; ls -la runs/; exit 1; }
	RUN=$(jq '[.variant_aggregates[].tasks_run] \| add' "$F")
	SUCCEEDED=$(jq '[.variant_aggregates[].tasks_succeeded] \| add' "$F")
	ERRORED=$(jq '[.variant_aggregates[].tasks_error] \| add' "$F")
	VARIANTS=$(jq '.variant_aggregates \| length' "$F")
	echo "smoke-variants: variants=$VARIANTS run=$RUN succeeded=$SUCCEEDED errored=$ERRORED"
	test "$VARIANTS" = "2" \|\| { echo "smoke-variants expected 2 variants, got $VARIANTS — multi-variant fan-out may be broken"; exit 1; }
	test "$RUN" = "$EXPECTED_SMOKE_VARIANTS_RUN" \|\| { echo "smoke-variants tasks_run drift: expected $EXPECTED_SMOKE_VARIANTS_RUN, got $RUN"; exit 1; }
	test "$SUCCEEDED" = "$EXPECTED_SMOKE_VARIANTS_SUCCEEDED" \|\| { echo "smoke-variants succeeded drift: expected $EXPECTED_SMOKE_VARIANTS_SUCCEEDED, got $SUCCEEDED"; exit 1; }
	test "$ERRORED" = "0" \|\| { echo "smoke-variants had errors"; exit 1; }

	- name: Re-evaluate hello_date via `coder-eval evaluate` (no agent)
	# PR #220 fixed a regression in evaluate-only mode (reference loading).
	# This step re-runs hello_date.yaml's criteria against the artifacts
	# the smoke-pass run already produced — no new agent invocation, so
	# zero extra LLM calls. A regression in the evaluate code path
	# surfaces as a non-zero exit code from `coder-eval evaluate`.
	run: \|
	WORK="runs/ci-smoke-pass/default/hello_date_smoke_test/00/artifacts/hello_date_smoke_test"
	test -d "$WORK" \|\| { echo "Expected artifacts dir not found: $WORK"; find runs/ci-smoke-pass -maxdepth 5 -type d 2>/dev/null; exit 1; }
	.venv/bin/coder-eval evaluate tasks/hello_date.yaml "$WORK" \
	--run-dir runs/ci-smoke-eval-only

	- name: Debug SSH session on failure
	if: failure()
	uses: mxschmitt/action-tmate@35b54afac29c97fb54faba5b513f8fbd1882f113 # v3.x
	with:
	limit-access-to-actor: true

	- name: Upload run artifacts on failure
	if: failure()
	uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
	with:
	name: e2e-smoke-runs
	path: \|
	runs/ci-smoke-pass/
	runs/ci-smoke-fail/
	runs/ci-smoke-variants/
	runs/ci-smoke-eval-only/
	retention-days: 7

	live-tests:
	name: Live Integration Tests (Settings Enforcement + Cost Budget)
	runs-on: uipath-ubuntu-24.04
	timeout-minutes: 15
	# Skip on fork PRs where secrets aren't available
	if: github.event_name != 'pull_request' \|\| github.event.pull_request.head.repo.full_name == github.repository

	env:
	# Job-level env: default to DirectRoute (no API_BACKEND override). The
	# settings-enforcement and cost-budget steps use this. The Bedrock
	# settings-enforcement step adds API_BACKEND=bedrock at step scope only.
	ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
	# Bedrock backend for the third settings-enforcement run. Job-level so
	# the preflight secrets check can validate them; the BedrockRoute is
	# only engaged via API_BACKEND=bedrock at step scope.
	AWS_BEARER_TOKEN_BEDROCK: ${{ secrets.AWS_BEARER_TOKEN_BEDROCK }}
	AWS_REGION: ${{ secrets.AWS_REGION }}
	BEDROCK_MODEL: ${{ secrets.BEDROCK_MODEL }}
	# The System One judge's own endpoint — unrelated to either route above.
	TYPESAFE_API_KEY: ${{ secrets.TYPESAFE_API_KEY }}

	steps:
	- name: Checkout code
	uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

	- name: Set up Python 3.13
	uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
	with:
	python-version: "3.13"

	- name: Set up Node.js 20
	uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
	with:
	node-version: "20"

	- name: Install Claude CLI
	run: npm install -g @anthropic-ai/claude-code

	# No bubblewrap/AppArmor setup needed: neither live-test file engages
	# AgentConfig.isolation=true, so the SDK doesn't shell out to bwrap.

	- name: Cache dependencies
	uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
	with:
	# ``.venv`` excluded: a hosted-Python patch bump orphans its
	# interpreter symlink (dangling ``.venv/bin/python`` -> exit 127).
	# ``uv sync --frozen`` rehydrates it from the cached wheels.
	path: \|
	~/.cache/uv
	~/.cache/pip
	key: ${{ runner.os }}-py3.13-live-${{ hashFiles('pyproject.toml', 'uv.lock') }}
	restore-keys: \|
	${{ runner.os }}-py3.13-live-
	${{ runner.os }}-py3.13-

	- name: Install uv
	run: \|
	python -m pip install --upgrade "pip>=26.2" # PYSEC-2026-3721 fix floor
	pip install uv

	- name: Install project dependencies (hash-verified from uv.lock)
	run: uv sync --frozen --extra dev --extra uipath --extra codex

	- name: Verify required secrets are present
	run: \|
	: "${ANTHROPIC_API_KEY:?ANTHROPIC_API_KEY missing}"
	: "${AWS_BEARER_TOKEN_BEDROCK:?AWS_BEARER_TOKEN_BEDROCK missing}"
	: "${AWS_REGION:?AWS_REGION missing}"
	: "${BEDROCK_MODEL:?BEDROCK_MODEL missing}"
	: "${TYPESAFE_API_KEY:?TYPESAFE_API_KEY missing}"
	echo "All live-test secrets present."

	- name: Run claude-settings enforcement live tests (DirectRoute)
	# No API_BACKEND override — Settings() defaults to DIRECT and uses ANTHROPIC_API_KEY.
	# `-ra` surfaces skipped tests in the summary so CI logs show what ran vs skipped.
	# `--strict-markers` rejects unregistered @pytest.mark.* (cheap typo insurance).
	# JUnit XML feeds the post-run "tests actually passed" assertion below.
	# `-n 4` overrides pyproject's `-n auto`: these hit the real API, so
	# concurrency must not vary with the runner's vCPU count. 4 = the old shape.
	run: \|
	mkdir -p tmp
	.venv/bin/pytest tests/test_claude_settings_enforcement_live.py \
	-m live -v --tb=short --strict-markers -ra -n 4 \
	--junit-xml=tmp/junit-settings.xml

	- name: Run claude-settings enforcement live tests (BedrockRoute)
	# Same test file, different backend. Settings-enforcement lives in the
	# Claude Code CLI and is backend-agnostic (per the file's docstring),
	# so this run is the only thing in CI that exercises BedrockRoute
	# end-to-end with the real Anthropic-on-Bedrock model.
	env:
	API_BACKEND: "bedrock"
	run: \|
	.venv/bin/pytest tests/test_claude_settings_enforcement_live.py \
	-m live -v --tb=short --strict-markers -ra -n 4 \
	--junit-xml=tmp/junit-settings-bedrock.xml

	- name: Run System One judge wire-contract live tests
	# The only thing in CI that talks to TypeSafe. The unit tests mock the
	# invoker, so nothing else catches a change to the answer shape the
	# reduction assumes — notably the STRING level keys ("0", "1", ...) in a
	# score answer's probabilities. `-n0`: three questions in one round trip,
	# so there is nothing to parallelize.
	run: \|
	.venv/bin/pytest tests/test_system_one_judge_live.py \
	-m live -n0 -v --tb=short --strict-markers -ra \
	--junit-xml=tmp/junit-system-one.xml

	- name: Assert live tests actually ran (not silently skipped)
	# Parse JUnit XML for passed count, not collected count. Pytest collects
	# @pytest.mark.skipif-marked tests even when the predicate is True, so a
	# collect-only count would still report N>0 even if every test silently
	# skipped — defeating the purpose of the assertion. Passed = tests - skipped
	# - errors - failures correctly catches the silent-skip regression class.
	run: \|
	python - <<'PY'
	import sys, xml.etree.ElementTree as ET
	def passed(path: str) -> int:
	# Aggregate across all <testsuite> elements: pytest 8.x normally
	# emits a single suite, but xdist or plugin variations can produce
	# multiple suites under a <testsuites> root. Counting only the
	# first would silently undercount and falsely fail the assertion.
	root = ET.parse(path).getroot()
	suites = [root] if root.tag == "testsuite" else root.findall("testsuite")
	if not suites:
	sys.exit(f"{path}: no <testsuite> element")
	total = sum(int(ts.get("tests", 0)) for ts in suites)
	skipped = sum(int(ts.get("skipped", 0)) for ts in suites)
	errors = sum(int(ts.get("errors", 0)) for ts in suites)
	failures = sum(int(ts.get("failures", 0)) for ts in suites)
	return total - skipped - errors - failures
	p_settings_direct = passed("tmp/junit-settings.xml")
	p_settings_bedrock = passed("tmp/junit-settings-bedrock.xml")
	p_system_one = passed("tmp/junit-system-one.xml")
	print(
	f"Passed: settings(direct)={p_settings_direct}, "
	f"settings(bedrock)={p_settings_bedrock}, system_one={p_system_one}"
	)
	if p_settings_direct < 1:
	sys.exit("test_claude_settings_enforcement_live.py (DirectRoute) reported zero PASSED tests")
	if p_settings_bedrock < 1:
	sys.exit("test_claude_settings_enforcement_live.py (BedrockRoute) reported zero PASSED tests")
	if p_system_one < 1:
	sys.exit("test_system_one_judge_live.py reported zero PASSED tests")
	PY

	- name: Run cost-budget smoke (max_usd → COST_BUDGET_EXCEEDED via DirectRoute)
	# Verifies the cost path end-to-end on DirectRoute (Anthropic reports
	# per-turn cost). A regression that broke per-turn cost reporting would
	# log "max_usd budget configured but no turn reported cost; skipping
	# cost check" and the task would land in tasks_failed via the
	# placeholder criterion instead of COST_BUDGET_EXCEEDED. The
	# final_status assertion below catches that silently-broken case.
	run: \|
	# Inverted exit semantics: coder-eval exits 1 when a task fails. We want that.
	if .venv/bin/coder-eval run tasks/smoke_cost_budget_exceeded.yaml \
	--model claude-haiku-4-5-20251001 \
	--run-dir runs/ci-smoke-cost-budget; then
	echo "FATAL: cost-budget smoke succeeded — cost enforcement appears broken."
	exit 1
	fi

	- name: Verify cost-budget smoke (COST_BUDGET_EXCEEDED, not just failed)
	run: \|
	F=runs/ci-smoke-cost-budget/experiment.json
	test -f "$F" \|\| { echo "$F missing"; ls -la runs/; exit 1; }
	RUN=$(jq '[.variant_aggregates[].tasks_run] \| add' "$F")
	FAILED=$(jq '[.variant_aggregates[].tasks_failed] \| add' "$F")
	ERRORED=$(jq '[.variant_aggregates[].tasks_error] \| add' "$F")
	echo "smoke-cost-budget: run=$RUN failed=$FAILED errored=$ERRORED"
	test "$RUN" = "1" \|\| { echo "smoke-cost-budget tasks_run drift: expected 1, got $RUN"; exit 1; }
	test "$FAILED" = "1" \|\| { echo "smoke-cost-budget expected 1 failure, got $FAILED"; exit 1; }
	test "$ERRORED" = "0" \|\| { echo "smoke-cost-budget unexpectedly errored"; exit 1; }
	# Specifically assert the budget gate fired — not just that the task failed.
	# A regression in per-turn cost reporting would still produce failed=1 via the
	# placeholder criterion, but final_status would be FAILURE, not COST_BUDGET_EXCEEDED.
	TASK_JSON=runs/ci-smoke-cost-budget/default/smoke_cost_budget_exceeded/00/task.json
	test -f "$TASK_JSON" \|\| { echo "$TASK_JSON missing"; find runs/ci-smoke-cost-budget -maxdepth 5 -type f; exit 1; }
	STATUS=$(jq -r '.final_status' "$TASK_JSON")
	echo "final_status=$STATUS"
	test "$STATUS" = "COST_BUDGET_EXCEEDED" \|\| { echo "Expected final_status=COST_BUDGET_EXCEEDED, got $STATUS — cost gate may not have fired (cost telemetry regression?)"; exit 1; }

	- name: Upload live-test artifacts on failure
	if: failure()
	uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
	with:
	name: live-tests-output
	path: \|
	tmp/
	runs/
	retention-days: 7

	codex-live-tests:
	name: Live Integration Tests (Codex)
	runs-on: uipath-ubuntu-24.04
	timeout-minutes: 15
	# Skip on fork PRs where secrets aren't available.
	if: github.event_name != 'pull_request' \|\| github.event.pull_request.head.repo.full_name == github.repository

	env:
	# Codex SDK auth. CODEX_API_KEY is the key; CODEX_BASE_URL routes to a
	# custom OpenAI-/responses-compatible endpoint and CODEX_MODEL pins the
	# model/deployment that endpoint expects.
	CODEX_API_KEY: ${{ secrets.CODEX_API_KEY }}
	CODEX_BASE_URL: ${{ secrets.CODEX_BASE_URL }}
	CODEX_MODEL: ${{ secrets.CODEX_MODEL }}

	steps:
	- name: Checkout code
	uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

	- name: Set up Python 3.13
	uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
	with:
	python-version: "3.13"

	- name: Install uv
	run: \|
	python -m pip install --upgrade "pip>=26.2" # PYSEC-2026-3721 fix floor
	pip install uv

	- name: Install project dependencies (with codex + litellm extras)
	run: uv sync --frozen --extra dev --extra uipath --extra codex --extra litellm

	- name: Verify required secrets are present
	run: \|
	if [ -z "$CODEX_API_KEY" ]; then
	echo "CODEX_API_KEY is not set."
	exit 1
	fi
	echo "CODEX_API_KEY present."

	- name: Run Codex + litellm-judge live tests
	run: \|
	mkdir -p tmp
	# Run serially: `-n0` overrides the global `-n auto` (addopts).
	# Parallel xdist workers share ~/.codex and race the Codex SQLite
	# state migration (`duplicate column name: thread_id`); serial init
	# migrates the fresh DB exactly once. test_litellm_judge_live.py
	# reuses these same CODEX_* secrets to exercise
	# checker_context.api_route.route: litellm end-to-end (PR #137
	# review: "nothing in the repo exercises the feature").
	.venv/bin/pytest tests/test_codex_agent_live.py tests/test_litellm_judge_live.py \
	-m live -n0 -v --tb=short --strict-markers -ra \
	--junit-xml=tmp/junit-codex-live.xml

	- name: Assert Codex live tests actually ran (not silently skipped)
	run: \|
	python - <<'PY'
	import sys, xml.etree.ElementTree as ET
	root = ET.parse("tmp/junit-codex-live.xml").getroot()
	suites = [root] if root.tag == "testsuite" else root.findall("testsuite")
	total = sum(int(s.get("tests", 0)) for s in suites)
	skipped = sum(int(s.get("skipped", 0)) for s in suites)
	errors = sum(int(s.get("errors", 0)) for s in suites)
	failures = sum(int(s.get("failures", 0)) for s in suites)
	passed = total - skipped - errors - failures
	print(f"codex-live passed={passed} skipped={skipped} errors={errors} failures={failures}")
	if passed < 1:
	sys.exit("Live Codex/litellm-judge tests reported zero PASSED tests (missing API key / silent skip?)")
	PY

	- name: Upload Codex live-test artifacts on failure
	if: failure()
	uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
	with:
	name: codex-live-output
	path: tmp/
	retention-days: 7

	byoa-live-tests:
	name: Live Integration Tests (BYOA Plugin)
	runs-on: uipath-ubuntu-24.04
	timeout-minutes: 15
	# Skip on fork PRs where secrets aren't available.
	if: github.event_name != 'pull_request' \|\| github.event.pull_request.head.repo.full_name == github.repository

	env:
	# DirectRoute: a plugin agent that subclasses ClaudeCodeAgent uses ANTHROPIC_API_KEY.
	ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}

	steps:
	- name: Checkout code
	uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

	- name: Set up Python 3.13
	uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
	with:
	python-version: "3.13"

	- name: Set up Node.js 20
	uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
	with:
	node-version: "20"

	- name: Install Claude CLI
	run: npm install -g @anthropic-ai/claude-code

	- name: Install uv
	run: \|
	python -m pip install --upgrade "pip>=26.2" # PYSEC-2026-3721 fix floor
	pip install uv

	- name: Install project dependencies (hash-verified from uv.lock)
	run: uv sync --frozen --extra dev --extra uipath

	# Install the out-of-tree fixture plugin so coder-eval discovers 'byoa-demo'
	# through the REAL coder_eval.plugins entry point (not a test monkeypatch).
	- name: Install BYOA demo plugin (entry-point discovery)
	run: uv pip install -e tests/fixtures/byoa_demo_plugin

	- name: Verify required secrets are present
	run: \|
	: "${ANTHROPIC_API_KEY:?ANTHROPIC_API_KEY missing}"
	echo "ANTHROPIC_API_KEY present."

	- name: Run BYOA plugin live test
	run: \|
	mkdir -p tmp
	.venv/bin/pytest tests/test_byoa_plugin_live.py \
	-m live -v --tb=short --strict-markers -ra \
	--junit-xml=tmp/junit-byoa-live.xml

	- name: Assert BYOA live test actually ran (not silently skipped)
	run: \|
	python - <<'PY'
	import sys, xml.etree.ElementTree as ET
	root = ET.parse("tmp/junit-byoa-live.xml").getroot()
	suites = [root] if root.tag == "testsuite" else root.findall("testsuite")
	total = sum(int(s.get("tests", 0)) for s in suites)
	skipped = sum(int(s.get("skipped", 0)) for s in suites)
	errors = sum(int(s.get("errors", 0)) for s in suites)
	failures = sum(int(s.get("failures", 0)) for s in suites)
	passed = total - skipped - errors - failures
	print(f"byoa-live passed={passed} skipped={skipped} errors={errors} failures={failures}")
	if passed < 1:
	sys.exit("test_byoa_plugin_live.py reported zero PASSED tests (missing API key / silent skip?)")
	PY

	- name: Upload BYOA live-test artifacts on failure
	if: failure()
	uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
	with:
	name: byoa-live-output
	path: tmp/
	retention-days: 7

	action-dogfood:
	name: Action Dogfood (composite action, real API)
	# Deliberately NOT on the `uipath-*` pool, and keep it that way: this job is the
	# executable proof of the published Action, and docs/tutorials/02-ci-pipeline.md
	# (the one consumer snippet naming a runner) says `ubuntu-latest`. Nothing else in
	# CI exercises the image integrators actually use. A bulk `runs-on:` migration has
	# swept this up twice — check it by hand.
	runs-on: ubuntu-latest
	timeout-minutes: 15
	# Skip on fork PRs where secrets aren't available (matches e2e-smoke).
	if: github.event_name != 'pull_request' \|\| github.event.pull_request.head.repo.full_name == github.repository
	steps:
	- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

	# The composite action is agent-agnostic and does NOT install a coding-agent
	# runtime. The dogfood task uses the default claude-code agent, so provide
	# Node + the Claude CLI here (as e2e-smoke does), before invoking the action.
	- name: Set up Node.js 20
	uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
	with:
	node-version: "20"
	- name: Install Claude CLI
	run: npm install -g @anthropic-ai/claude-code

	- name: Run coder-eval via local action
	id: dogfood
	uses: ./
	with:
	# Exercises what tests/test_action_inputs.py cannot: it asserts the argv
	# both step scripts build, but only a real runner proves that
	# `working-directory:` works on a composite step and that a plugin
	# installed via `--with` is discovered at runtime. Hence a relative
	# run-dir (landing under tasks/), the bare task filename and the `../`
	# plugin path, all resolved from `working-directory`.
	version: local
	working-directory: tasks
	run-dir: runs/ci-action-dogfood
	extra-packages: ../tests/fixtures/byoa_demo_plugin
	# The bracketed `-D` override is the case one-argv-entry-per-line exists
	# for: `[...]` is a bash character class, so a whitespace-split input
	# would collapse the list whenever a file in the cwd matched. It adds
	# Glob to hello_date.yaml's three tools, so the assertions below can tell
	# "arrived" from "ignored".
	args: \|
	hello_date.yaml
	--model
	claude-haiku-4-5-20251001
	-D
	agent.allowed_tools=[Read,Write,Bash,Glob]
	# ANTHROPIC_API_KEY reaching the run is proven by the API-backed task
	# succeeding; the second line exercises multi-line env parsing.
	env: \|
	ANTHROPIC_API_KEY=${{ secrets.ANTHROPIC_API_KEY }}
	CE_DOGFOOD_MARKER=1

	# In `tasks/` because that is the assertion: `run-dir` is reported exactly as
	# passed, so a relative one is relative to `working-directory`.
	- name: Verify outputs and JUnit file
	working-directory: tasks
	env:
	JUNIT: ${{ steps.dogfood.outputs.junit-path }}
	RUNDIR: ${{ steps.dogfood.outputs.run-dir }}
	run: \|
	set -euo pipefail
	test -n "$JUNIT" && test -f "$JUNIT" \|\| { echo "junit output missing"; exit 1; }
	# Well-formedness check on a file this job just generated (trusted input;
	# our writer emits no DTDs/entities) — stdlib ET is fine here.
	python3 -c "import sys, xml.etree.ElementTree as ET; ET.parse(sys.argv[1])" "$JUNIT"
	test -f "$RUNDIR/run.json" \|\| { echo "run.json missing"; exit 1; }
	# An absolute path would mean working-directory was ignored.
	case "$RUNDIR" in /*) echo "run-dir output was rewritten to an absolute path: $RUNDIR"; exit 1 ;; esac

	# The action does not touch $GITHUB_STEP_SUMMARY: this is both the assertion
	# that `run-md-path` is real and the recipe the docs hand consumers.
	- name: Append the run report to the job summary
	if: always()
	working-directory: tasks
	env:
	RUN_MD: ${{ steps.dogfood.outputs.run-md-path }}
	run: \|
	set -euo pipefail
	test -f "$RUN_MD" \|\| { echo "run-md-path output does not exist: $RUN_MD"; exit 1; }
	cat "$RUN_MD" >> "$GITHUB_STEP_SUMMARY"

	- name: Verify the plugin was discovered and the bracketed override arrived
	working-directory: tasks
	env:
	RUNDIR: ${{ steps.dogfood.outputs.run-dir }}
	run: \|
	set -euo pipefail

	# `coder-eval` on PATH is the action's uv tool shim, so this interrogates
	# its environment: a task naming the fixture's agent kind validates only
	# if the entry point was discovered there, else plan exits 1 with
	# "No agent registered for type 'byoa-demo'".
	cat > byoa-probe.yaml <<'YAML'
	task_id: "action_extra_packages_probe"
	description: "Validates only when the byoa-demo plugin is discoverable."
	initial_prompt: "not executed - plan validates without running an agent"
	agent:
	type: "byoa-demo"
	success_criteria:
	- type: "file_exists"
	path: "app.py"
	description: "not executed"
	YAML
	coder-eval plan byoa-probe.yaml
	rm -f byoa-probe.yaml

	# And the `-D` value survived as a 4-element list, not word-split or
	# glob-rewritten.
	RUN_JSON="$RUNDIR/run.json" python3 <<'PY'
	import json, os, sys

	data = json.load(open(os.environ["RUN_JSON"], encoding="utf-8"))
	rows = data.get("task_results") or []
	if not rows:
	sys.exit("run.json has no task_results to check the -D override against")
	tools = (rows[0].get("agent_config") or {}).get("allowed_tools")
	expected = ["Read", "Write", "Bash", "Glob"]
	if tools != expected:
	sys.exit(f"-D override did not arrive intact: allowed_tools={tools!r}, expected {expected!r}")
	print(f"bracketed -D override resolved to {tools!r}")
	PY

	- name: Upload dogfood run on failure
	if: failure()
	uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
	with:
	name: action-dogfood-runs
	path: tasks/runs/ci-action-dogfood/
	retention-days: 7

	delegate-live-tests:
	# Live Delegate e2e: spawns the real Node host (agents/delegate/delegate_host.mjs)
	# wrapping the public @uipath/delegate-sdk against a real UiPath backend. Runs on
	# every PR/push like the other live-test jobs (codex-live-tests, byoa-live-tests)
	# now that the DELEGATE_ROPC_* secrets are provisioned; the presence-check gate
	# below skips cleanly ONLY on a fork PR (no secrets available there) and fails
	# loudly on a same-repo run with a missing secret, matching the other live jobs.
	# Auth is minted fresh each run via the
	# OAuth2 Resource Owner Password Credentials (ROPC) grant — the same
	# `grant_type=password` flow documented in coder_eval_uipath/eval_runner/CI_DESIGN.md
	# and implemented in coder_eval_uipath/eval_runner/scripts/ci/refresh-auth.sh —
	# rather than a long-lived AUTH_TOKEN secret, since a minted token's ~1h TTL
	# would make a static secret stale between runs. A dedicated bot user's
	# username/password is the durable credential; the Delegate SDK reads the
	# minted token straight from AUTH_TOKEN/TENANT_ID/ORG_ID (see
	# docs/agents/DELEGATE.md), so no sdk-auth.json or client-secret refresh chain
	# is needed here.
	name: Delegate Live E2E (ROPC)
	runs-on: uipath-ubuntu-24.04
	timeout-minutes: 15
	# Skip on fork PRs where secrets aren't available (matches e2e-smoke).
	if: github.event_name != 'pull_request' \|\| github.event.pull_request.head.repo.full_name == github.repository

	steps:
	- name: Checkout code
	uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

	- name: Check Delegate ROPC secrets present
	id: gate
	env:
	DELEGATE_ROPC_CLIENT_ID: ${{ secrets.DELEGATE_ROPC_CLIENT_ID }}
	DELEGATE_ROPC_CLIENT_SECRET: ${{ secrets.DELEGATE_ROPC_CLIENT_SECRET }}
	DELEGATE_ROPC_USERNAME: ${{ secrets.DELEGATE_ROPC_USERNAME }}
	DELEGATE_ROPC_PASSWORD: ${{ secrets.DELEGATE_ROPC_PASSWORD }}
	DELEGATE_ORG_ID: ${{ secrets.DELEGATE_ORG_ID }}
	DELEGATE_TENANT_ID: ${{ secrets.DELEGATE_TENANT_ID }}
	DELEGATE_ORG_SLUG: ${{ secrets.DELEGATE_ORG_SLUG }}
	DELEGATE_TENANT_SLUG: ${{ secrets.DELEGATE_TENANT_SLUG }}
	# A fork PR (this job's own `if:` skips the whole job for those) is the
	# only case a missing secret is expected -- everywhere else, silently
	# reporting success having run nothing is exactly the failure a required
	# check exists to catch, so a same-repo run with a rotated/dropped secret
	# fails loudly instead.
	IS_FORK_PR: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository }}
	run: \|
	missing=""
	for name in DELEGATE_ROPC_CLIENT_ID DELEGATE_ROPC_CLIENT_SECRET DELEGATE_ROPC_USERNAME \
	DELEGATE_ROPC_PASSWORD DELEGATE_ORG_ID DELEGATE_TENANT_ID \
	DELEGATE_ORG_SLUG DELEGATE_TENANT_SLUG; do
	if [ -z "${!name}" ]; then
	missing="$missing $name"
	fi
	done
	if [ -z "$missing" ]; then
	echo "present=true" >> "$GITHUB_OUTPUT"
	elif [ "$IS_FORK_PR" = "true" ]; then
	echo "present=false" >> "$GITHUB_OUTPUT"
	echo "Delegate ROPC secrets not available on a fork PR — skipping the live run."
	else
	echo "present=false" >> "$GITHUB_OUTPUT"
	echo "::error::Delegate ROPC secrets missing on a same-repo run:$missing"
	exit 1
	fi

	- name: Set up Python 3.13
	if: steps.gate.outputs.present == 'true'
	uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
	with:
	python-version: "3.13"

	- name: Set up Node.js 20
	if: steps.gate.outputs.present == 'true'
	uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
	with:
	node-version: "20"

	- name: Install uv
	if: steps.gate.outputs.present == 'true'
	run: \|
	python -m pip install --upgrade "pip>=26.2" # PYSEC-2026-3721 fix floor
	pip install uv

	- name: Install project dependencies (hash-verified from uv.lock)
	if: steps.gate.outputs.present == 'true'
	run: uv sync --frozen --extra dev

	- name: Install @uipath/delegate-sdk (public npm, no token needed)
	if: steps.gate.outputs.present == 'true'
	working-directory: src/coder_eval/agents/delegate
	# --safe-chain-skip-minimum-package-age (not a growing exclusion-list entry
	# per platform-specific optionalDependency name): the SDK's per-platform
	# native runtime packages (@uipath/delegate-runtime-*) publish at the same
	# instant as the SDK itself, so a name-list exclusion just needing another
	# entry for each new one is the wrong shape for a first-party UiPath package
	# already trusted at the SDK-name level.
	run: npm install --safe-chain-skip-minimum-package-age

	- name: ROPC login — mint a fresh Delegate access token
	if: steps.gate.outputs.present == 'true'
	id: ropc
	env:
	DELEGATE_UIPATH_ENV: ${{ github.event.inputs.delegate_uipath_env \|\| 'alpha' }}
	CLIENT_ID: ${{ secrets.DELEGATE_ROPC_CLIENT_ID }}
	CLIENT_SECRET: ${{ secrets.DELEGATE_ROPC_CLIENT_SECRET }}
	CE_USERNAME: ${{ secrets.DELEGATE_ROPC_USERNAME }}
	CE_PASSWORD: ${{ secrets.DELEGATE_ROPC_PASSWORD }}
	ORG_ID: ${{ secrets.DELEGATE_ORG_ID }}
	run: \|
	set -euo pipefail
	# The mint destination is resolved from a fixed allowlist, never from
	# the raw workflow_dispatch input string directly — CLIENT_SECRET and
	# CE_PASSWORD travel in this request's body, so accepting an arbitrary
	# attacker-supplied URL here would be a credential-exfiltration path
	# open to anyone who can trigger workflow_dispatch, not just admins.
	case "${DELEGATE_UIPATH_ENV}" in
	alpha) UIPATH_URL="https://alpha.uipath.com" ;;
	staging) UIPATH_URL="https://staging.uipath.com" ;;
	production) UIPATH_URL="https://cloud.uipath.com" ;;
	*) echo "Unknown delegate_uipath_env: ${DELEGATE_UIPATH_ENV}" >&2; exit 1 ;;
	esac

	# Same grant as coder_eval_uipath/eval_runner/scripts/ci/refresh-auth.sh:
	# a bot user's username/password against the identity server, scoped
	# to the org via acr_values (the ROPC client carries no scope of its
	# own narrower than the org). ConversationalAgents is the scope the
	# Delegate backend itself checks; the rest matches refresh-auth.sh's
	# alpha scope list so this mint is provably not narrower than the one
	# already validated there.
	SCOPES="openid profile email offline_access ConversationalAgents AutopilotForEveryone Orchestrator OrchestratorApiUserAccess StudioWebBackend"
	# The POST body is built here and piped to curl on stdin (--data-binary
	# @-), never passed as a --data-urlencode argv value: argv (unlike
	# stdin) is world-readable via /proc/<pid>/cmdline to any other process
	# on this self-hosted runner for the life of the request, and
	# CLIENT_SECRET/CE_PASSWORD are durable credentials, not the ~1h token.
	body=$(jq -rn \
	--arg client_id "$CLIENT_ID" \
	--arg client_secret "$CLIENT_SECRET" \
	--arg username "$CE_USERNAME" \
	--arg password "$CE_PASSWORD" \
	--arg acr_values "tenant:${ORG_ID}" \
	--arg scope "$SCOPES" \
	'"grant_type=password" +
	"&client_id=" + ($client_id\|@uri) +
	"&client_secret=" + ($client_secret\|@uri) +
	"&username=" + ($username\|@uri) +
	"&password=" + ($password\|@uri) +
	"&acr_values=" + ($acr_values\|@uri) +
	"&scope=" + ($scope\|@uri)')
	response=$(printf '%s' "$body" \| curl -sS -L --connect-timeout 15 --max-time 60 -X POST "${UIPATH_URL}/identity_/connect/token" \
	-H "Content-Type: application/x-www-form-urlencoded" \
	-H "x-uipath-internal-accountid: ${ORG_ID}" \
	--data-binary @-)

	if err=$(echo "$response" \| jq -e -r '.error // empty') && [ -n "$err" ]; then
	echo "ROPC token request failed: $(echo "$response" \| jq -r '.error_description // .error')" >&2
	exit 1
	fi

	access_token=$(echo "$response" \| jq -r '.access_token')
	if [ -z "$access_token" ] \|\| [ "$access_token" = "null" ]; then
	echo "ROPC token response did not contain an access_token" >&2
	exit 1
	fi

	echo "::add-mask::$access_token"
	echo "access_token=$access_token" >> "$GITHUB_OUTPUT"

	- name: Diagnose minted token's org/account claims
	# Prints ONLY non-secret JWT claims (never the token itself) to check
	# whether the ROPC-minted token's own account/org actually matches the
	# DELEGATE_ORG_ID/DELEGATE_ORG_SLUG secrets used to build the backend
	# URL -- a mismatch here (not a coder_eval bug) is what a live run's
	# "Account id does not match" 401 from the connections_ API points at.
	# continue-on-error: this step is advisory -- its own script already
	# catches and reports unexpected input, but a step failure must still
	# never gate the real `coder-eval run` below that it exists to help
	# explain, not mask.
	if: steps.gate.outputs.present == 'true'
	continue-on-error: true
	env:
	ACCESS_TOKEN: ${{ steps.ropc.outputs.access_token }}
	EXPECTED_ORG_ID: ${{ secrets.DELEGATE_ORG_ID }}
	EXPECTED_TENANT_ID: ${{ secrets.DELEGATE_TENANT_ID }}
	run: \|
	set -euo pipefail
	payload="$(echo "$ACCESS_TOKEN" \| cut -d. -f2)"
	# Explicit MATCH/NO MATCH verdicts, not a side-by-side print: a secret's
	# exact value is auto-masked to *** everywhere in logs, including inside
	# an unrelated claims dump, so printing both for a human to eyeball is
	# unreliable -- the comparison has to happen here, not in the reader's head.
	python3 .github/scripts/diagnose_delegate_token_claims.py "$payload"
	echo "A 'NO MATCH' above means the secret points at a different org/tenant than"
	echo "the one this bot user's token actually belongs to -- that mismatch, not this"
	echo "agent's code, is the root cause of a downstream 'Account id does not match' 401."

	- name: Run fizzbuzz delegate task (assert PASS)
	if: steps.gate.outputs.present == 'true'
	env:
	DELEGATE_SDK_NODE_MODULES: ${{ github.workspace }}/src/coder_eval/agents/delegate
	DELEGATE_ENV: ${{ github.event.inputs.delegate_uipath_env \|\| 'alpha' }}
	AUTH_TOKEN: ${{ steps.ropc.outputs.access_token }}
	TENANT_ID: ${{ secrets.DELEGATE_TENANT_ID }}
	ORG_ID: ${{ secrets.DELEGATE_ORG_ID }}
	# Forwarded into `auth.organizationName` / `auth.tenantName` by
	# DelegateAgent._build_init_options (see .claude/notes/agents.md §
	# Delegate agent) — needed alongside the GUIDs above whenever
	# `environment` (rather than `backendUrl`) resolves the backend.
	ORG_SLUG: ${{ secrets.DELEGATE_ORG_SLUG }}
	TENANT_SLUG: ${{ secrets.DELEGATE_TENANT_SLUG }}
	run: .venv/bin/coder-eval run tasks/delegate/fizzbuzz_delegate.yaml --run-dir runs/delegate-live

	- name: Verify Delegate live run PASSED
	if: steps.gate.outputs.present == 'true'
	run: \|
	python3 - <<'PY'
	import json
	from pathlib import Path

	results = list(Path("runs/delegate-live").rglob("task.json"))
	assert results, "no task.json written"
	for path in results:
	data = json.loads(path.read_text())
	assert data.get("final_status") == "SUCCESS", (
	f"{path}: expected SUCCESS, got final_status={data.get('final_status')}"
	)
	print(f"Live Delegate run PASSED ({path}):", data.get("final_status"))
	PY

	- name: Run Delegate live unit tests (assert PASS)
	if: steps.gate.outputs.present == 'true'
	env:
	DELEGATE_SDK_NODE_MODULES: ${{ github.workspace }}/src/coder_eval/agents/delegate
	DELEGATE_ENV: ${{ github.event.inputs.delegate_uipath_env \|\| 'alpha' }}
	AUTH_TOKEN: ${{ steps.ropc.outputs.access_token }}
	TENANT_ID: ${{ secrets.DELEGATE_TENANT_ID }}
	ORG_ID: ${{ secrets.DELEGATE_ORG_ID }}
	ORG_SLUG: ${{ secrets.DELEGATE_ORG_SLUG }}
	TENANT_SLUG: ${{ secrets.DELEGATE_TENANT_SLUG }}
	run: \|
	mkdir -p tmp
	.venv/bin/pytest tests/test_delegate_agent_live.py \
	-m live -n0 -v --tb=short --strict-markers -ra \
	--junit-xml=tmp/junit-delegate-live.xml

	- name: Assert Delegate live tests actually ran (not silently skipped)
	if: steps.gate.outputs.present == 'true'
	run: \|
	python3 - <<'PY'
	import sys, xml.etree.ElementTree as ET
	root = ET.parse("tmp/junit-delegate-live.xml").getroot()
	suites = [root] if root.tag == "testsuite" else root.findall("testsuite")
	total = sum(int(s.get("tests", 0)) for s in suites)
	skipped = sum(int(s.get("skipped", 0)) for s in suites)
	errors = sum(int(s.get("errors", 0)) for s in suites)
	failures = sum(int(s.get("failures", 0)) for s in suites)
	passed = total - skipped - errors - failures
	print(f"delegate-live passed={passed} skipped={skipped} errors={errors} failures={failures}")
	if passed < 1:
	sys.exit("Live Delegate tests reported zero PASSED tests (missing prerequisite / silent skip?)")
	PY

	- name: Upload Delegate live-test artifacts on failure
	if: failure() && steps.gate.outputs.present == 'true'
	uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
	with:
	name: delegate-live-output
	path: runs/delegate-live/
	retention-days: 7

Navigation Menu

fix(criteria): search the whole command in command_executed, not just the first 2000 chars #737

Workflow file

fix(criteria): search the whole command in command_executed, not just the first 2000 chars #737

Uh oh!

Workflow file for this run