pr-checks.yml

name: PR Quality Checks

on:
  pull_request:
    branches: [main, develop]
  merge_group:
  push:
    branches: [main, develop]
  workflow_dispatch:
    inputs:
      delegate_uipath_env:
        description: "UiPath cloud environment to ROPC-login the Delegate live job against"
        required: false
        type: choice
        default: "alpha"
        options:
          - "alpha"
          - "staging"
          - "production"

# Cancel outdated runs when new commits are pushed
concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

# Least-privilege default. No job in this workflow writes to the repo via
# GITHUB_TOKEN — actions/upload-artifact and actions/cache use their own APIs.
permissions:
  contents: read

# Hard-disable usage telemetry for the ENTIRE workflow. The app ships a baked-in
# default Application Insights connection string (so a fresh install reports with
# no config), which means CI's real `coder-eval run` invocations would otherwise
# emit to the production telemetry resource and pollute dashboards. The in-process
# pytest autouse guard can't reach those subprocesses, so disable at the env level
# here — TELEMETRY_ENABLED is the single canonical disable gate.
env:
  TELEMETRY_ENABLED: "false"
  # The `uipath-*` pool enforces a minimum package-age safe-chain check on installs.
  # Workflow-level so every installing job inherits it; per-job copies are how some
  # jobs previously ended up with no exclusions at all. The literal is the operative
  # value — no secret of that name exists at repo or org level, so the bare `secrets.`
  # reference this replaced resolved to an empty list. (Image builds carry their own
  # list in docker/Dockerfile; deliberately not the same set.)
  SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS: ${{ secrets.SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS || 'openai-codex-cli-bin,openai-codex' }}

jobs:
  quality-gate:
    name: Quality Gate (Format, Lint, Type, Test, Security)
    # Fork PRs go to stock GitHub-hosted runners: this job runs the PR's own
    # `uv.lock` build hooks and test files, and the repo is public, so untrusted code
    # must not land on the shared pool image. Everything else uses the pool.
    runs-on: ${{ (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository) && 'ubuntu-latest' || 'uipath-ubuntu-latest' }}
    timeout-minutes: 10

    # A dummy Anthropic key so CI tests that construct a client don't fail on a
    # missing key.
    env:
      ANTHROPIC_API_KEY: "sk-ant-test-dummy-key-for-ci-tests-only"

    steps:
      - name: Checkout code
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2

      - name: Set up Python 3.13
        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405  # v6.2.0
        with:
          python-version: "3.13"

      - name: Cache dependencies
        uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae  # v5.0.5
        with:
          # ``.venv`` is deliberately NOT cached: its interpreter symlinks
          # point at the exact hosted-Python patch, so a patch bump (e.g.
          # 3.13.13 -> 3.13.14) leaves a dangling ``.venv/bin/python`` and
          # every ``.venv/bin/*`` fails with exit 127 (cannot execute).
          # ``uv sync --frozen`` rehydrates ``.venv`` from the cached wheels.
          path: |
            ~/.cache/uv
            ~/.cache/pip
            .pytest_cache
            .ruff_cache
          key: ${{ runner.os }}-py3.13-${{ hashFiles('pyproject.toml', 'uv.lock') }}
          restore-keys: |
            ${{ runner.os }}-py3.13-

      - name: Install uv
        run: |
          python -m pip install --upgrade "pip>=26.2"  # PYSEC-2026-3721 fix floor
          pip install uv

      - name: Install project dependencies (hash-verified from uv.lock)
        # --extra harbor: pyright below type-checks src/coder_eval/harbor/agent.py
        # against harbor's real types, not a scoped ignore.
        run: uv sync --frozen --extra dev --extra uipath --extra codex --extra litellm --extra harbor

      # PHASE 1: Fast checks (fail early)
      - name: Check code formatting (ruff format)
        run: .venv/bin/ruff format --check src/ tests/

      - name: Lint code (ruff check)
        run: .venv/bin/ruff check src/ tests/

      - name: Custom architectural lint (CE001+)
        run: .venv/bin/pytest tests/test_custom_lint.py -v --tb=short --no-header -p no:warnings

      - name: Prose budget (docstring/comment ratchet)
        run: .venv/bin/python -m tests.lint.prose_budget

      # PHASE 2: Type checking
      - name: Type check with pyright
        run: .venv/bin/pyright

      # The CE036 contract engine lives under tests/, which [tool.pyright] excludes
      # -- and `exclude` beats both a CLI file arg and an `include` entry, so it can
      # only be reached through a config of its own, derived from [tool.pyright] so
      # the two passes cannot drift. Mirrors `make typecheck`.
      - name: Type check the CE036 contract engine
        run: |
          .venv/bin/python -m tests.lint.pyright_config .pyright-tests.json
          .venv/bin/pyright -p .pyright-tests.json

      # PHASE 3: Security scanning
      - name: Security - Dependency vulnerabilities (pip-audit)
        run: .venv/bin/pip-audit --desc --skip-editable --ignore-vuln CVE-2026-4539 --ignore-vuln CVE-2026-3219 --ignore-vuln PYSEC-2025-183 # pygments 2.19.2 ReDoS + pip 26.0.1 tar/ZIP ambiguity + pyjwt 2.12.1 weak-encryption (disputed by supplier; key length is application-chosen); no fixes available on PyPI yet — revisit quarterly

      - name: Security - OSV vulnerability scan (osv-scanner)
        # Complements pip-audit: pip-audit queries the PyPI advisory DB;
        # osv-scanner queries OSV.dev which aggregates GitHub Security
        # Advisories and multiple ecosystems' DBs. Running both reduces the
        # chance an advisory slips through.
        #
        # Scope: only coder_eval's own ``uv.lock`` — the same surface
        # pip-audit covers. Sibling lockfiles in this repo
        # (``evalboard/pnpm-lock.yaml``, template node_modules) are
        # intentionally out of scope here; widen scope in a follow-up once
        # those packages have a maintainer-owned bump cadence.
        #
        # Binary pinned by version + sha256 against the upstream release
        # (https://github.com/google/osv-scanner/releases) so a registry
        # compromise can't swap the binary under us.
        env:
          OSV_VERSION: "v2.3.8"
          OSV_SHA256: "bc98e15319ed0d515e3f9235287ba53cdc5535d576d24fd573978ecfe9ab92dc"
        run: |
          set -euo pipefail
          # RUNNER_TEMP is per-job-isolated on hosted runners and properly-configured
          # self-hosted runners — avoids TOCTOU on shared /tmp between concurrent jobs.
          OSV_BIN="${RUNNER_TEMP}/osv-scanner"
          curl --fail --silent --show-error --location \
            "https://github.com/google/osv-scanner/releases/download/${OSV_VERSION}/osv-scanner_linux_amd64" \
            --output "${OSV_BIN}"
          echo "${OSV_SHA256}  ${OSV_BIN}" | sha256sum -c -
          chmod +x "${OSV_BIN}"
          "${OSV_BIN}" --config=osv-scanner.toml --lockfile=uv.lock

      - name: Security - Code analysis (bandit)
        run: .venv/bin/bandit -r src/ -ll --format json -o bandit-report.json

      - name: Upload security reports
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7.0.1
        with:
          name: security-reports
          path: |
            bandit-report.json
          retention-days: 30

      # PHASE 4: Test suite with coverage (hermetic; live tests run in the E2E job)
      - name: Run test suite with coverage
        run: |
          .venv/bin/pytest tests/ -v \
            -m "not live and not lint" \
            --cov=coder_eval \
            --cov-report=term-missing \
            --cov-report=xml \
            --cov-fail-under=80

      - name: Upload coverage report
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7.0.1
        with:
          name: coverage-report
          path: coverage.xml
          retention-days: 30

      # Summary
      - name: Summary
        if: always()
        run: |
          echo "✅ Quality gate complete!"
          echo "📊 All checks passed: formatting, linting, types, security, tests"

  evalboard:
    # The dashboard's own gate. `evalboard/` ships ~460 vitest assertions, and
    # until this job existed NOTHING ran them: not a workflow, not a Makefile
    # target, not a pre-commit hook. The pricing guard was consequently red on
    # `main` for weeks while a 3x-wrong Opus rate and five unpriced in-use models
    # shipped to the board. An unrun assertion is documentation, not enforcement.
    #
    # Rate-table drift is no longer this job's concern: lib/pricing.generated.ts
    # is GENERATED from src/coder_eval/pricing.py, and CE065 in `quality-gate`
    # fails a reprice that was not regenerated. What runs here is the CONSUMPTION
    # half (pricing-generated.test.ts) — a generated file that is missing, empty
    # or narrow fails the board's own build.
    #
    # Deliberately NOT path-filtered. `paths:` is workflow-scoped in GitHub
    # Actions, and a skipped required check blocks a PR rather than passing it —
    # so the filter buys nothing and costs a merge-blocking pending status.
    name: Evalboard (Types, Tests, Build)
    # Fork-PR carve-out — see `quality-gate`. `pnpm install --frozen-lockfile` runs
    # the PR's own lockfile install scripts, same untrusted-code class.
    runs-on: ${{ (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository) && 'ubuntu-latest' || 'uipath-ubuntu-latest' }}
    timeout-minutes: 15
    steps:
      - name: Checkout code
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - name: Set up Node.js 20
        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
        with:
          node-version: "20"

      - name: Enable pnpm
        # The version comes from evalboard/package.json's `packageManager` field,
        # so corepack pins it without a second place to keep in sync.
        run: corepack enable

      - name: Install dependencies (lockfile-pinned)
        working-directory: evalboard
        run: pnpm install --frozen-lockfile

      - name: Verify (tsc --noEmit && vitest run && next build)
        working-directory: evalboard
        run: pnpm verify

  plugin-validate:
    # Proves the Claude Code plugin marketplace is installable and that the suite
    # `check-skill` scaffolds is real: the manifests pass strict validation, and the
    # bundled activation template both schema-validates and expands to one task per
    # dataset row. Needs no credentials — nothing here invokes a model.
    name: Claude Code Plugin (manifests + offline scaffold)
    runs-on: ubuntu-latest
    timeout-minutes: 10
    steps:
      - name: Checkout code
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      # `claude plugin validate` ships in the Claude Code npm package.
      - name: Set up Node.js 20
        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
        with:
          node-version: "20"

      - name: Install Claude CLI
        run: npm install -g @anthropic-ai/claude-code

      - name: Validate plugin manifest (strict)
        run: claude plugin validate ./plugins/coder-eval --strict

      - name: Validate marketplace manifest (strict)
        run: claude plugin validate . --strict

      - name: Install uv
        uses: astral-sh/setup-uv@38f3f104447c67c051c4a08e39b64a148898af3a # v4.2.0

      # Runs OUTSIDE the source tree, the way a user's copy of the template does:
      # no experiments/, no tasks/, no coder_eval checkout on the path. `plan` is a
      # schema check only (it exits 0 even when dataset.paths names a nonexistent
      # file), so the row-count assert goes through expand_dataset — otherwise this
      # step would pass even if activation-rows.jsonl were never copied.
      # Reproduce locally with: SCRATCH=$(mktemp -d) VENV=$(mktemp -d)/venv
      - name: Scaffold assert (no source tree)
        run: |
          set -euo pipefail
          SCRATCH="$RUNNER_TEMP/scratch"
          VENV="$RUNNER_TEMP/venv"
          mkdir -p "$SCRATCH"
          cp plugins/coder-eval/reference/templates/activation.yaml "$SCRATCH/"
          cp plugins/coder-eval/reference/templates/activation-rows.jsonl "$SCRATCH/"
          # A venv (not `uv tool install`) because the expansion assert needs
          # `coder_eval` importable, not just the `coder-eval` CLI on PATH.
          uv venv "$VENV"
          VIRTUAL_ENV="$VENV" uv pip install .
          cd "$SCRATCH"
          "$VENV/bin/coder-eval" plan activation.yaml
          "$VENV/bin/python" - <<'PY'
          from pathlib import Path
          from coder_eval.orchestration.task_loader import expand_dataset, load_task
          task, _ = load_task(Path("activation.yaml"))
          rows = expand_dataset(task, Path("."))
          assert len(rows) == 6, f"expected 6 row-tasks, got {len(rows)}"
          labels = {c.expected_skill for t in rows for c in t.success_criteria}
          assert labels == {"my-skill", ""}, labels
          print(f"ok: {len(rows)} row-tasks")
          PY

  no-uipath-extra:
    # Proves that `pip install coder-eval` (without the optional `[uipath]`
    # extra) yields a working framework: imports succeed, the criterion
    # registry validates, and the uipath-specific code paths fail with a
    # clear hint instead of an import error.
    name: No-Extra Install (uipath optional)
    # Fork-PR carve-out — see the comment on `quality-gate` above.
    runs-on: ${{ (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository) && 'ubuntu-latest' || 'uipath-ubuntu-latest' }}
    timeout-minutes: 5
    steps:
      - name: Checkout code
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2

      - name: Set up Python 3.13
        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405  # v6.2.0
        with:
          python-version: "3.13"

      - name: Install uv
        run: |
          python -m pip install --upgrade "pip>=26.2"  # PYSEC-2026-3721 fix floor
          pip install uv

      - name: Install project without [uipath] extra
        run: |
          uv venv .venv --clear
          # Only the `dev` extra is requested — `[uipath]` is deliberately omitted.
          uv pip install --python .venv/bin/python -e ".[dev]"

      - name: Verify uipath SDK is NOT installed
        run: |
          set -e
          ! .venv/bin/python -c "import uipath" 2>/dev/null
          echo "✓ uipath absent as expected"

      - name: Verify framework imports succeed without the extra
        run: |
          .venv/bin/python -c "
          import coder_eval
          import coder_eval.models
          from coder_eval.criteria import init_criteria, CriterionRegistry
          init_criteria(validate=True)
          assert 'llm_judge' in CriterionRegistry.list_types()
          assert 'uipath_eval' in CriterionRegistry.list_types()
          print('✓ framework imports + criterion registry OK')
          "

      - name: Run optional-dependency test suite
        run: .venv/bin/pytest tests/test_optional_dependencies.py -v --no-header

  windows-smoke:
    name: Windows Smoke Test
    runs-on: uipath-windows-latest
    # 15min headroom: the smoke task itself completes in ~7min, but the
    # actions/cache post-step on Windows is slow when ``.venv`` is large.
    # We also exclude ``.venv`` from the cached paths (uv re-creates it
    # fast from the cached ``uv`` download dir), so 15min is comfortable
    # with margin for transient network slowness on the dep install.
    timeout-minutes: 15
    # Skip on fork PRs where secrets aren't available (matches e2e-smoke).
    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
    defaults:
      run:
        shell: bash

    env:
      # Job-level dummy key keeps the unit-test step deterministic and isolated
      # from real-API leakage. The e2e step below overrides to Bedrock at step scope.
      ANTHROPIC_API_KEY: "sk-ant-test-dummy-key-for-ci-tests-only"
      # Bedrock secrets at job scope so the e2e step can engage BedrockRoute.
      # See e2e-smoke for rationale on routing smoke through Bedrock.
      AWS_BEARER_TOKEN_BEDROCK: ${{ secrets.AWS_BEARER_TOKEN_BEDROCK }}
      AWS_REGION: ${{ secrets.AWS_REGION }}
      BEDROCK_MODEL: ${{ secrets.BEDROCK_MODEL }}

    steps:
      - name: Checkout code
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2

      - name: Set up Python 3.13
        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405  # v6.2.0
        with:
          python-version: "3.13"

      - name: Cache dependencies
        uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae  # v5.0.5
        with:
          # ``.venv`` is deliberately excluded: tarring/untarring tens of
          # thousands of small wheel files on Windows is what was tripping
          # the 10-minute job timeout. Caching only the uv/pip download
          # caches keeps ``uv sync --frozen`` fast (it hydrates ``.venv``
          # from the wheel cache without re-downloading) at a fraction of
          # the compress/decompress cost.
          path: |
            ~/AppData/Local/uv/cache
            ~/AppData/Local/pip/Cache
            .pytest_cache
            .ruff_cache
          key: ${{ runner.os }}-py3.13-${{ hashFiles('pyproject.toml', 'uv.lock') }}
          restore-keys: |
            ${{ runner.os }}-py3.13-

      - name: Install uv
        run: |
          python -m pip install --upgrade "pip>=26.2"  # PYSEC-2026-3721 fix floor
          pip install uv

      - name: Install project dependencies (hash-verified from uv.lock)
        # --extra harbor: pyright below type-checks src/coder_eval/harbor/agent.py
        # against harbor's real types, not a scoped ignore.
        run: uv sync --frozen --extra dev --extra uipath --extra codex --extra litellm --extra harbor

      - name: Check code formatting (ruff format)
        run: .venv/Scripts/ruff format --check src/ tests/

      - name: Lint code (ruff check)
        run: .venv/Scripts/ruff check src/ tests/

      - name: Type check with pyright
        run: .venv/Scripts/pyright

      - name: Type check the CE036 contract engine
        run: |
          .venv/Scripts/python -m tests.lint.pyright_config .pyright-tests.json
          .venv/Scripts/pyright -p .pyright-tests.json

      - name: Run test suite
        run: .venv/Scripts/pytest tests/ -v -m "not live and not lint"

      - name: Set up Node.js 20
        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020  # v4.4.0
        with:
          node-version: "20"

      - name: Install Claude CLI (Windows)
        run: npm install -g @anthropic-ai/claude-code

      - name: Verify Claude CLI on PATH (Windows)
        # Fail-fast diagnostic: if `claude` isn't resolvable here, the next
        # step's failure surfaces inside the SDK rather than at the CLI layer,
        # which is harder to triage. Cheap insurance.
        run: claude --version

      - name: Run hello_date smoke task
        env:
          # Step-level override engages BedrockRoute; job-level dummy
          # ANTHROPIC_API_KEY stays in scope for the unit-test step above.
          API_BACKEND: "bedrock"
        # No --model: under API_BACKEND=bedrock the agent layer would prepend
        # eu.anthropic./us.anthropic. to a bare alias like
        # claude-haiku-4-5-20251001 and produce an inference-profile id Bedrock
        # rejects with 400. Falling back to BEDROCK_MODEL (which is already a
        # valid Bedrock profile id) is the same pattern live-tests uses.
        run: .venv/Scripts/coder-eval run tasks/hello_date.yaml --run-dir runs/win-smoke

      - name: Verify hello_date result
        shell: bash
        run: |
          F=runs/win-smoke/experiment.json
          test -f "$F" || { echo "$F missing"; ls -la runs/ 2>/dev/null; exit 1; }
          SUCCEEDED=$(jq '[.variant_aggregates[].tasks_succeeded] | add' "$F")
          FAILED=$(jq '[.variant_aggregates[].tasks_failed] | add' "$F")
          echo "windows hello_date: succeeded=$SUCCEEDED failed=$FAILED"
          test "$SUCCEEDED" = "1" || { echo "Expected 1 success on Windows hello_date, got $SUCCEEDED"; exit 1; }
          test "$FAILED" = "0" || { echo "Windows hello_date failed"; exit 1; }

      - name: Upload Windows smoke artifacts on failure
        if: failure()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7.0.1
        with:
          name: windows-smoke-runs
          path: runs/win-smoke/
          retention-days: 7

  e2e-smoke:
    name: E2E Smoke Tests (Real API)
    runs-on: uipath-ubuntu-latest
    # 15 (was 10): the bucket now includes anti_cheat_reference, a driver: docker
    # task that spins its own container on top of the two image builds this job
    # already does. Headroom, not an expected duration.
    timeout-minutes: 15
    # Skip on fork PRs where secrets aren't available
    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository

    env:
      # Route smoke runs through Bedrock to keep Anthropic-credit spend off the
      # critical path. ClaudeCodeAgent's BedrockRoute is exercised end-to-end
      # by the live-tests BedrockRoute step, so the same path is used here.
      # The DirectRoute path is still tested every PR by live-tests' DirectRoute
      # step, so smoke coverage of the direct-API code path is not lost.
      API_BACKEND: "bedrock"
      AWS_BEARER_TOKEN_BEDROCK: ${{ secrets.AWS_BEARER_TOKEN_BEDROCK }}
      AWS_REGION: ${{ secrets.AWS_REGION }}
      BEDROCK_MODEL: ${{ secrets.BEDROCK_MODEL }}
      # Grades smoke_system_one_judge. Not a Bedrock credential: the System One
      # judge calls TypeSafe directly and ignores the run's API backend. A
      # missing key escalates that task to ERROR rather than failing a criterion,
      # so the preflight step below fails loudly instead.
      TYPESAFE_API_KEY: ${{ secrets.TYPESAFE_API_KEY }}
      # tasks_run for --tags smoke-pass. 9 task files (hello_date, dataset_example,
      # smoke_llm_judge, smoke_agent_judge, smoke_system_one_judge, byod_smoke_test,
      # agentless_smoke_test, anti_cheat_reference, record_cli_responses);
      # dataset_example fans out to 2 inline rows, so 10 sub-tasks. If you
      # add/remove a smoke-pass task or change the dataset row count, bump these.
      #
      # anti_cheat_reference lives in a SUBDIRECTORY, which `tasks/*.yaml` does not
      # match — the smoke-pass step names its path explicitly. Keep that in sync.
      EXPECTED_SMOKE_PASS_RUN: "10"
      EXPECTED_SMOKE_PASS_SUCCEEDED: "10"
      # smoke-fail bucket: three tasks expected to fail.
      #   1. smoke_negative_path: file_contains criterion is unsatisfiable
      #      (sentinel-string regression detection for success-checker).
      #   2. smoke_task_timeout: a `sleep 300` Bash call vs task_timeout: 30
      #      (regression detection for the orchestrator's task_timeout watchdog).
      #   3. smoke_budget_exceeded: max_input_tokens: 1 (unsatisfiable) — guards
      #      run_limits enforcement (regression detection: a disabled budget gate
      #      would let the task pass criteria and land in tasks_succeeded).
      EXPECTED_SMOKE_FAIL_RUN: "3"
      EXPECTED_SMOKE_FAIL_FAILED: "3"
      # smoke-variants bucket: one task fanned out across 2 experiment variants
      # (experiments/smoke_variants.yaml) → tasks_run = 1 * 2.
      EXPECTED_SMOKE_VARIANTS_RUN: "2"
      EXPECTED_SMOKE_VARIANTS_SUCCEEDED: "2"

    steps:
      - name: Checkout code
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2

      - name: Set up Python 3.13
        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405  # v6.2.0
        with:
          python-version: "3.13"

      - name: Set up Node.js 20
        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020  # v4.4.0
        with:
          node-version: "20"

      - name: Install Claude CLI
        run: npm install -g @anthropic-ai/claude-code

      - name: Cache dependencies
        uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae  # v5.0.5
        with:
          # ``.venv`` excluded: a hosted-Python patch bump orphans its
          # interpreter symlink (dangling ``.venv/bin/python`` -> exit 127).
          # ``uv sync --frozen`` rehydrates it from the cached wheels.
          path: |
            ~/.cache/uv
            ~/.cache/pip
          key: ${{ runner.os }}-py3.13-e2e-${{ hashFiles('pyproject.toml', 'uv.lock') }}
          restore-keys: |
            ${{ runner.os }}-py3.13-e2e-
            ${{ runner.os }}-py3.13-

      - name: Install uv
        run: |
          python -m pip install --upgrade "pip>=26.2"  # PYSEC-2026-3721 fix floor
          pip install uv

      - name: Install project dependencies (hash-verified from uv.lock)
        run: uv sync --frozen --extra dev --extra uipath --extra codex

      - name: Build coder-eval-agent base Docker image
        run: make docker-image

      - name: Build BYOD template Docker image
        run: docker build -t byod-custom-image:0.1.0 templates/byod_smoke_test/

      # All three smoke steps below omit --model. Under API_BACKEND=bedrock,
      # the agent layer prepends eu.anthropic./us.anthropic. to a bare alias
      # like claude-haiku-4-5-20251001 and produces an inference-profile id
      # that Bedrock rejects with 400 (no such cross-region profile). Falling
      # back to BEDROCK_MODEL — a valid pre-formatted Bedrock profile id — is
      # the same pattern live-tests uses (see test_claude_settings_enforcement_live._model_for_env).
      # `tasks/*.yaml` is NOT recursive, so subdirectory tasks are listed
      # explicitly. anti_cheat_reference is the adversarial probe that the agent
      # cannot read the reference solution during its turn; it needs the
      # coder-eval-agent image built above (it is a driver: docker task).
      # record_cli_responses is the record_cli per-invocation-response probe and
      # is also driver: docker, so it needs that same image; it is flat in
      # tasks/, so the glob already matches it.
      - name: Verify smoke secrets present
        # smoke_system_one_judge grades through TypeSafe. Without the key the
        # criterion raises JudgeInfrastructureError and the task lands in
        # tasks_errored, which reads as "the harness broke" rather than "the
        # secret is missing". Fail here, where the message says which.
        run: |
          : "${TYPESAFE_API_KEY:?TYPESAFE_API_KEY missing — needed by smoke_system_one_judge}"
          echo "All smoke secrets present."

      - name: Run smoke-pass bucket (expect all to succeed)
        run: |
          .venv/bin/coder-eval run tasks/*.yaml tasks/anti_cheat_reference/*.yaml \
            --tags smoke-pass \
            --run-dir runs/ci-smoke-pass

      - name: Run smoke-fail bucket (expect failure detection)
        run: |
          # Inverted exit semantics: coder-eval exits 1 when a task fails. We *want* that.
          if .venv/bin/coder-eval run tasks/*.yaml \
              --tags smoke-fail \
              --run-dir runs/ci-smoke-fail; then
            echo "FATAL: smoke-fail task succeeded — failure detection appears broken."
            exit 1
          fi

      - name: Run smoke-variants bucket (multi-variant resolver)
        run: |
          .venv/bin/coder-eval run tasks/*.yaml \
            --tags smoke-variants \
            --experiment experiments/smoke_variants.yaml \
            --run-dir runs/ci-smoke-variants

      - name: Verify smoke-pass bucket
        run: |
          F=runs/ci-smoke-pass/experiment.json
          test -f "$F" || { echo "$F missing"; ls -la runs/; exit 1; }
          RUN=$(jq '[.variant_aggregates[].tasks_run] | add' "$F")
          SUCCEEDED=$(jq '[.variant_aggregates[].tasks_succeeded] | add' "$F")
          FAILED=$(jq '[.variant_aggregates[].tasks_failed] | add' "$F")
          ERRORED=$(jq '[.variant_aggregates[].tasks_error] | add' "$F")
          echo "smoke-pass: run=$RUN succeeded=$SUCCEEDED failed=$FAILED errored=$ERRORED"
          test "$RUN" = "$EXPECTED_SMOKE_PASS_RUN" || { echo "smoke-pass tasks_run drift: expected $EXPECTED_SMOKE_PASS_RUN, got $RUN"; exit 1; }
          test "$SUCCEEDED" = "$EXPECTED_SMOKE_PASS_SUCCEEDED" || { echo "smoke-pass succeeded drift: expected $EXPECTED_SMOKE_PASS_SUCCEEDED, got $SUCCEEDED"; exit 1; }
          test "$FAILED" = "0" || { echo "smoke-pass had unexpected failures"; exit 1; }
          test "$ERRORED" = "0" || { echo "smoke-pass had errors"; exit 1; }

      # The four wall-clock buckets (head + generation + UNION(tool) + tail)
      # must account for each turn's own duration. This is the TWO-SIDED gate:
      # the committed golden sensor only catches an OVERSHOOT, so a bucket that
      # claims LESS time than it should — the defect class this area keeps
      # producing — passes every test in the suite. It needs live task.json
      # files, which the smoke-pass run above already leaves on disk.
      #
      # COVERS CLAUDE-CODE ONLY: experiments/default.yaml sets type: claude-code,
      # so every turn here is that harness. The other four are covered by
      # tests/test_timing_identity_contract.py, which is ms-exact but synthetic.
      - name: Verify timing residual (claude-code only)
        run: |
          .venv/bin/python scripts/timing/decompose_run.py \
            $(find runs/ci-smoke-pass -name task.json) --max-residual-pct 5

      - name: Verify smoke-fail bucket
        run: |
          F=runs/ci-smoke-fail/experiment.json
          test -f "$F" || { echo "$F missing"; ls -la runs/; exit 1; }
          RUN=$(jq '[.variant_aggregates[].tasks_run] | add' "$F")
          FAILED=$(jq '[.variant_aggregates[].tasks_failed] | add' "$F")
          ERRORED=$(jq '[.variant_aggregates[].tasks_error] | add' "$F")
          echo "smoke-fail: run=$RUN failed=$FAILED errored=$ERRORED"
          test "$RUN" = "$EXPECTED_SMOKE_FAIL_RUN" || { echo "smoke-fail tasks_run drift: expected $EXPECTED_SMOKE_FAIL_RUN, got $RUN"; exit 1; }
          test "$FAILED" = "$EXPECTED_SMOKE_FAIL_FAILED" || { echo "smoke-fail expected $EXPECTED_SMOKE_FAIL_FAILED failure(s), got $FAILED — failure detection may be broken"; exit 1; }
          test "$ERRORED" = "0" || { echo "smoke-fail unexpectedly errored (not the same as failing)"; exit 1; }

      - name: Verify smoke-variants bucket
        run: |
          F=runs/ci-smoke-variants/experiment.json
          test -f "$F" || { echo "$F missing"; ls -la runs/; exit 1; }
          RUN=$(jq '[.variant_aggregates[].tasks_run] | add' "$F")
          SUCCEEDED=$(jq '[.variant_aggregates[].tasks_succeeded] | add' "$F")
          ERRORED=$(jq '[.variant_aggregates[].tasks_error] | add' "$F")
          VARIANTS=$(jq '.variant_aggregates | length' "$F")
          echo "smoke-variants: variants=$VARIANTS run=$RUN succeeded=$SUCCEEDED errored=$ERRORED"
          test "$VARIANTS" = "2" || { echo "smoke-variants expected 2 variants, got $VARIANTS — multi-variant fan-out may be broken"; exit 1; }
          test "$RUN" = "$EXPECTED_SMOKE_VARIANTS_RUN" || { echo "smoke-variants tasks_run drift: expected $EXPECTED_SMOKE_VARIANTS_RUN, got $RUN"; exit 1; }
          test "$SUCCEEDED" = "$EXPECTED_SMOKE_VARIANTS_SUCCEEDED" || { echo "smoke-variants succeeded drift: expected $EXPECTED_SMOKE_VARIANTS_SUCCEEDED, got $SUCCEEDED"; exit 1; }
          test "$ERRORED" = "0" || { echo "smoke-variants had errors"; exit 1; }

      - name: Re-evaluate hello_date via `coder-eval evaluate` (no agent)
        # PR #220 fixed a regression in evaluate-only mode (reference loading).
        # This step re-runs hello_date.yaml's criteria against the artifacts
        # the smoke-pass run already produced — no new agent invocation, so
        # zero extra LLM calls. A regression in the evaluate code path
        # surfaces as a non-zero exit code from `coder-eval evaluate`.
        run: |
          WORK="runs/ci-smoke-pass/default/hello_date_smoke_test/00/artifacts/hello_date_smoke_test"
          test -d "$WORK" || { echo "Expected artifacts dir not found: $WORK"; find runs/ci-smoke-pass -maxdepth 5 -type d 2>/dev/null; exit 1; }
          .venv/bin/coder-eval evaluate tasks/hello_date.yaml "$WORK" \
            --run-dir runs/ci-smoke-eval-only

      - name: Debug SSH session on failure
        if: failure()
        uses: mxschmitt/action-tmate@35b54afac29c97fb54faba5b513f8fbd1882f113  # v3.x
        with:
          limit-access-to-actor: true

      - name: Upload run artifacts on failure
        if: failure()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7.0.1
        with:
          name: e2e-smoke-runs
          path: |
            runs/ci-smoke-pass/
            runs/ci-smoke-fail/
            runs/ci-smoke-variants/
            runs/ci-smoke-eval-only/
          retention-days: 7

  live-tests:
    name: Live Integration Tests (Settings Enforcement + Cost Budget)
    runs-on: uipath-ubuntu-24.04
    timeout-minutes: 15
    # Skip on fork PRs where secrets aren't available
    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository

    env:
      # Job-level env: default to DirectRoute (no API_BACKEND override). The
      # settings-enforcement and cost-budget steps use this. The Bedrock
      # settings-enforcement step adds API_BACKEND=bedrock at step scope only.
      ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
      # Bedrock backend for the third settings-enforcement run. Job-level so
      # the preflight secrets check can validate them; the BedrockRoute is
      # only engaged via API_BACKEND=bedrock at step scope.
      AWS_BEARER_TOKEN_BEDROCK: ${{ secrets.AWS_BEARER_TOKEN_BEDROCK }}
      AWS_REGION: ${{ secrets.AWS_REGION }}
      BEDROCK_MODEL: ${{ secrets.BEDROCK_MODEL }}
      # The System One judge's own endpoint — unrelated to either route above.
      TYPESAFE_API_KEY: ${{ secrets.TYPESAFE_API_KEY }}

    steps:
      - name: Checkout code
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2

      - name: Set up Python 3.13
        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405  # v6.2.0
        with:
          python-version: "3.13"

      - name: Set up Node.js 20
        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020  # v4.4.0
        with:
          node-version: "20"

      - name: Install Claude CLI
        run: npm install -g @anthropic-ai/claude-code

      # No bubblewrap/AppArmor setup needed: neither live-test file engages
      # AgentConfig.isolation=true, so the SDK doesn't shell out to bwrap.

      - name: Cache dependencies
        uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae  # v5.0.5
        with:
          # ``.venv`` excluded: a hosted-Python patch bump orphans its
          # interpreter symlink (dangling ``.venv/bin/python`` -> exit 127).
          # ``uv sync --frozen`` rehydrates it from the cached wheels.
          path: |
            ~/.cache/uv
            ~/.cache/pip
          key: ${{ runner.os }}-py3.13-live-${{ hashFiles('pyproject.toml', 'uv.lock') }}
          restore-keys: |
            ${{ runner.os }}-py3.13-live-
            ${{ runner.os }}-py3.13-

      - name: Install uv
        run: |
          python -m pip install --upgrade "pip>=26.2"  # PYSEC-2026-3721 fix floor
          pip install uv

      - name: Install project dependencies (hash-verified from uv.lock)
        run: uv sync --frozen --extra dev --extra uipath --extra codex

      - name: Verify required secrets are present
        run: |
          : "${ANTHROPIC_API_KEY:?ANTHROPIC_API_KEY missing}"
          : "${AWS_BEARER_TOKEN_BEDROCK:?AWS_BEARER_TOKEN_BEDROCK missing}"
          : "${AWS_REGION:?AWS_REGION missing}"
          : "${BEDROCK_MODEL:?BEDROCK_MODEL missing}"
          : "${TYPESAFE_API_KEY:?TYPESAFE_API_KEY missing}"
          echo "All live-test secrets present."

      - name: Run claude-settings enforcement live tests (DirectRoute)
        # No API_BACKEND override — Settings() defaults to DIRECT and uses ANTHROPIC_API_KEY.
        # `-ra` surfaces skipped tests in the summary so CI logs show what ran vs skipped.
        # `--strict-markers` rejects unregistered @pytest.mark.* (cheap typo insurance).
        # JUnit XML feeds the post-run "tests actually passed" assertion below.
        # `-n 4` overrides pyproject's `-n auto`: these hit the real API, so
        # concurrency must not vary with the runner's vCPU count. 4 = the old shape.
        run: |
          mkdir -p tmp
          .venv/bin/pytest tests/test_claude_settings_enforcement_live.py \
            -m live -v --tb=short --strict-markers -ra -n 4 \
            --junit-xml=tmp/junit-settings.xml

      - name: Run claude-settings enforcement live tests (BedrockRoute)
        # Same test file, different backend. Settings-enforcement lives in the
        # Claude Code CLI and is backend-agnostic (per the file's docstring),
        # so this run is the only thing in CI that exercises BedrockRoute
        # end-to-end with the real Anthropic-on-Bedrock model.
        env:
          API_BACKEND: "bedrock"
        run: |
          .venv/bin/pytest tests/test_claude_settings_enforcement_live.py \
            -m live -v --tb=short --strict-markers -ra -n 4 \
            --junit-xml=tmp/junit-settings-bedrock.xml

      - name: Run System One judge wire-contract live tests
        # The only thing in CI that talks to TypeSafe. The unit tests mock the
        # invoker, so nothing else catches a change to the answer shape the
        # reduction assumes — notably the STRING level keys ("0", "1", ...) in a
        # score answer's probabilities. `-n0`: three questions in one round trip,
        # so there is nothing to parallelize.
        run: |
          .venv/bin/pytest tests/test_system_one_judge_live.py \
            -m live -n0 -v --tb=short --strict-markers -ra \
            --junit-xml=tmp/junit-system-one.xml

      - name: Assert live tests actually ran (not silently skipped)
        # Parse JUnit XML for *passed* count, not collected count. Pytest collects
        # @pytest.mark.skipif-marked tests even when the predicate is True, so a
        # collect-only count would still report N>0 even if every test silently
        # skipped — defeating the purpose of the assertion. Passed = tests - skipped
        # - errors - failures correctly catches the silent-skip regression class.
        run: |
          python - <<'PY'
          import sys, xml.etree.ElementTree as ET
          def passed(path: str) -> int:
              # Aggregate across all <testsuite> elements: pytest 8.x normally
              # emits a single suite, but xdist or plugin variations can produce
              # multiple suites under a <testsuites> root. Counting only the
              # first would silently undercount and falsely fail the assertion.
              root = ET.parse(path).getroot()
              suites = [root] if root.tag == "testsuite" else root.findall("testsuite")
              if not suites:
                  sys.exit(f"{path}: no <testsuite> element")
              total = sum(int(ts.get("tests", 0)) for ts in suites)
              skipped = sum(int(ts.get("skipped", 0)) for ts in suites)
              errors = sum(int(ts.get("errors", 0)) for ts in suites)
              failures = sum(int(ts.get("failures", 0)) for ts in suites)
              return total - skipped - errors - failures
          p_settings_direct = passed("tmp/junit-settings.xml")
          p_settings_bedrock = passed("tmp/junit-settings-bedrock.xml")
          p_system_one = passed("tmp/junit-system-one.xml")
          print(
              f"Passed: settings(direct)={p_settings_direct}, "
              f"settings(bedrock)={p_settings_bedrock}, system_one={p_system_one}"
          )
          if p_settings_direct < 1:
              sys.exit("test_claude_settings_enforcement_live.py (DirectRoute) reported zero PASSED tests")
          if p_settings_bedrock < 1:
              sys.exit("test_claude_settings_enforcement_live.py (BedrockRoute) reported zero PASSED tests")
          if p_system_one < 1:
              sys.exit("test_system_one_judge_live.py reported zero PASSED tests")
          PY

      - name: Run cost-budget smoke (max_usd → COST_BUDGET_EXCEEDED via DirectRoute)
        # Verifies the cost path end-to-end on DirectRoute (Anthropic reports
        # per-turn cost). A regression that broke per-turn cost reporting would
        # log "max_usd budget configured but no turn reported cost; skipping
        # cost check" and the task would land in tasks_failed via the
        # placeholder criterion instead of COST_BUDGET_EXCEEDED. The
        # final_status assertion below catches that silently-broken case.
        run: |
          # Inverted exit semantics: coder-eval exits 1 when a task fails. We want that.
          if .venv/bin/coder-eval run tasks/smoke_cost_budget_exceeded.yaml \
              --model claude-haiku-4-5-20251001 \
              --run-dir runs/ci-smoke-cost-budget; then
            echo "FATAL: cost-budget smoke succeeded — cost enforcement appears broken."
            exit 1
          fi

      - name: Verify cost-budget smoke (COST_BUDGET_EXCEEDED, not just failed)
        run: |
          F=runs/ci-smoke-cost-budget/experiment.json
          test -f "$F" || { echo "$F missing"; ls -la runs/; exit 1; }
          RUN=$(jq '[.variant_aggregates[].tasks_run] | add' "$F")
          FAILED=$(jq '[.variant_aggregates[].tasks_failed] | add' "$F")
          ERRORED=$(jq '[.variant_aggregates[].tasks_error] | add' "$F")
          echo "smoke-cost-budget: run=$RUN failed=$FAILED errored=$ERRORED"
          test "$RUN" = "1" || { echo "smoke-cost-budget tasks_run drift: expected 1, got $RUN"; exit 1; }
          test "$FAILED" = "1" || { echo "smoke-cost-budget expected 1 failure, got $FAILED"; exit 1; }
          test "$ERRORED" = "0" || { echo "smoke-cost-budget unexpectedly errored"; exit 1; }
          # Specifically assert the budget gate fired — not just that the task failed.
          # A regression in per-turn cost reporting would still produce failed=1 via the
          # placeholder criterion, but final_status would be FAILURE, not COST_BUDGET_EXCEEDED.
          TASK_JSON=runs/ci-smoke-cost-budget/default/smoke_cost_budget_exceeded/00/task.json
          test -f "$TASK_JSON" || { echo "$TASK_JSON missing"; find runs/ci-smoke-cost-budget -maxdepth 5 -type f; exit 1; }
          STATUS=$(jq -r '.final_status' "$TASK_JSON")
          echo "final_status=$STATUS"
          test "$STATUS" = "COST_BUDGET_EXCEEDED" || { echo "Expected final_status=COST_BUDGET_EXCEEDED, got $STATUS — cost gate may not have fired (cost telemetry regression?)"; exit 1; }

      - name: Upload live-test artifacts on failure
        if: failure()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7.0.1
        with:
          name: live-tests-output
          path: |
            tmp/
            runs/
          retention-days: 7

  codex-live-tests:
    name: Live Integration Tests (Codex)
    runs-on: uipath-ubuntu-24.04
    timeout-minutes: 15
    # Skip on fork PRs where secrets aren't available.
    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository

    env:
      # Codex SDK auth. CODEX_API_KEY is the key; CODEX_BASE_URL routes to a
      # custom OpenAI-/responses-compatible endpoint and CODEX_MODEL pins the
      # model/deployment that endpoint expects.
      CODEX_API_KEY: ${{ secrets.CODEX_API_KEY }}
      CODEX_BASE_URL: ${{ secrets.CODEX_BASE_URL }}
      CODEX_MODEL: ${{ secrets.CODEX_MODEL }}

    steps:
      - name: Checkout code
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2

      - name: Set up Python 3.13
        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405  # v6.2.0
        with:
          python-version: "3.13"

      - name: Install uv
        run: |
          python -m pip install --upgrade "pip>=26.2"  # PYSEC-2026-3721 fix floor
          pip install uv

      - name: Install project dependencies (with codex + litellm extras)
        run: uv sync --frozen --extra dev --extra uipath --extra codex --extra litellm

      - name: Verify required secrets are present
        run: |
          if [ -z "$CODEX_API_KEY" ]; then
            echo "CODEX_API_KEY is not set."
            exit 1
          fi
          echo "CODEX_API_KEY present."

      - name: Run Codex + litellm-judge live tests
        run: |
          mkdir -p tmp
          # Run serially: `-n0` overrides the global `-n auto` (addopts).
          # Parallel xdist workers share ~/.codex and race the Codex SQLite
          # state migration (`duplicate column name: thread_id`); serial init
          # migrates the fresh DB exactly once. test_litellm_judge_live.py
          # reuses these same CODEX_* secrets to exercise
          # checker_context.api_route.route: litellm end-to-end (PR #137
          # review: "nothing in the repo exercises the feature").
          .venv/bin/pytest tests/test_codex_agent_live.py tests/test_litellm_judge_live.py \
            -m live -n0 -v --tb=short --strict-markers -ra \
            --junit-xml=tmp/junit-codex-live.xml

      - name: Assert Codex live tests actually ran (not silently skipped)
        run: |
          python - <<'PY'
          import sys, xml.etree.ElementTree as ET
          root = ET.parse("tmp/junit-codex-live.xml").getroot()
          suites = [root] if root.tag == "testsuite" else root.findall("testsuite")
          total = sum(int(s.get("tests", 0)) for s in suites)
          skipped = sum(int(s.get("skipped", 0)) for s in suites)
          errors = sum(int(s.get("errors", 0)) for s in suites)
          failures = sum(int(s.get("failures", 0)) for s in suites)
          passed = total - skipped - errors - failures
          print(f"codex-live passed={passed} skipped={skipped} errors={errors} failures={failures}")
          if passed < 1:
              sys.exit("Live Codex/litellm-judge tests reported zero PASSED tests (missing API key / silent skip?)")
          PY

      - name: Upload Codex live-test artifacts on failure
        if: failure()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7.0.1
        with:
          name: codex-live-output
          path: tmp/
          retention-days: 7

  byoa-live-tests:
    name: Live Integration Tests (BYOA Plugin)
    runs-on: uipath-ubuntu-24.04
    timeout-minutes: 15
    # Skip on fork PRs where secrets aren't available.
    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository

    env:
      # DirectRoute: a plugin agent that subclasses ClaudeCodeAgent uses ANTHROPIC_API_KEY.
      ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}

    steps:
      - name: Checkout code
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2

      - name: Set up Python 3.13
        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405  # v6.2.0
        with:
          python-version: "3.13"

      - name: Set up Node.js 20
        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020  # v4.4.0
        with:
          node-version: "20"

      - name: Install Claude CLI
        run: npm install -g @anthropic-ai/claude-code

      - name: Install uv
        run: |
          python -m pip install --upgrade "pip>=26.2"  # PYSEC-2026-3721 fix floor
          pip install uv

      - name: Install project dependencies (hash-verified from uv.lock)
        run: uv sync --frozen --extra dev --extra uipath

      # Install the out-of-tree fixture plugin so coder-eval discovers 'byoa-demo'
      # through the REAL coder_eval.plugins entry point (not a test monkeypatch).
      - name: Install BYOA demo plugin (entry-point discovery)
        run: uv pip install -e tests/fixtures/byoa_demo_plugin

      - name: Verify required secrets are present
        run: |
          : "${ANTHROPIC_API_KEY:?ANTHROPIC_API_KEY missing}"
          echo "ANTHROPIC_API_KEY present."

      - name: Run BYOA plugin live test
        run: |
          mkdir -p tmp
          .venv/bin/pytest tests/test_byoa_plugin_live.py \
            -m live -v --tb=short --strict-markers -ra \
            --junit-xml=tmp/junit-byoa-live.xml

      - name: Assert BYOA live test actually ran (not silently skipped)
        run: |
          python - <<'PY'
          import sys, xml.etree.ElementTree as ET
          root = ET.parse("tmp/junit-byoa-live.xml").getroot()
          suites = [root] if root.tag == "testsuite" else root.findall("testsuite")
          total = sum(int(s.get("tests", 0)) for s in suites)
          skipped = sum(int(s.get("skipped", 0)) for s in suites)
          errors = sum(int(s.get("errors", 0)) for s in suites)
          failures = sum(int(s.get("failures", 0)) for s in suites)
          passed = total - skipped - errors - failures
          print(f"byoa-live passed={passed} skipped={skipped} errors={errors} failures={failures}")
          if passed < 1:
              sys.exit("test_byoa_plugin_live.py reported zero PASSED tests (missing API key / silent skip?)")
          PY

      - name: Upload BYOA live-test artifacts on failure
        if: failure()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7.0.1
        with:
          name: byoa-live-output
          path: tmp/
          retention-days: 7

  action-dogfood:
    name: Action Dogfood (composite action, real API)
    # Deliberately NOT on the `uipath-*` pool, and keep it that way: this job is the
    # executable proof of the published Action, and docs/tutorials/02-ci-pipeline.md
    # (the one consumer snippet naming a runner) says `ubuntu-latest`. Nothing else in
    # CI exercises the image integrators actually use. A bulk `runs-on:` migration has
    # swept this up twice — check it by hand.
    runs-on: ubuntu-latest
    timeout-minutes: 15
    # Skip on fork PRs where secrets aren't available (matches e2e-smoke).
    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2

      # The composite action is agent-agnostic and does NOT install a coding-agent
      # runtime. The dogfood task uses the default claude-code agent, so provide
      # Node + the Claude CLI here (as e2e-smoke does), before invoking the action.
      - name: Set up Node.js 20
        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020  # v4.4.0
        with:
          node-version: "20"
      - name: Install Claude CLI
        run: npm install -g @anthropic-ai/claude-code

      - name: Run coder-eval via local action
        id: dogfood
        uses: ./
        with:
          # Exercises what tests/test_action_inputs.py cannot: it asserts the argv
          # both step scripts build, but only a real runner proves that
          # `working-directory:` works on a composite step and that a plugin
          # installed via `--with` is discovered at runtime. Hence a relative
          # run-dir (landing under tasks/), the bare task filename and the `../`
          # plugin path, all resolved from `working-directory`.
          version: local
          working-directory: tasks
          run-dir: runs/ci-action-dogfood
          extra-packages: ../tests/fixtures/byoa_demo_plugin
          # The bracketed `-D` override is the case one-argv-entry-per-line exists
          # for: `[...]` is a bash character class, so a whitespace-split input
          # would collapse the list whenever a file in the cwd matched. It adds
          # Glob to hello_date.yaml's three tools, so the assertions below can tell
          # "arrived" from "ignored".
          args: |
            hello_date.yaml
            --model
            claude-haiku-4-5-20251001
            -D
            agent.allowed_tools=[Read,Write,Bash,Glob]
          # ANTHROPIC_API_KEY reaching the run is proven by the API-backed task
          # succeeding; the second line exercises multi-line env parsing.
          env: |
            ANTHROPIC_API_KEY=${{ secrets.ANTHROPIC_API_KEY }}
            CE_DOGFOOD_MARKER=1

      # In `tasks/` because that is the assertion: `run-dir` is reported exactly as
      # passed, so a relative one is relative to `working-directory`.
      - name: Verify outputs and JUnit file
        working-directory: tasks
        env:
          JUNIT: ${{ steps.dogfood.outputs.junit-path }}
          RUNDIR: ${{ steps.dogfood.outputs.run-dir }}
        run: |
          set -euo pipefail
          test -n "$JUNIT" && test -f "$JUNIT" || { echo "junit output missing"; exit 1; }
          # Well-formedness check on a file this job just generated (trusted input;
          # our writer emits no DTDs/entities) — stdlib ET is fine here.
          python3 -c "import sys, xml.etree.ElementTree as ET; ET.parse(sys.argv[1])" "$JUNIT"
          test -f "$RUNDIR/run.json" || { echo "run.json missing"; exit 1; }
          # An absolute path would mean working-directory was ignored.
          case "$RUNDIR" in /*) echo "run-dir output was rewritten to an absolute path: $RUNDIR"; exit 1 ;; esac

      # The action does not touch $GITHUB_STEP_SUMMARY: this is both the assertion
      # that `run-md-path` is real and the recipe the docs hand consumers.
      - name: Append the run report to the job summary
        if: always()
        working-directory: tasks
        env:
          RUN_MD: ${{ steps.dogfood.outputs.run-md-path }}
        run: |
          set -euo pipefail
          test -f "$RUN_MD" || { echo "run-md-path output does not exist: $RUN_MD"; exit 1; }
          cat "$RUN_MD" >> "$GITHUB_STEP_SUMMARY"

      - name: Verify the plugin was discovered and the bracketed override arrived
        working-directory: tasks
        env:
          RUNDIR: ${{ steps.dogfood.outputs.run-dir }}
        run: |
          set -euo pipefail

          # `coder-eval` on PATH is the action's uv tool shim, so this interrogates
          # its environment: a task naming the fixture's agent kind validates only
          # if the entry point was discovered there, else plan exits 1 with
          # "No agent registered for type 'byoa-demo'".
          cat > byoa-probe.yaml <<'YAML'
          task_id: "action_extra_packages_probe"
          description: "Validates only when the byoa-demo plugin is discoverable."
          initial_prompt: "not executed - plan validates without running an agent"
          agent:
            type: "byoa-demo"
          success_criteria:
            - type: "file_exists"
              path: "app.py"
              description: "not executed"
          YAML
          coder-eval plan byoa-probe.yaml
          rm -f byoa-probe.yaml

          # And the `-D` value survived as a 4-element list, not word-split or
          # glob-rewritten.
          RUN_JSON="$RUNDIR/run.json" python3 <<'PY'
          import json, os, sys

          data = json.load(open(os.environ["RUN_JSON"], encoding="utf-8"))
          rows = data.get("task_results") or []
          if not rows:
              sys.exit("run.json has no task_results to check the -D override against")
          tools = (rows[0].get("agent_config") or {}).get("allowed_tools")
          expected = ["Read", "Write", "Bash", "Glob"]
          if tools != expected:
              sys.exit(f"-D override did not arrive intact: allowed_tools={tools!r}, expected {expected!r}")
          print(f"bracketed -D override resolved to {tools!r}")
          PY

      - name: Upload dogfood run on failure
        if: failure()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7.0.1
        with:
          name: action-dogfood-runs
          path: tasks/runs/ci-action-dogfood/
          retention-days: 7

  delegate-live-tests:
    # Live Delegate e2e: spawns the real Node host (agents/delegate/delegate_host.mjs)
    # wrapping the public @uipath/delegate-sdk against a real UiPath backend. Runs on
    # every PR/push like the other live-test jobs (codex-live-tests, byoa-live-tests)
    # now that the DELEGATE_ROPC_* secrets are provisioned; the presence-check gate
    # below skips cleanly ONLY on a fork PR (no secrets available there) and fails
    # loudly on a same-repo run with a missing secret, matching the other live jobs.
    # Auth is minted fresh each run via the
    # OAuth2 Resource Owner Password Credentials (ROPC) grant — the same
    # `grant_type=password` flow documented in coder_eval_uipath/eval_runner/CI_DESIGN.md
    # and implemented in coder_eval_uipath/eval_runner/scripts/ci/refresh-auth.sh —
    # rather than a long-lived AUTH_TOKEN secret, since a minted token's ~1h TTL
    # would make a static secret stale between runs. A dedicated bot user's
    # username/password is the durable credential; the Delegate SDK reads the
    # minted token straight from AUTH_TOKEN/TENANT_ID/ORG_ID (see
    # docs/agents/DELEGATE.md), so no sdk-auth.json or client-secret refresh chain
    # is needed here.
    name: Delegate Live E2E (ROPC)
    runs-on: uipath-ubuntu-24.04
    timeout-minutes: 15
    # Skip on fork PRs where secrets aren't available (matches e2e-smoke).
    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository

    steps:
      - name: Checkout code
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2

      - name: Check Delegate ROPC secrets present
        id: gate
        env:
          DELEGATE_ROPC_CLIENT_ID: ${{ secrets.DELEGATE_ROPC_CLIENT_ID }}
          DELEGATE_ROPC_CLIENT_SECRET: ${{ secrets.DELEGATE_ROPC_CLIENT_SECRET }}
          DELEGATE_ROPC_USERNAME: ${{ secrets.DELEGATE_ROPC_USERNAME }}
          DELEGATE_ROPC_PASSWORD: ${{ secrets.DELEGATE_ROPC_PASSWORD }}
          DELEGATE_ORG_ID: ${{ secrets.DELEGATE_ORG_ID }}
          DELEGATE_TENANT_ID: ${{ secrets.DELEGATE_TENANT_ID }}
          DELEGATE_ORG_SLUG: ${{ secrets.DELEGATE_ORG_SLUG }}
          DELEGATE_TENANT_SLUG: ${{ secrets.DELEGATE_TENANT_SLUG }}
          # A fork PR (this job's own `if:` skips the whole job for those) is the
          # only case a missing secret is expected -- everywhere else, silently
          # reporting success having run nothing is exactly the failure a required
          # check exists to catch, so a same-repo run with a rotated/dropped secret
          # fails loudly instead.
          IS_FORK_PR: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository }}
        run: |
          missing=""
          for name in DELEGATE_ROPC_CLIENT_ID DELEGATE_ROPC_CLIENT_SECRET DELEGATE_ROPC_USERNAME \
                      DELEGATE_ROPC_PASSWORD DELEGATE_ORG_ID DELEGATE_TENANT_ID \
                      DELEGATE_ORG_SLUG DELEGATE_TENANT_SLUG; do
            if [ -z "${!name}" ]; then
              missing="$missing $name"
            fi
          done
          if [ -z "$missing" ]; then
            echo "present=true" >> "$GITHUB_OUTPUT"
          elif [ "$IS_FORK_PR" = "true" ]; then
            echo "present=false" >> "$GITHUB_OUTPUT"
            echo "Delegate ROPC secrets not available on a fork PR — skipping the live run."
          else
            echo "present=false" >> "$GITHUB_OUTPUT"
            echo "::error::Delegate ROPC secrets missing on a same-repo run:$missing"
            exit 1
          fi

      - name: Set up Python 3.13
        if: steps.gate.outputs.present == 'true'
        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405  # v6.2.0
        with:
          python-version: "3.13"

      - name: Set up Node.js 20
        if: steps.gate.outputs.present == 'true'
        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020  # v4.4.0
        with:
          node-version: "20"

      - name: Install uv
        if: steps.gate.outputs.present == 'true'
        run: |
          python -m pip install --upgrade "pip>=26.2"  # PYSEC-2026-3721 fix floor
          pip install uv

      - name: Install project dependencies (hash-verified from uv.lock)
        if: steps.gate.outputs.present == 'true'
        run: uv sync --frozen --extra dev

      - name: Install @uipath/delegate-sdk (public npm, no token needed)
        if: steps.gate.outputs.present == 'true'
        working-directory: src/coder_eval/agents/delegate
        # --safe-chain-skip-minimum-package-age (not a growing exclusion-list entry
        # per platform-specific optionalDependency name): the SDK's per-platform
        # native runtime packages (@uipath/delegate-runtime-*) publish at the same
        # instant as the SDK itself, so a name-list exclusion just needing another
        # entry for each new one is the wrong shape for a first-party UiPath package
        # already trusted at the SDK-name level.
        run: npm install --safe-chain-skip-minimum-package-age

      - name: ROPC login — mint a fresh Delegate access token
        if: steps.gate.outputs.present == 'true'
        id: ropc
        env:
          DELEGATE_UIPATH_ENV: ${{ github.event.inputs.delegate_uipath_env || 'alpha' }}
          CLIENT_ID: ${{ secrets.DELEGATE_ROPC_CLIENT_ID }}
          CLIENT_SECRET: ${{ secrets.DELEGATE_ROPC_CLIENT_SECRET }}
          CE_USERNAME: ${{ secrets.DELEGATE_ROPC_USERNAME }}
          CE_PASSWORD: ${{ secrets.DELEGATE_ROPC_PASSWORD }}
          ORG_ID: ${{ secrets.DELEGATE_ORG_ID }}
        run: |
          set -euo pipefail
          # The mint destination is resolved from a fixed allowlist, never from
          # the raw workflow_dispatch input string directly — CLIENT_SECRET and
          # CE_PASSWORD travel in this request's body, so accepting an arbitrary
          # attacker-supplied URL here would be a credential-exfiltration path
          # open to anyone who can trigger workflow_dispatch, not just admins.
          case "${DELEGATE_UIPATH_ENV}" in
            alpha) UIPATH_URL="https://alpha.uipath.com" ;;
            staging) UIPATH_URL="https://staging.uipath.com" ;;
            production) UIPATH_URL="https://cloud.uipath.com" ;;
            *) echo "Unknown delegate_uipath_env: ${DELEGATE_UIPATH_ENV}" >&2; exit 1 ;;
          esac

          # Same grant as coder_eval_uipath/eval_runner/scripts/ci/refresh-auth.sh:
          # a bot user's username/password against the identity server, scoped
          # to the org via acr_values (the ROPC client carries no scope of its
          # own narrower than the org). ConversationalAgents is the scope the
          # Delegate backend itself checks; the rest matches refresh-auth.sh's
          # alpha scope list so this mint is provably not narrower than the one
          # already validated there.
          SCOPES="openid profile email offline_access ConversationalAgents AutopilotForEveryone Orchestrator OrchestratorApiUserAccess StudioWebBackend"
          # The POST body is built here and piped to curl on stdin (--data-binary
          # @-), never passed as a --data-urlencode argv value: argv (unlike
          # stdin) is world-readable via /proc/<pid>/cmdline to any other process
          # on this self-hosted runner for the life of the request, and
          # CLIENT_SECRET/CE_PASSWORD are durable credentials, not the ~1h token.
          body=$(jq -rn \
              --arg client_id "$CLIENT_ID" \
              --arg client_secret "$CLIENT_SECRET" \
              --arg username "$CE_USERNAME" \
              --arg password "$CE_PASSWORD" \
              --arg acr_values "tenant:${ORG_ID}" \
              --arg scope "$SCOPES" \
              '"grant_type=password" +
               "&client_id=" + ($client_id|@uri) +
               "&client_secret=" + ($client_secret|@uri) +
               "&username=" + ($username|@uri) +
               "&password=" + ($password|@uri) +
               "&acr_values=" + ($acr_values|@uri) +
               "&scope=" + ($scope|@uri)')
          response=$(printf '%s' "$body" | curl -sS -L --connect-timeout 15 --max-time 60 -X POST "${UIPATH_URL}/identity_/connect/token" \
              -H "Content-Type: application/x-www-form-urlencoded" \
              -H "x-uipath-internal-accountid: ${ORG_ID}" \
              --data-binary @-)

          if err=$(echo "$response" | jq -e -r '.error // empty') && [ -n "$err" ]; then
              echo "ROPC token request failed: $(echo "$response" | jq -r '.error_description // .error')" >&2
              exit 1
          fi

          access_token=$(echo "$response" | jq -r '.access_token')
          if [ -z "$access_token" ] || [ "$access_token" = "null" ]; then
              echo "ROPC token response did not contain an access_token" >&2
              exit 1
          fi

          echo "::add-mask::$access_token"
          echo "access_token=$access_token" >> "$GITHUB_OUTPUT"

      - name: Diagnose minted token's org/account claims
        # Prints ONLY non-secret JWT claims (never the token itself) to check
        # whether the ROPC-minted token's own account/org actually matches the
        # DELEGATE_ORG_ID/DELEGATE_ORG_SLUG secrets used to build the backend
        # URL -- a mismatch here (not a coder_eval bug) is what a live run's
        # "Account id does not match" 401 from the connections_ API points at.
        # continue-on-error: this step is advisory -- its own script already
        # catches and reports unexpected input, but a step failure must still
        # never gate the real `coder-eval run` below that it exists to help
        # explain, not mask.
        if: steps.gate.outputs.present == 'true'
        continue-on-error: true
        env:
          ACCESS_TOKEN: ${{ steps.ropc.outputs.access_token }}
          EXPECTED_ORG_ID: ${{ secrets.DELEGATE_ORG_ID }}
          EXPECTED_TENANT_ID: ${{ secrets.DELEGATE_TENANT_ID }}
        run: |
          set -euo pipefail
          payload="$(echo "$ACCESS_TOKEN" | cut -d. -f2)"
          # Explicit MATCH/NO MATCH verdicts, not a side-by-side print: a secret's
          # exact value is auto-masked to *** everywhere in logs, including inside
          # an unrelated claims dump, so printing both for a human to eyeball is
          # unreliable -- the comparison has to happen here, not in the reader's head.
          python3 .github/scripts/diagnose_delegate_token_claims.py "$payload"
          echo "A 'NO MATCH' above means the secret points at a different org/tenant than"
          echo "the one this bot user's token actually belongs to -- that mismatch, not this"
          echo "agent's code, is the root cause of a downstream 'Account id does not match' 401."

      - name: Run fizzbuzz delegate task (assert PASS)
        if: steps.gate.outputs.present == 'true'
        env:
          DELEGATE_SDK_NODE_MODULES: ${{ github.workspace }}/src/coder_eval/agents/delegate
          DELEGATE_ENV: ${{ github.event.inputs.delegate_uipath_env || 'alpha' }}
          AUTH_TOKEN: ${{ steps.ropc.outputs.access_token }}
          TENANT_ID: ${{ secrets.DELEGATE_TENANT_ID }}
          ORG_ID: ${{ secrets.DELEGATE_ORG_ID }}
          # Forwarded into `auth.organizationName` / `auth.tenantName` by
          # DelegateAgent._build_init_options (see .claude/notes/agents.md §
          # Delegate agent) — needed alongside the GUIDs above whenever
          # `environment` (rather than `backendUrl`) resolves the backend.
          ORG_SLUG: ${{ secrets.DELEGATE_ORG_SLUG }}
          TENANT_SLUG: ${{ secrets.DELEGATE_TENANT_SLUG }}
        run: .venv/bin/coder-eval run tasks/delegate/fizzbuzz_delegate.yaml --run-dir runs/delegate-live

      - name: Verify Delegate live run PASSED
        if: steps.gate.outputs.present == 'true'
        run: |
          python3 - <<'PY'
          import json
          from pathlib import Path

          results = list(Path("runs/delegate-live").rglob("task.json"))
          assert results, "no task.json written"
          for path in results:
              data = json.loads(path.read_text())
              assert data.get("final_status") == "SUCCESS", (
                  f"{path}: expected SUCCESS, got final_status={data.get('final_status')}"
              )
              print(f"Live Delegate run PASSED ({path}):", data.get("final_status"))
          PY

      - name: Run Delegate live unit tests (assert PASS)
        if: steps.gate.outputs.present == 'true'
        env:
          DELEGATE_SDK_NODE_MODULES: ${{ github.workspace }}/src/coder_eval/agents/delegate
          DELEGATE_ENV: ${{ github.event.inputs.delegate_uipath_env || 'alpha' }}
          AUTH_TOKEN: ${{ steps.ropc.outputs.access_token }}
          TENANT_ID: ${{ secrets.DELEGATE_TENANT_ID }}
          ORG_ID: ${{ secrets.DELEGATE_ORG_ID }}
          ORG_SLUG: ${{ secrets.DELEGATE_ORG_SLUG }}
          TENANT_SLUG: ${{ secrets.DELEGATE_TENANT_SLUG }}
        run: |
          mkdir -p tmp
          .venv/bin/pytest tests/test_delegate_agent_live.py \
            -m live -n0 -v --tb=short --strict-markers -ra \
            --junit-xml=tmp/junit-delegate-live.xml

      - name: Assert Delegate live tests actually ran (not silently skipped)
        if: steps.gate.outputs.present == 'true'
        run: |
          python3 - <<'PY'
          import sys, xml.etree.ElementTree as ET
          root = ET.parse("tmp/junit-delegate-live.xml").getroot()
          suites = [root] if root.tag == "testsuite" else root.findall("testsuite")
          total = sum(int(s.get("tests", 0)) for s in suites)
          skipped = sum(int(s.get("skipped", 0)) for s in suites)
          errors = sum(int(s.get("errors", 0)) for s in suites)
          failures = sum(int(s.get("failures", 0)) for s in suites)
          passed = total - skipped - errors - failures
          print(f"delegate-live passed={passed} skipped={skipped} errors={errors} failures={failures}")
          if passed < 1:
              sys.exit("Live Delegate tests reported zero PASSED tests (missing prerequisite / silent skip?)")
          PY

      - name: Upload Delegate live-test artifacts on failure
        if: failure() && steps.gate.outputs.present == 'true'
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7.0.1
        with:
          name: delegate-live-output
          path: runs/delegate-live/
          retention-days: 7