From b942bc8b813648f5d0a7cc4ebd4ed3017701e9ba Mon Sep 17 00:00:00 2001 From: Amit Paz Date: Fri, 26 Jun 2026 12:14:43 +0300 Subject: [PATCH] =?UTF-8?q?feat(action):=20GitHub=20Marketplace=20Action?= =?UTF-8?q?=20=E2=80=94=20composite=20action.yml=20(#14)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root action.yml makes agenteval a first-class GitHub Action (matching promptfoo): installs agenteval from PyPI and runs `agenteval ci` as a build gate, failing when pass-rate/regression breaches thresholds. - action.yml: composite action; inputs mirror the ci command (suite, agent, min-pass-rate, max-regression, baseline, format, output, parallel, version, python-version) + Marketplace branding. Inputs are passed via env (NOT interpolated into the run body) to avoid GitHub Actions shell-injection; every expansion is quoted. - README: a GitHub Action usage section. PyPI-based (no ghcr needed) → fully functional today. A prebuilt ghcr.io image for faster/hermetic runs is an optional follow-up (owner-gated on org-namespace publish, #96) — documented in OWNER-ACTIONS, not blocking this action. Tests: test_action_yml.py (valid composite Marketplace action; required suite/agent inputs + defaults; steps invoke `agenteval ci` via env, no `${{ }}` injection in the script body). pytest 3/3, ruff clean. Closes #14. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_019tXZpN29akdmG8AEjgSZwk --- README.md | 21 ++++++++++ action.yml | 85 ++++++++++++++++++++++++++++++++++++++++ tests/test_action_yml.py | 42 ++++++++++++++++++++ 3 files changed, 148 insertions(+) create mode 100644 action.yml create mode 100644 tests/test_action_yml.py diff --git a/README.md b/README.md index b13c6a3..e4efc38 100644 --- a/README.md +++ b/README.md @@ -275,6 +275,27 @@ agenteval import --from agentlens --db sessions.db --output suite.yaml [--grader --- +## GitHub Action + +Gate your CI on the eval suite — the action (root `action.yml`) installs AgentEval +and runs `agenteval ci`, failing the build when the pass rate or regression +breaches your thresholds: + +```yaml +- uses: agentkitai/agenteval@v1 + with: + suite: suite.yaml + agent: my_agent:run + min-pass-rate: "0.9" + max-regression: "5" +``` + +Inputs mirror `agenteval ci` (`baseline`, `format`, `output`, `parallel`, +`version`, `python-version`). A prebuilt `ghcr.io` image for faster, hermetic +runs is a planned follow-up. + +--- + ## Grader Reference ### `exact` diff --git a/action.yml b/action.yml new file mode 100644 index 0000000..832413e --- /dev/null +++ b/action.yml @@ -0,0 +1,85 @@ +name: "AgentEval" +description: "Run an AgentEval suite as a CI gate — fail the build when the agent pass rate or regression breaches your thresholds." +author: "agentkitai" +branding: + icon: "check-circle" + color: "purple" + +inputs: + suite: + description: "Path to the eval suite file (YAML/JSON)." + required: true + agent: + description: "Agent callable as 'module:func'." + required: true + min-pass-rate: + description: "Minimum pass rate (0–1) to pass the gate." + required: false + default: "0.8" + max-regression: + description: "Maximum allowed regression percentage vs the baseline." + required: false + default: "10" + baseline: + description: "Baseline run ID for regression detection (optional)." + required: false + default: "" + format: + description: "Output format: text | json | junit." + required: false + default: "text" + output: + description: "Write the report to this file (optional)." + required: false + default: "" + parallel: + description: "Max concurrent cases." + required: false + default: "1" + version: + description: "agenteval version to install from PyPI (e.g. '1.2.3'); empty = latest." + required: false + default: "" + python-version: + description: "Python version to set up." + required: false + default: "3.12" + +runs: + using: "composite" + steps: + - uses: actions/setup-python@v5 + with: + python-version: ${{ inputs.python-version }} + # Inputs are passed via env (NOT interpolated into the script body) to avoid + # GitHub Actions shell-injection; every expansion is quoted. + - name: Install agenteval + shell: bash + env: + VERSION: ${{ inputs.version }} + run: | + if [ -n "$VERSION" ]; then + pip install "agenteval==$VERSION" + else + pip install agenteval + fi + - name: Run AgentEval CI gate + shell: bash + env: + SUITE: ${{ inputs.suite }} + AGENT: ${{ inputs.agent }} + MIN_PASS_RATE: ${{ inputs.min-pass-rate }} + MAX_REGRESSION: ${{ inputs.max-regression }} + BASELINE: ${{ inputs.baseline }} + FORMAT: ${{ inputs.format }} + OUTPUT: ${{ inputs.output }} + PARALLEL: ${{ inputs.parallel }} + run: | + args=(ci "$SUITE" --agent "$AGENT" \ + --min-pass-rate "$MIN_PASS_RATE" \ + --max-regression "$MAX_REGRESSION" \ + --format "$FORMAT" \ + --parallel "$PARALLEL") + [ -n "$BASELINE" ] && args+=(--baseline "$BASELINE") + [ -n "$OUTPUT" ] && args+=(--output "$OUTPUT") + agenteval "${args[@]}" diff --git a/tests/test_action_yml.py b/tests/test_action_yml.py new file mode 100644 index 0000000..75cdc3e --- /dev/null +++ b/tests/test_action_yml.py @@ -0,0 +1,42 @@ +"""Validate the GitHub Marketplace action.yml (#14).""" + +from pathlib import Path + +import yaml + +ACTION = Path(__file__).resolve().parent.parent / "action.yml" + + +def _load(): + with open(ACTION, encoding="utf-8") as f: + return yaml.safe_load(f) + + +def test_action_yml_is_valid_marketplace_action(): + a = _load() + assert a["name"] and a["description"] + assert "branding" in a # required for Marketplace listing + assert a["runs"]["using"] == "composite" + + +def test_required_inputs_match_the_ci_command(): + a = _load() + inputs = a["inputs"] + assert inputs["suite"]["required"] is True + assert inputs["agent"]["required"] is True + # optional inputs carry defaults + assert inputs["min-pass-rate"]["default"] == "0.8" + assert inputs["max-regression"]["default"] == "10" + + +def test_steps_invoke_agenteval_ci_via_env_not_injection(): + a = _load() + steps = a["runs"]["steps"] + run_step = next(s for s in steps if "AgentEval CI gate" in s.get("name", "")) + body = run_step["run"] + # entrypoint is the ci command + assert "agenteval " in body and "ci " in body + # hardened: inputs come from env vars, NOT `${{ inputs.* }}` in the script body + assert "${{" not in body + assert "$SUITE" in body and "$AGENT" in body + assert set(run_step["env"]) >= {"SUITE", "AGENT", "MIN_PASS_RATE", "MAX_REGRESSION"}