feat(action): GitHub Marketplace Action — composite action.yml (#14)

README.md

@@ -275,6 +275,27 @@ agenteval import --from agentlens --db sessions.db --output suite.yaml [--grader
 
 ---
 
+## GitHub Action
+
+Gate your CI on the eval suite — the action (root `action.yml`) installs AgentEval
+and runs `agenteval ci`, failing the build when the pass rate or regression
+breaches your thresholds:
+
+```yaml
+- uses: agentkitai/agenteval@v1
+  with:
+    suite: suite.yaml
+    agent: my_agent:run
+    min-pass-rate: "0.9"
+    max-regression: "5"
+```
+
+Inputs mirror `agenteval ci` (`baseline`, `format`, `output`, `parallel`,
+`version`, `python-version`). A prebuilt `ghcr.io` image for faster, hermetic
+runs is a planned follow-up.
+
+---
+
 ## Grader Reference
 
 ### `exact`

action.yml

@@ -0,0 +1,85 @@
+name: "AgentEval"
+description: "Run an AgentEval suite as a CI gate — fail the build when the agent pass rate or regression breaches your thresholds."
+author: "agentkitai"
+branding:
+  icon: "check-circle"
+  color: "purple"
+
+inputs:
+  suite:
+    description: "Path to the eval suite file (YAML/JSON)."
+    required: true
+  agent:
+    description: "Agent callable as 'module:func'."
+    required: true
+  min-pass-rate:
+    description: "Minimum pass rate (0–1) to pass the gate."
+    required: false
+    default: "0.8"
+  max-regression:
+    description: "Maximum allowed regression percentage vs the baseline."
+    required: false
+    default: "10"
+  baseline:
+    description: "Baseline run ID for regression detection (optional)."
+    required: false
+    default: ""
+  format:
+    description: "Output format: text | json | junit."
+    required: false
+    default: "text"
+  output:
+    description: "Write the report to this file (optional)."
+    required: false
+    default: ""
+  parallel:
+    description: "Max concurrent cases."
+    required: false
+    default: "1"
+  version:
+    description: "agenteval version to install from PyPI (e.g. '1.2.3'); empty = latest."
+    required: false
+    default: ""
+  python-version:
+    description: "Python version to set up."
+    required: false
+    default: "3.12"
+
+runs:
+  using: "composite"
+  steps:
+    - uses: actions/setup-python@v5
+      with:
+        python-version: ${{ inputs.python-version }}
+    # Inputs are passed via env (NOT interpolated into the script body) to avoid
+    # GitHub Actions shell-injection; every expansion is quoted.
+    - name: Install agenteval
+      shell: bash
+      env:
+        VERSION: ${{ inputs.version }}
+      run: |
+        if [ -n "$VERSION" ]; then
+          pip install "agenteval==$VERSION"
+        else
+          pip install agenteval
+        fi
+    - name: Run AgentEval CI gate
+      shell: bash
+      env:
+        SUITE: ${{ inputs.suite }}
+        AGENT: ${{ inputs.agent }}
+        MIN_PASS_RATE: ${{ inputs.min-pass-rate }}
+        MAX_REGRESSION: ${{ inputs.max-regression }}
+        BASELINE: ${{ inputs.baseline }}
+        FORMAT: ${{ inputs.format }}
+        OUTPUT: ${{ inputs.output }}
+        PARALLEL: ${{ inputs.parallel }}
+      run: |
+        args=(ci "$SUITE" --agent "$AGENT" \
+          --min-pass-rate "$MIN_PASS_RATE" \
+          --max-regression "$MAX_REGRESSION" \
+          --format "$FORMAT" \
+          --parallel "$PARALLEL")
+        [ -n "$BASELINE" ] && args+=(--baseline "$BASELINE")
+        [ -n "$OUTPUT" ] && args+=(--output "$OUTPUT")
+        agenteval "${args[@]}"

tests/test_action_yml.py

@@ -0,0 +1,42 @@
+"""Validate the GitHub Marketplace action.yml (#14)."""
+
+from pathlib import Path
+
+import yaml
+
+ACTION = Path(__file__).resolve().parent.parent / "action.yml"
+
+
+def _load():
+    with open(ACTION, encoding="utf-8") as f:
+        return yaml.safe_load(f)
+
+
+def test_action_yml_is_valid_marketplace_action():
+    a = _load()
+    assert a["name"] and a["description"]
+    assert "branding" in a  # required for Marketplace listing
+    assert a["runs"]["using"] == "composite"
+
+
+def test_required_inputs_match_the_ci_command():
+    a = _load()
+    inputs = a["inputs"]
+    assert inputs["suite"]["required"] is True
+    assert inputs["agent"]["required"] is True
+    # optional inputs carry defaults
+    assert inputs["min-pass-rate"]["default"] == "0.8"
+    assert inputs["max-regression"]["default"] == "10"
+
+
+def test_steps_invoke_agenteval_ci_via_env_not_injection():
+    a = _load()
+    steps = a["runs"]["steps"]
+    run_step = next(s for s in steps if "AgentEval CI gate" in s.get("name", ""))
+    body = run_step["run"]
+    # entrypoint is the ci command
+    assert "agenteval " in body and "ci " in body
+    # hardened: inputs come from env vars, NOT `${{ inputs.* }}` in the script body
+    assert "${{" not in body
+    assert "$SUITE" in body and "$AGENT" in body
+    assert set(run_step["env"]) >= {"SUITE", "AGENT", "MIN_PASS_RATE", "MAX_REGRESSION"}