diff --git a/README.md b/README.md index 6dcdc22..e1a8344 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ [![Python 3.9+](https://img.shields.io/badge/python-3.9+-blue.svg)]() [![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](LICENSE) -**Testing and evaluation framework for AI agents.** Define test suites in YAML, grade agent outputs with 10 pluggable graders, track results over time, and detect regressions with statistical comparison. +**Testing and evaluation framework for AI agents.** Define test suites in YAML, grade agent outputs with 11 pluggable graders, track results over time, and detect regressions with statistical comparison. --- @@ -15,7 +15,7 @@ AI agents are **hard to test**. They're non-deterministic, they call tools, and - 🎯 **YAML-based test suites** — Define inputs, expected outputs, and grading criteria declaratively - 📊 **Statistical regression detection** — Welch's t-test across multiple runs, not just pass/fail -- 🔌 **10 built-in graders** — Exact match, contains, regex, tool-check, LLM-judge, custom, JSON-schema, semantic, latency, and cost +- 🔌 **11 built-in graders** — Exact match, contains, regex, tool-check, trajectory, LLM-judge, custom, JSON-schema, semantic, latency, and cost - 🔗 **AgentLens integration** — Import production sessions as test cases, and emit eval runs back as tamper-evident audit evidence - 💰 **Cost & latency tracking** — Know what each eval costs in tokens and dollars - 🗄️ **SQLite result storage** — Every run is persisted for historical comparison @@ -94,7 +94,7 @@ Cost: $0.0023 Avg latency: 220ms ## Features -### 🎯 10 Built-in Graders +### 🎯 11 Built-in Graders | Grader | What it checks | Expected / config fields | |--------|---------------|----------------| @@ -102,6 +102,7 @@ Cost: $0.0023 Avg latency: 220ms | `contains` | Substring presence | `output_contains: [list]` | | `regex` | Pattern matching | `pattern` | | `tool-check` | Tools were called | `tools_called: [list]` | +| `trajectory` | Agent tool-call sequence matches an expected ordered path (LCS-scored) | `grader_config: {expected: [tool names], allow_extra: bool, max_steps: N}` | | `llm-judge` | LLM evaluates quality | `criteria` (free-form) | | `custom` | Your own function | `grader_config: {function: "mod:fn"}` | | `json_schema` | Output validates against a JSON Schema | `grader_config: {schema: {...}}` or `{schema_file: path}` | diff --git a/action.yml b/action.yml index 832413e..2efd998 100644 --- a/action.yml +++ b/action.yml @@ -59,9 +59,9 @@ runs: VERSION: ${{ inputs.version }} run: | if [ -n "$VERSION" ]; then - pip install "agenteval==$VERSION" + pip install "agentevalkit==$VERSION" else - pip install agenteval + pip install agentevalkit fi - name: Run AgentEval CI gate shell: bash diff --git a/examples/docker/docker-compose.yml b/examples/docker/docker-compose.yml index 3b0443e..e079e95 100644 --- a/examples/docker/docker-compose.yml +++ b/examples/docker/docker-compose.yml @@ -9,7 +9,6 @@ services: depends_on: - redis environment: - - AGENTEVAL_REDIS_URL=redis://redis:6379/0 - OPENAI_API_KEY=${OPENAI_API_KEY:-} volumes: - .:/work @@ -20,8 +19,7 @@ services: depends_on: - redis environment: - - AGENTEVAL_REDIS_URL=redis://redis:6379/0 - OPENAI_API_KEY=${OPENAI_API_KEY:-} volumes: - .:/work - command: ["agenteval", "worker", "--redis-url", "redis://redis:6379/0"] + command: ["agenteval", "worker", "--broker", "redis://redis:6379/0"] diff --git a/examples/github-actions/with-comparison.yml b/examples/github-actions/with-comparison.yml index 9b0202b..2dcf24c 100644 --- a/examples/github-actions/with-comparison.yml +++ b/examples/github-actions/with-comparison.yml @@ -22,10 +22,19 @@ jobs: run: pip install agentevalkit - name: Run evaluation suite - run: agenteval run --suite suite.yaml --agent my_agent:run --format json -o results.json + # --report writes the JSON report; the run is also stored in the SQLite DB + # (agenteval.db) under a generated run ID used by `agenteval compare`. + run: agenteval run --suite suite.yaml --agent my_agent:run --report results.json - name: Compare with baseline - run: agenteval compare --baseline baseline.json --current results.json --format markdown -o comparison.md + # compare takes positional run IDs stored in the SQLite DB — not JSON files. + # BASELINE_RUN_ID is a previously stored baseline run; the current run is the + # most recent one in the DB. Its stdout is captured for the PR comment. + env: + BASELINE_RUN_ID: ${{ vars.BASELINE_RUN_ID }} + run: | + CURRENT_RUN_ID=$(agenteval list --limit 1 | grep -Eo '^[0-9a-f]{12}' | head -1) + agenteval compare "$BASELINE_RUN_ID" "$CURRENT_RUN_ID" | tee comparison.txt - name: Post PR comment if: github.event_name == 'pull_request' @@ -33,10 +42,10 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | gh pr comment ${{ github.event.number }} \ - --body-file comparison.md \ + --body-file comparison.txt \ --edit-last || \ gh pr comment ${{ github.event.number }} \ - --body-file comparison.md + --body-file comparison.txt - name: Upload artifacts if: always() @@ -45,4 +54,4 @@ jobs: name: eval-artifacts path: | results.json - comparison.md + comparison.txt diff --git a/examples/github-actions/with-gates.yml b/examples/github-actions/with-gates.yml index a366149..f098724 100644 --- a/examples/github-actions/with-gates.yml +++ b/examples/github-actions/with-gates.yml @@ -23,40 +23,37 @@ jobs: run: pip install agentevalkit - name: Run evaluation suite - run: agenteval run --suite suite.yaml --agent my_agent:run --format json -o results.json + # --report writes the JSON report; the run is also stored in the SQLite DB + # (agenteval.db) under a generated run ID used by `agenteval compare`. + run: agenteval run --suite suite.yaml --agent my_agent:run --report results.json + + - name: Write gate policy + # `compare --gate` takes a single gate-policy YAML file. The valid keys come + # from GatePolicy: min_pass_rate, score_threshold, max_regressions, + # max_cost_increase_pct, max_latency_increase_pct. (There is no absolute + # max_latency_ms / avg_score gate.) + run: | + cat > gates.yaml <<'EOF' + min_pass_rate: 0.95 + score_threshold: 0.8 + max_regressions: 0 + max_latency_increase_pct: 10 + EOF - name: Compare with gates - run: | - agenteval compare \ - --baseline baseline.json \ - --current results.json \ - --gate pass_rate:0.95 \ - --gate avg_score:0.8 \ - --gate max_latency_ms:5000 \ - --format json -o gate-results.json - - - name: Check gate status - run: | - python3 -c " - import json, sys - data = json.load(open('gate-results.json')) - if not data.get('gates_passed', False): - for g in data.get('failures', []): - print(f\"GATE FAILED: {g['gate']} — got {g['actual']}, required {g['threshold']}\") - sys.exit(1) - print('All quality gates passed.') - " - - - name: Post PR comment - if: always() && github.event_name == 'pull_request' + # compare takes positional run IDs stored in the SQLite DB — not JSON files. + # The current run must be the SECOND id: the gate applies to the last target. + # compare writes no JSON; it signals pass/fail via its EXIT CODE (non-zero + # fails the job when a gate is violated). env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + BASELINE_RUN_ID: ${{ vars.BASELINE_RUN_ID }} run: | - STATUS=$(python3 -c "import json; d=json.load(open('gate-results.json')); print('passed' if d.get('gates_passed') else 'FAILED')") - gh pr comment ${{ github.event.number }} \ - --body "## AgentEval Gate Results: ${STATUS} - $(cat gate-results.json | python3 -m json.tool)" \ - --edit-last || \ - gh pr comment ${{ github.event.number }} \ - --body "## AgentEval Gate Results: ${STATUS} - $(cat gate-results.json | python3 -m json.tool)" + CURRENT_RUN_ID=$(agenteval list --limit 1 | grep -Eo '^[0-9a-f]{12}' | head -1) + agenteval compare "$BASELINE_RUN_ID" "$CURRENT_RUN_ID" --gate gates.yaml + + - name: Upload artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: eval-artifacts + path: results.json