docs: fix staleness across agenteval docs

README.md

@@ -5,7 +5,7 @@
 [![Python 3.9+](https://img.shields.io/badge/python-3.9+-blue.svg)]()
 [![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](LICENSE)
 
-**Testing and evaluation framework for AI agents.** Define test suites in YAML, grade agent outputs with 10 pluggable graders, track results over time, and detect regressions with statistical comparison.
+**Testing and evaluation framework for AI agents.** Define test suites in YAML, grade agent outputs with 11 pluggable graders, track results over time, and detect regressions with statistical comparison.
 
 ---
 
@@ -15,7 +15,7 @@ AI agents are **hard to test**. They're non-deterministic, they call tools, and
 
 - 🎯 **YAML-based test suites** — Define inputs, expected outputs, and grading criteria declaratively
 - 📊 **Statistical regression detection** — Welch's t-test across multiple runs, not just pass/fail
-- 🔌 **10 built-in graders** — Exact match, contains, regex, tool-check, LLM-judge, custom, JSON-schema, semantic, latency, and cost
+- 🔌 **11 built-in graders** — Exact match, contains, regex, tool-check, trajectory, LLM-judge, custom, JSON-schema, semantic, latency, and cost
 - 🔗 **AgentLens integration** — Import production sessions as test cases, and emit eval runs back as tamper-evident audit evidence
 - 💰 **Cost & latency tracking** — Know what each eval costs in tokens and dollars
 - 🗄️ **SQLite result storage** — Every run is persisted for historical comparison
@@ -94,14 +94,15 @@ Cost: $0.0023  Avg latency: 220ms
 
 ## Features
 
-### 🎯 10 Built-in Graders
+### 🎯 11 Built-in Graders
 
 | Grader | What it checks | Expected / config fields |
 |--------|---------------|----------------|
 | `exact` | Exact string match | `output` |
 | `contains` | Substring presence | `output_contains: [list]` |
 | `regex` | Pattern matching | `pattern` |
 | `tool-check` | Tools were called | `tools_called: [list]` |
+| `trajectory` | Agent tool-call sequence matches an expected ordered path (LCS-scored) | `grader_config: {expected: [tool names], allow_extra: bool, max_steps: N}` |
 | `llm-judge` | LLM evaluates quality | `criteria` (free-form) |
 | `custom` | Your own function | `grader_config: {function: "mod:fn"}` |
 | `json_schema` | Output validates against a JSON Schema | `grader_config: {schema: {...}}` or `{schema_file: path}` |

action.yml

@@ -59,9 +59,9 @@ runs:
         VERSION: ${{ inputs.version }}
       run: |
         if [ -n "$VERSION" ]; then
-          pip install "agenteval==$VERSION"
+          pip install "agentevalkit==$VERSION"
         else
-          pip install agenteval
+          pip install agentevalkit
         fi
     - name: Run AgentEval CI gate
       shell: bash

examples/docker/docker-compose.yml

@@ -9,7 +9,6 @@ services:
     depends_on:
       - redis
     environment:
-      - AGENTEVAL_REDIS_URL=redis://redis:6379/0
       - OPENAI_API_KEY=${OPENAI_API_KEY:-}
     volumes:
       - .:/work
@@ -20,8 +19,7 @@ services:
     depends_on:
       - redis
     environment:
-      - AGENTEVAL_REDIS_URL=redis://redis:6379/0
       - OPENAI_API_KEY=${OPENAI_API_KEY:-}
     volumes:
       - .:/work
-    command: ["agenteval", "worker", "--redis-url", "redis://redis:6379/0"]
+    command: ["agenteval", "worker", "--broker", "redis://redis:6379/0"]

examples/github-actions/with-comparison.yml

@@ -22,21 +22,30 @@ jobs:
         run: pip install agentevalkit
 
       - name: Run evaluation suite
-        run: agenteval run --suite suite.yaml --agent my_agent:run --format json -o results.json
+        # --report writes the JSON report; the run is also stored in the SQLite DB
+        # (agenteval.db) under a generated run ID used by `agenteval compare`.
+        run: agenteval run --suite suite.yaml --agent my_agent:run --report results.json
 
       - name: Compare with baseline
-        run: agenteval compare --baseline baseline.json --current results.json --format markdown -o comparison.md
+        # compare takes positional run IDs stored in the SQLite DB — not JSON files.
+        # BASELINE_RUN_ID is a previously stored baseline run; the current run is the
+        # most recent one in the DB. Its stdout is captured for the PR comment.
+        env:
+          BASELINE_RUN_ID: ${{ vars.BASELINE_RUN_ID }}
+        run: |
+          CURRENT_RUN_ID=$(agenteval list --limit 1 | grep -Eo '^[0-9a-f]{12}' | head -1)
+          agenteval compare "$BASELINE_RUN_ID" "$CURRENT_RUN_ID" | tee comparison.txt
 
       - name: Post PR comment
         if: github.event_name == 'pull_request'
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
           gh pr comment ${{ github.event.number }} \
-            --body-file comparison.md \
+            --body-file comparison.txt \
             --edit-last || \
           gh pr comment ${{ github.event.number }} \
-            --body-file comparison.md
+            --body-file comparison.txt
 
       - name: Upload artifacts
         if: always()
@@ -45,4 +54,4 @@ jobs:
           name: eval-artifacts
           path: |
             results.json
-            comparison.md
+            comparison.txt

examples/github-actions/with-gates.yml

@@ -23,40 +23,37 @@ jobs:
         run: pip install agentevalkit
 
       - name: Run evaluation suite
-        run: agenteval run --suite suite.yaml --agent my_agent:run --format json -o results.json
+        # --report writes the JSON report; the run is also stored in the SQLite DB
+        # (agenteval.db) under a generated run ID used by `agenteval compare`.
+        run: agenteval run --suite suite.yaml --agent my_agent:run --report results.json
+
+      - name: Write gate policy
+        # `compare --gate` takes a single gate-policy YAML file. The valid keys come
+        # from GatePolicy: min_pass_rate, score_threshold, max_regressions,
+        # max_cost_increase_pct, max_latency_increase_pct. (There is no absolute
+        # max_latency_ms / avg_score gate.)
+        run: |
+          cat > gates.yaml <<'EOF'
+          min_pass_rate: 0.95
+          score_threshold: 0.8
+          max_regressions: 0
+          max_latency_increase_pct: 10
+          EOF
 
       - name: Compare with gates
-        run: |
-          agenteval compare \
-            --baseline baseline.json \
-            --current results.json \
-            --gate pass_rate:0.95 \
-            --gate avg_score:0.8 \
-            --gate max_latency_ms:5000 \
-            --format json -o gate-results.json
-
-      - name: Check gate status
-        run: |
-          python3 -c "
-          import json, sys
-          data = json.load(open('gate-results.json'))
-          if not data.get('gates_passed', False):
-              for g in data.get('failures', []):
-                  print(f\"GATE FAILED: {g['gate']} — got {g['actual']}, required {g['threshold']}\")
-              sys.exit(1)
-          print('All quality gates passed.')
-          "
-
-      - name: Post PR comment
-        if: always() && github.event_name == 'pull_request'
+        # compare takes positional run IDs stored in the SQLite DB — not JSON files.
+        # The current run must be the SECOND id: the gate applies to the last target.
+        # compare writes no JSON; it signals pass/fail via its EXIT CODE (non-zero
+        # fails the job when a gate is violated).
         env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          BASELINE_RUN_ID: ${{ vars.BASELINE_RUN_ID }}
         run: |
-          STATUS=$(python3 -c "import json; d=json.load(open('gate-results.json')); print('passed' if d.get('gates_passed') else 'FAILED')")
-          gh pr comment ${{ github.event.number }} \
-            --body "## AgentEval Gate Results: ${STATUS}
-          $(cat gate-results.json | python3 -m json.tool)" \
-            --edit-last || \
-          gh pr comment ${{ github.event.number }} \
-            --body "## AgentEval Gate Results: ${STATUS}
-          $(cat gate-results.json | python3 -m json.tool)"
+          CURRENT_RUN_ID=$(agenteval list --limit 1 | grep -Eo '^[0-9a-f]{12}' | head -1)
+          agenteval compare "$BASELINE_RUN_ID" "$CURRENT_RUN_ID" --gate gates.yaml
+
+      - name: Upload artifacts
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: eval-artifacts
+          path: results.json