From fb5af8621bb1fc600b65f987770f630343cdeade Mon Sep 17 00:00:00 2001 From: Agent IX Date: Sat, 27 Jun 2026 11:57:50 -0700 Subject: [PATCH] Convert evals to cli-evals --- Makefile | 6 +- README.md | 48 +++++++++- cli-agent-evals.config.mjs | 54 +++++++++++ evals/run.mjs | 178 ++----------------------------------- 4 files changed, 107 insertions(+), 179 deletions(-) create mode 100644 cli-agent-evals.config.mjs mode change 100644 => 100755 evals/run.mjs diff --git a/Makefile b/Makefile index afd49ff..30f85c9 100644 --- a/Makefile +++ b/Makefile @@ -29,11 +29,13 @@ REPEATS ?= 1 .PHONY: evals evals: - node evals/run.mjs --canary --model $(MODEL) --repeats $(REPEATS) + pnpm --dir ../cli-agent-evals run build + node ../cli-agent-evals/bin/cli-evals.js run --suite ./cli-agent-evals.config.mjs --canary --agent claude --model $(MODEL) --repeats $(REPEATS) .PHONY: evals-all evals-all: - node evals/run.mjs --all --model $(MODEL) --repeats $(REPEATS) + pnpm --dir ../cli-agent-evals run build + node ../cli-agent-evals/bin/cli-evals.js run --suite ./cli-agent-evals.config.mjs --all --agent claude --model $(MODEL) --repeats $(REPEATS) # Community install smoke test (clean-room Docker: public npm + agent plugins). # Verifies an outside developer can `npm i -g @agent-ix/quoin` and install the diff --git a/README.md b/README.md index 2b72de2..b95c24d 100644 --- a/README.md +++ b/README.md @@ -212,10 +212,50 @@ The technical specification for this library was itself authored with the `spec- ### Evals -Eval scenarios live in [spec/evals.md](spec/evals.md) and run via the agent-pty-driven -harness in `evals/` (`make evals` / `make evals-all`). The harness profiles a **real** agent -running the skills/workflows and records token/tool/latency metrics from the Claude Code -transcript. Unit tests cover the mechanical CLI behavior; see [evals/README.md](evals/README.md). +Eval scenarios live in [spec/evals.md](spec/evals.md) and run through the shared +[`@agent-ix/cli-agent-evals`](../cli-agent-evals) toolkit. The suite definition is +[`cli-agent-evals.config.mjs`](cli-agent-evals.config.mjs); quoin-specific fixtures, +module seeding, prompts, and assertions remain under [`evals/`](evals/). + +Live runs profile a **real** agent running the skills/workflows and record token, +tool, and latency metrics from the available transcript. Unit tests cover the +mechanical CLI behavior. + +```bash +make evals +make evals-all +node ../cli-agent-evals/bin/cli-evals.js run \ + --suite ./cli-agent-evals.config.mjs \ + --canary \ + --agent claude \ + --model sonnet +``` + +Agent plugin setup for authoring/running evals from an agent: + +```bash +claude plugin marketplace add agent-ix/cli-agent-evals +claude plugin install cli-agent-evals + +codex plugin marketplace add agent-ix/cli-agent-evals +codex plugin add cli-agent-evals + +gh skill install agent-ix/cli-agent-evals --all --scope user --agent opencode +gh skill install agent-ix/cli-agent-evals --all --scope user --agent github-copilot +``` + +Minimal integration pattern: + +```js +import { defineSuite } from "../cli-agent-evals/dist/index.js"; +import { SCENARIOS } from "./evals/scenarios/index.mjs"; + +export default defineSuite({ + name: "quoin", + rootDir: import.meta.dirname, + scenarios: SCENARIOS, +}); +``` ## About diff --git a/cli-agent-evals.config.mjs b/cli-agent-evals.config.mjs new file mode 100644 index 0000000..a39387a --- /dev/null +++ b/cli-agent-evals.config.mjs @@ -0,0 +1,54 @@ +import { defineSuite } from "../cli-agent-evals/dist/index.js"; +import { makeScenarioWorkspace, transcriptPathFor } from "./evals/lib/env.mjs"; +import { buildTaskBrief, kickoffLine } from "./evals/lib/prompt.mjs"; +import { assertExpectations } from "./evals/lib/assert.mjs"; +import { extractMetrics } from "./evals/lib/metrics.mjs"; +import { buildSeedOnce } from "./evals/lib/seed.mjs"; +import { findQuire, shimDir } from "./evals/lib/resolve.mjs"; +import { SCENARIOS } from "./evals/scenarios/index.mjs"; + +let seed; + +export default defineSuite({ + name: "quoin", + rootDir: import.meta.dirname, + scenarios: SCENARIOS.map((scenario) => ({ + ...scenario, + canary: scenario.canary ?? ["EV-001", "EV-008"].includes(scenario.id), + })), + workspace(scenario, suite) { + seed ??= buildSeedOnce({ + force: false, + log: (message) => console.log(message), + }); + const ctx = makeScenarioWorkspace(scenario.id); + ctx.workDir = ctx.work; + ctx.reportDir = `${suite.rootDir}/evals/reports`; + return ctx; + }, + buildTaskBrief, + kickoffLine, + shimPath: () => shimDir(), + extraEnv(ctx) { + return { + IX_HOME: ctx.ixHome, + IX_SCHEMA_PATH: ctx.modulesDir, + }; + }, + agents: { + claude: { + parseMetrics: extractMetrics, + transcriptPath(ctx) { + return transcriptPathFor(ctx.cwd, ctx.sessionId); + }, + }, + }, + assert(ctx, scenario, run) { + const extraEnv = scenario.env ? scenario.env(ctx) : {}; + return assertExpectations(ctx, scenario.expect ?? {}, run, { + quireBin: findQuire(), + modulesDir: ctx.modulesDir, + extraEnv, + }); + }, +}); diff --git a/evals/run.mjs b/evals/run.mjs old mode 100644 new mode 100755 index 5ed6e0a..9a5daab --- a/evals/run.mjs +++ b/evals/run.mjs @@ -1,174 +1,6 @@ #!/usr/bin/env node -// quoin agent-pty eval harness. -// -// Drives the REAL `claude` agent (via agent-pty) through the spec-authoring CLI + -// skills for each selected scenario, captures REAL metrics from the Claude Code -// session transcript, asserts success (files + `quire validate`), and writes -// evals/reports/latest.json. -// -// Usage: -// node evals/run.mjs --canary --model -// node evals/run.mjs --all --model [--repeats N] -// node evals/run.mjs --filter EV-008 --model [--keep] [--reseed] -// -// A scenario selector (--canary | --all | --filter) is REQUIRED so a full, -// token-costly run is never triggered by accident. --model is REQUIRED so token -// counts are comparable across runs. - -import { - ixSpecBin, - findQuire, - findAgentPty, - quireVersion, - shimDir, -} from "./lib/resolve.mjs"; -import { buildSeedOnce } from "./lib/seed.mjs"; -import { makeScenarioWorkspace, transcriptPathFor } from "./lib/env.mjs"; -import { runAgent } from "./lib/agent.mjs"; -import { extractMetrics } from "./lib/metrics.mjs"; -import { assertExpectations } from "./lib/assert.mjs"; -import { - buildResult, - buildReport, - writeLatest, - printSummaryTable, - reportsDir, - rebuildFromTranscripts, -} from "./lib/report.mjs"; -import { selectScenarios } from "./scenarios/index.mjs"; -import { readFileSync, existsSync } from "node:fs"; -import { join } from "node:path"; - -function arg(name) { - const i = process.argv.indexOf(name); - return i === -1 ? undefined : process.argv[i + 1]; -} -function flag(name) { - return process.argv.includes(name); -} - -async function main() { - // --rebuild: re-derive metrics + the summary table from the prior run's - // transcripts (no agent runs). Useful after a metrics-code change. - if (flag("--rebuild")) { - const path = join(reportsDir(), "latest.json"); - if (!existsSync(path)) throw new Error(`no prior report at ${path}`); - const prior = JSON.parse(readFileSync(path, "utf8")); - const report = rebuildFromTranscripts(prior, extractMetrics, { - generatedAt: prior.generatedAt, - }); - writeLatest(report); - printSummaryTable(report); - console.log(`\nrebuilt: ${path}`); - return; - } - - const model = arg("--model"); - const repeats = Number(arg("--repeats") ?? "1"); - const keep = flag("--keep"); - const selection = { - canary: flag("--canary"), - all: flag("--all"), - filter: arg("--filter"), - }; - - if (!model) - throw new Error( - "--model is required (token counts must be comparable)", - ); - const scenarios = selectScenarios(selection); - if (scenarios.length === 0) { - throw new Error( - "no scenarios selected. Pass --canary, --all, or --filter EV-00X.", - ); - } - - const seed = buildSeedOnce({ - force: flag("--reseed"), - log: (m) => console.log(m), - }); - const shimPath = shimDir(); - const quireBin = findQuire(); - const { entry: agentPtyPath } = await findAgentPty(); - - console.log( - `running ${scenarios.length} scenario(s) x${repeats} | model=${model} | quire=${quireVersion(quireBin)}`, - ); - - const results = []; - for (const scenario of scenarios) { - const runs = []; - for (let i = 0; i < repeats; i++) { - const label = - repeats > 1 ? `${scenario.id} (${i + 1}/${repeats})` : scenario.id; - process.stdout.write(`▶ ${label} ... `); - const ctx = makeScenarioWorkspace(scenario.id); - try { - scenario.setup?.(ctx); - // setup may have repointed ctx.cwd (e.g. multi-repo) — finalize the path. - ctx.transcriptPath = transcriptPathFor(ctx.cwd, ctx.sessionId); - const extraEnv = scenario.env ? scenario.env(ctx) : {}; - const runResult = await runAgent(scenario, ctx, { - model, - modulesDir: ctx.modulesDir, - shimPath, - extraEnv, - }); - const metrics = extractMetrics(ctx.transcriptPath); - const assertion = assertExpectations(ctx, scenario.expect, runResult, { - quireBin, - modulesDir: ctx.modulesDir, - extraEnv, - }); - const ok = assertion.ok; - runs.push({ - ok, - runResult, - metrics, - assertion, - workdir: ctx.work, - sessionId: ctx.sessionId, - transcriptPath: ctx.transcriptPath, - }); - console.log( - `${ok ? "PASS" : "FAIL"} ${(runResult.wallMs / 1000).toFixed(0)}s ` + - `[${runResult.exitReason}] tokens=${metrics.tokenUsage.total} tools=${metrics.toolCalls}` + - (ok ? "" : `\n ${assertion.failures.join("; ")}`), - ); - if (!ok && runResult.exitReason === "timeout") { - console.log(` screen tail:\n${indent(runResult.screenTail)}`); - } - } finally { - if (!keep) ctx.cleanup(); - } - } - results.push(buildResult(scenario, runs)); - } - - const report = buildReport(results, { - generatedAt: new Date().toISOString(), - model, - repeats, - quireBin, - quireVersion: quireVersion(quireBin), - ixSpecBin: ixSpecBin(), - agentPtyPath, - seedDir: seed.seedDir, - }); - const path = writeLatest(report); - printSummaryTable(report); - console.log(`\nreport: ${path}`); - process.exit(report.ok ? 0 : 1); -} - -function indent(text, pad = " ") { - return (text ?? "") - .split("\n") - .map((l) => pad + l) - .join("\n"); -} - -main().catch((err) => { - console.error(err instanceof Error ? err.stack : String(err)); - process.exit(1); -}); +console.error( + "quoin evals now run through @agent-ix/cli-agent-evals.\n" + + "Use: make evals, make evals-all, or node ../cli-agent-evals/bin/cli-evals.js run --suite ./cli-agent-evals.config.mjs ...", +); +process.exit(2);