[codex] convert evals to cli-evals

Makefile

@@ -29,11 +29,13 @@ REPEATS ?= 1
 
 .PHONY: evals
 evals:
-	node evals/run.mjs --canary --model $(MODEL) --repeats $(REPEATS)
+	pnpm --dir ../cli-agent-evals run build
+	node ../cli-agent-evals/bin/cli-evals.js run --suite ./cli-agent-evals.config.mjs --canary --agent claude --model $(MODEL) --repeats $(REPEATS)
 
 .PHONY: evals-all
 evals-all:
-	node evals/run.mjs --all --model $(MODEL) --repeats $(REPEATS)
+	pnpm --dir ../cli-agent-evals run build
+	node ../cli-agent-evals/bin/cli-evals.js run --suite ./cli-agent-evals.config.mjs --all --agent claude --model $(MODEL) --repeats $(REPEATS)
 
 # Community install smoke test (clean-room Docker: public npm + agent plugins).
 # Verifies an outside developer can `npm i -g @agent-ix/quoin` and install the

README.md

@@ -212,10 +212,50 @@ The technical specification for this library was itself authored with the `spec-
 
 ### Evals
 
-Eval scenarios live in [spec/evals.md](spec/evals.md) and run via the agent-pty-driven
-harness in `evals/` (`make evals` / `make evals-all`). The harness profiles a **real** agent
-running the skills/workflows and records token/tool/latency metrics from the Claude Code
-transcript. Unit tests cover the mechanical CLI behavior; see [evals/README.md](evals/README.md).
+Eval scenarios live in [spec/evals.md](spec/evals.md) and run through the shared
+[`@agent-ix/cli-agent-evals`](../cli-agent-evals) toolkit. The suite definition is
+[`cli-agent-evals.config.mjs`](cli-agent-evals.config.mjs); quoin-specific fixtures,
+module seeding, prompts, and assertions remain under [`evals/`](evals/).
+
+Live runs profile a **real** agent running the skills/workflows and record token,
+tool, and latency metrics from the available transcript. Unit tests cover the
+mechanical CLI behavior.
+
+```bash
+make evals
+make evals-all
+node ../cli-agent-evals/bin/cli-evals.js run \
+  --suite ./cli-agent-evals.config.mjs \
+  --canary \
+  --agent claude \
+  --model sonnet
+```
+
+Agent plugin setup for authoring/running evals from an agent:
+
+```bash
+claude plugin marketplace add agent-ix/cli-agent-evals
+claude plugin install cli-agent-evals
+
+codex plugin marketplace add agent-ix/cli-agent-evals
+codex plugin add cli-agent-evals
+
+gh skill install agent-ix/cli-agent-evals --all --scope user --agent opencode
+gh skill install agent-ix/cli-agent-evals --all --scope user --agent github-copilot
+```
+
+Minimal integration pattern:
+
+```js
+import { defineSuite } from "../cli-agent-evals/dist/index.js";
+import { SCENARIOS } from "./evals/scenarios/index.mjs";
+
+export default defineSuite({
+  name: "quoin",
+  rootDir: import.meta.dirname,
+  scenarios: SCENARIOS,
+});
+```
 
 ## About
 

cli-agent-evals.config.mjs

@@ -0,0 +1,54 @@
+import { defineSuite } from "../cli-agent-evals/dist/index.js";
+import { makeScenarioWorkspace, transcriptPathFor } from "./evals/lib/env.mjs";
+import { buildTaskBrief, kickoffLine } from "./evals/lib/prompt.mjs";
+import { assertExpectations } from "./evals/lib/assert.mjs";
+import { extractMetrics } from "./evals/lib/metrics.mjs";
+import { buildSeedOnce } from "./evals/lib/seed.mjs";
+import { findQuire, shimDir } from "./evals/lib/resolve.mjs";
+import { SCENARIOS } from "./evals/scenarios/index.mjs";
+
+let seed;
+
+export default defineSuite({
+  name: "quoin",
+  rootDir: import.meta.dirname,
+  scenarios: SCENARIOS.map((scenario) => ({
+    ...scenario,
+    canary: scenario.canary ?? ["EV-001", "EV-008"].includes(scenario.id),
+  })),
+  workspace(scenario, suite) {
+    seed ??= buildSeedOnce({
+      force: false,
+      log: (message) => console.log(message),
+    });
+    const ctx = makeScenarioWorkspace(scenario.id);
+    ctx.workDir = ctx.work;
+    ctx.reportDir = `${suite.rootDir}/evals/reports`;
+    return ctx;
+  },
+  buildTaskBrief,
+  kickoffLine,
+  shimPath: () => shimDir(),
+  extraEnv(ctx) {
+    return {
+      IX_HOME: ctx.ixHome,
+      IX_SCHEMA_PATH: ctx.modulesDir,
+    };
+  },
+  agents: {
+    claude: {
+      parseMetrics: extractMetrics,
+      transcriptPath(ctx) {
+        return transcriptPathFor(ctx.cwd, ctx.sessionId);
+      },
+    },
+  },
+  assert(ctx, scenario, run) {
+    const extraEnv = scenario.env ? scenario.env(ctx) : {};
+    return assertExpectations(ctx, scenario.expect ?? {}, run, {
+      quireBin: findQuire(),
+      modulesDir: ctx.modulesDir,
+      extraEnv,
+    });
+  },
+});

evals/run.mjs

@@ -1,174 +1,6 @@
 #!/usr/bin/env node
-// quoin agent-pty eval harness.
-//
-// Drives the REAL `claude` agent (via agent-pty) through the spec-authoring CLI +
-// skills for each selected scenario, captures REAL metrics from the Claude Code
-// session transcript, asserts success (files + `quire validate`), and writes
-// evals/reports/latest.json.
-//
-// Usage:
-//   node evals/run.mjs --canary --model <id>
-//   node evals/run.mjs --all --model <id> [--repeats N]
-//   node evals/run.mjs --filter EV-008 --model <id> [--keep] [--reseed]
-//
-// A scenario selector (--canary | --all | --filter) is REQUIRED so a full,
-// token-costly run is never triggered by accident. --model is REQUIRED so token
-// counts are comparable across runs.
-
-import {
-  ixSpecBin,
-  findQuire,
-  findAgentPty,
-  quireVersion,
-  shimDir,
-} from "./lib/resolve.mjs";
-import { buildSeedOnce } from "./lib/seed.mjs";
-import { makeScenarioWorkspace, transcriptPathFor } from "./lib/env.mjs";
-import { runAgent } from "./lib/agent.mjs";
-import { extractMetrics } from "./lib/metrics.mjs";
-import { assertExpectations } from "./lib/assert.mjs";
-import {
-  buildResult,
-  buildReport,
-  writeLatest,
-  printSummaryTable,
-  reportsDir,
-  rebuildFromTranscripts,
-} from "./lib/report.mjs";
-import { selectScenarios } from "./scenarios/index.mjs";
-import { readFileSync, existsSync } from "node:fs";
-import { join } from "node:path";
-
-function arg(name) {
-  const i = process.argv.indexOf(name);
-  return i === -1 ? undefined : process.argv[i + 1];
-}
-function flag(name) {
-  return process.argv.includes(name);
-}
-
-async function main() {
-  // --rebuild: re-derive metrics + the summary table from the prior run's
-  // transcripts (no agent runs). Useful after a metrics-code change.
-  if (flag("--rebuild")) {
-    const path = join(reportsDir(), "latest.json");
-    if (!existsSync(path)) throw new Error(`no prior report at ${path}`);
-    const prior = JSON.parse(readFileSync(path, "utf8"));
-    const report = rebuildFromTranscripts(prior, extractMetrics, {
-      generatedAt: prior.generatedAt,
-    });
-    writeLatest(report);
-    printSummaryTable(report);
-    console.log(`\nrebuilt: ${path}`);
-    return;
-  }
-
-  const model = arg("--model");
-  const repeats = Number(arg("--repeats") ?? "1");
-  const keep = flag("--keep");
-  const selection = {
-    canary: flag("--canary"),
-    all: flag("--all"),
-    filter: arg("--filter"),
-  };
-
-  if (!model)
-    throw new Error(
-      "--model <id> is required (token counts must be comparable)",
-    );
-  const scenarios = selectScenarios(selection);
-  if (scenarios.length === 0) {
-    throw new Error(
-      "no scenarios selected. Pass --canary, --all, or --filter EV-00X.",
-    );
-  }
-
-  const seed = buildSeedOnce({
-    force: flag("--reseed"),
-    log: (m) => console.log(m),
-  });
-  const shimPath = shimDir();
-  const quireBin = findQuire();
-  const { entry: agentPtyPath } = await findAgentPty();
-
-  console.log(
-    `running ${scenarios.length} scenario(s) x${repeats} | model=${model} | quire=${quireVersion(quireBin)}`,
-  );
-
-  const results = [];
-  for (const scenario of scenarios) {
-    const runs = [];
-    for (let i = 0; i < repeats; i++) {
-      const label =
-        repeats > 1 ? `${scenario.id} (${i + 1}/${repeats})` : scenario.id;
-      process.stdout.write(`▶ ${label} ... `);
-      const ctx = makeScenarioWorkspace(scenario.id);
-      try {
-        scenario.setup?.(ctx);
-        // setup may have repointed ctx.cwd (e.g. multi-repo) — finalize the path.
-        ctx.transcriptPath = transcriptPathFor(ctx.cwd, ctx.sessionId);
-        const extraEnv = scenario.env ? scenario.env(ctx) : {};
-        const runResult = await runAgent(scenario, ctx, {
-          model,
-          modulesDir: ctx.modulesDir,
-          shimPath,
-          extraEnv,
-        });
-        const metrics = extractMetrics(ctx.transcriptPath);
-        const assertion = assertExpectations(ctx, scenario.expect, runResult, {
-          quireBin,
-          modulesDir: ctx.modulesDir,
-          extraEnv,
-        });
-        const ok = assertion.ok;
-        runs.push({
-          ok,
-          runResult,
-          metrics,
-          assertion,
-          workdir: ctx.work,
-          sessionId: ctx.sessionId,
-          transcriptPath: ctx.transcriptPath,
-        });
-        console.log(
-          `${ok ? "PASS" : "FAIL"} ${(runResult.wallMs / 1000).toFixed(0)}s ` +
-            `[${runResult.exitReason}] tokens=${metrics.tokenUsage.total} tools=${metrics.toolCalls}` +
-            (ok ? "" : `\n   ${assertion.failures.join("; ")}`),
-        );
-        if (!ok && runResult.exitReason === "timeout") {
-          console.log(`   screen tail:\n${indent(runResult.screenTail)}`);
-        }
-      } finally {
-        if (!keep) ctx.cleanup();
-      }
-    }
-    results.push(buildResult(scenario, runs));
-  }
-
-  const report = buildReport(results, {
-    generatedAt: new Date().toISOString(),
-    model,
-    repeats,
-    quireBin,
-    quireVersion: quireVersion(quireBin),
-    ixSpecBin: ixSpecBin(),
-    agentPtyPath,
-    seedDir: seed.seedDir,
-  });
-  const path = writeLatest(report);
-  printSummaryTable(report);
-  console.log(`\nreport: ${path}`);
-  process.exit(report.ok ? 0 : 1);
-}
-
-function indent(text, pad = "   ") {
-  return (text ?? "")
-    .split("\n")
-    .map((l) => pad + l)
-    .join("\n");
-}
-
-main().catch((err) => {
-  console.error(err instanceof Error ? err.stack : String(err));
-  process.exit(1);
-});
+console.error(
+  "quoin evals now run through @agent-ix/cli-agent-evals.\n" +
+    "Use: make evals, make evals-all, or node ../cli-agent-evals/bin/cli-evals.js run --suite ./cli-agent-evals.config.mjs ...",
+);
+process.exit(2);