From f7feb39a468a631e55cddb76e9734471c6e76195 Mon Sep 17 00:00:00 2001 From: salimlaimeche Date: Wed, 1 Jul 2026 00:10:11 +0200 Subject: [PATCH] feat: add CLI history baseline and regression ergonomics --- README.md | 11 + docs/ALPHA_READINESS.md | 3 +- docs/BACKLOG.md | 4 +- docs/PROJECT_AUDIT.md | 4 +- packages/cli/README.md | 56 +++- packages/cli/src/index.test.ts | 311 +++++++++++++++++++++- packages/cli/src/index.ts | 454 ++++++++++++++++++++++++++++++++- 7 files changed, 823 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 00f79c5..d5bb3ac 100644 --- a/README.md +++ b/README.md @@ -108,6 +108,17 @@ Write a timestamped local report bundle: bun run --filter '@ignitionai/agent-trainer-cli' dev -- eval run ./examples/context-engineering/experiment.ts --bundle reports ``` +Record local experiment history and run a regression check against the latest baseline: + +```bash +bun run --filter '@ignitionai/agent-trainer-cli' dev -- eval run ./examples/context-engineering/experiment.ts \ + --history .ignition/experiment-history.jsonl \ + --baseline latest \ + --regression \ + --max-score-drop 0.03 \ + --record-history +``` + Run the sample CI regression gate: ```bash diff --git a/docs/ALPHA_READINESS.md b/docs/ALPHA_READINESS.md index b4e7468..03a09d1 100644 --- a/docs/ALPHA_READINESS.md +++ b/docs/ALPHA_READINESS.md @@ -27,7 +27,7 @@ All packages declare `license: MIT`, matching the root `LICENSE` file. | `@ignitionai/agent-trainer-adapter-langgraph` | ready | ready | ready | ready | partial | Structural adapter only; no persistence, streaming or graph internals. | | `@ignitionai/agent-trainer-adapter-mastra` | ready | ready | ready | ready | partial | Structural adapter only; no memory, tool or full Mastra coverage. | | `@ignitionai/agent-trainer-adapter-vercel-ai` | ready | ready | ready | ready | partial | Structural adapter only; no streaming, tools or live provider calls. | -| `@ignitionai/agent-trainer-cli` | ready | ready | ready | ready | partial | Runs typed experiments and writes standalone reports or timestamped bundles; no history/baseline/regression flags yet. | +| `@ignitionai/agent-trainer-cli` | ready | ready | ready | ready | partial | Runs typed experiments, writes reports/bundles, records local history, selects baselines and runs regression checks; no watch mode or remote execution. | | `@ignitionai/agent-trainer-core` | ready | ready | ready | ready | partial | Foundational helpers have dedicated tests; runtime schema validation remains outside the current helper surface. | | `@ignitionai/agent-trainer-environment` | ready | ready | ready | ready | partial | Tested episode runner with safety guards and a deterministic RAG episode example; no production runtime or optimization loop. | | `@ignitionai/agent-trainer-evals` | ready | ready | ready | ready | partial | Current rewards are tested; RAG presets and richer scoring are still missing. | @@ -55,7 +55,6 @@ All packages declare `license: MIT`, matching the root `LICENSE` file. ## Known Work After Internal Alpha -- Add CLI history, baseline selection and regression command ergonomics after report bundles and CI examples. - Add deeper mocked examples for ecosystem adapters. - Dogfood the alpha inside IgnitionRAG and collect trajectory/reward evidence. - Add a lightweight policy optimization loop after real dogfood produces trajectory data. diff --git a/docs/BACKLOG.md b/docs/BACKLOG.md index 84aa14a..a14dbf3 100644 --- a/docs/BACKLOG.md +++ b/docs/BACKLOG.md @@ -2793,7 +2793,7 @@ Next PR: Status: -- current +- completed Branch: @@ -2850,7 +2850,7 @@ Next PR: Status: -- planned +- current Branch: diff --git a/docs/PROJECT_AUDIT.md b/docs/PROJECT_AUDIT.md index d223620..efaa40a 100644 --- a/docs/PROJECT_AUDIT.md +++ b/docs/PROJECT_AUDIT.md @@ -75,12 +75,12 @@ If a package exists but is intentionally narrow, minimal or untested, it is part ### `@ignitionai/agent-trainer-cli` -- Purpose: run typed experiment modules locally and write JSON/Markdown reports or report bundles. +- Purpose: run typed experiment modules locally, write JSON/Markdown reports or report bundles, record local history and run baseline regression checks. - Main exports: `parseCliArgs`, `runCli`, `CliCommand`, `CliEnvironment`. - Stability level: stable for the current local CLI surface. - Tests present: yes. - Example present: yes, `examples/context-engineering/experiment.ts` through the CLI. -- Known limitations: no watch mode, no persistent history flag, no baseline selection flag, no regression-gate command, no remote execution. +- Known limitations: no watch mode, no hosted history, no remote execution and no provider-backed regression scoring. ### `@ignitionai/agent-trainer-core` diff --git a/packages/cli/README.md b/packages/cli/README.md index d491863..1411348 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -60,6 +60,60 @@ reports/ └─ metadata.json ``` +## Keep Local History + +Use `--history` with `--record-history` to append the result to a local JSONL history file: + +```bash +bun run --filter '@ignitionai/agent-trainer-cli' dev -- eval run ./examples/context-engineering/experiment.ts \ + --history .ignition/experiment-history.jsonl \ + --record-history +``` + +List recent entries: + +```bash +bun run --filter '@ignitionai/agent-trainer-cli' dev -- eval history list .ignition/experiment-history.jsonl \ + --experiment context-engineering-strategies \ + --limit 5 +``` + +Inspect the latest entry for an experiment: + +```bash +bun run --filter '@ignitionai/agent-trainer-cli' dev -- eval history show .ignition/experiment-history.jsonl latest \ + --experiment context-engineering-strategies +``` + +The history file is newline-delimited JSON using `ignition.experiment-history-entry.v1`. + +## Run Regression Checks + +Use `--baseline latest` to compare the current run against the latest matching history entry. +`--regression` makes the command fail when the comparison exceeds the allowed thresholds: + +```bash +bun run --filter '@ignitionai/agent-trainer-cli' dev -- eval run ./examples/context-engineering/experiment.ts \ + --history .ignition/experiment-history.jsonl \ + --baseline latest \ + --regression \ + --max-score-drop 0.03 \ + --max-latency-increase-ms 250 \ + --max-cost-increase-usd 0.002 \ + --regression-markdown reports/regression.md +``` + +You can also pass a concrete history entry id instead of `latest`: + +```bash +bun run --filter '@ignitionai/agent-trainer-cli' dev -- eval run ./examples/context-engineering/experiment.ts \ + --history .ignition/experiment-history.jsonl \ + --baseline context-engineering-strategies-2026-01-01T00-00-00-000Z \ + --regression +``` + +Use `--variant ` one or more times when only specific variants should be checked. + ## Non-goals -The CLI does not implement watch mode, remote execution, hosted dashboards, auth, provider keys, persistent history or regression gates. +The CLI does not implement watch mode, remote execution, hosted dashboards, auth, provider keys, hosted history or provider-backed regression scoring. diff --git a/packages/cli/src/index.test.ts b/packages/cli/src/index.test.ts index c2a16cd..0454162 100644 --- a/packages/cli/src/index.test.ts +++ b/packages/cli/src/index.test.ts @@ -4,9 +4,16 @@ import { join } from "node:path"; import { createDataset, createMockAdapter, + type ExperimentResult, type RewardFunction, + type VariantSummary, } from "@ignitionai/agent-trainer-core"; -import { defineExperiment, type ExperimentDefinition } from "@ignitionai/agent-trainer-experiments"; +import { + appendExperimentHistory, + defineExperiment, + type ExperimentDefinition, + readExperimentHistory, +} from "@ignitionai/agent-trainer-experiments"; import { describe, expect, it } from "vitest"; import { parseCliArgs, runCli } from "./index"; @@ -32,10 +39,110 @@ describe("parseCliArgs", () => { jsonOutputPath: "reports/result.json", markdownOutputPath: "reports/result.md", bundleOutputDirectory: "reports/bundles", + regressionOptions: {}, + }, + }); + }); + + it("parses history, baseline and regression options for eval runs", () => { + expect( + parseCliArgs([ + "eval", + "run", + "./experiment.ts", + "--history", + ".ignition/history.jsonl", + "--baseline", + "latest", + "--regression", + "--max-score-drop", + "0.05", + "--max-latency-increase-ms", + "100", + "--max-cost-increase-usd", + "0.001", + "--variant", + "strong-agent", + "--regression-markdown", + "reports/regression.md", + "--record-history", + ]), + ).toEqual({ + ok: true, + command: { + kind: "eval-run", + experimentPath: "./experiment.ts", + historyPath: ".ignition/history.jsonl", + baseline: "latest", + regression: true, + regressionMarkdownOutputPath: "reports/regression.md", + recordHistory: true, + regressionOptions: { + maxScoreDrop: 0.05, + maxLatencyIncreaseMs: 100, + maxCostIncreaseUsd: 0.001, + variantIds: ["strong-agent"], + }, + }, + }); + }); + + it("parses history list and show commands", () => { + expect( + parseCliArgs([ + "eval", + "history", + "list", + ".ignition/history.jsonl", + "--experiment", + "cli-definition-demo", + "--limit", + "2", + ]), + ).toEqual({ + ok: true, + command: { + kind: "eval-history-list", + historyPath: ".ignition/history.jsonl", + experimentName: "cli-definition-demo", + limit: 2, + }, + }); + + expect( + parseCliArgs([ + "eval", + "history", + "show", + ".ignition/history.jsonl", + "latest", + "--experiment", + "cli-definition-demo", + ]), + ).toEqual({ + ok: true, + command: { + kind: "eval-history-show", + historyPath: ".ignition/history.jsonl", + selector: "latest", + experimentName: "cli-definition-demo", }, }); }); + it("rejects regression flags that cannot select a baseline", () => { + expect(parseCliArgs(["eval", "run", "./experiment.ts", "--regression"])).toEqual({ + ok: false, + message: "--regression requires --baseline.", + exitCode: 1, + }); + expect(parseCliArgs(["eval", "run", "./experiment.ts", "--baseline", "latest"])).toEqual({ + ok: false, + message: "--baseline requires --history.", + exitCode: 1, + }); + }); + it("reports an invalid command clearly", () => { const parsed = parseCliArgs(["train", "run"]); @@ -189,9 +296,163 @@ describe("runCli", () => { }); expect(output.out.join("\n")).toContain(`Report bundle: ${bundleDirectory}`); }); + + it("lists and shows experiment history entries", async () => { + const workspace = await mkdtemp(join(tmpdir(), "ignition-cli-history-")); + const historyPath = join(workspace, ".ignition", "history.jsonl"); + await appendExperimentHistory(historyPath, createHistoryResult(0.82), { + recordedAt: "2026-01-01T00:00:00.000Z", + }); + await appendExperimentHistory(historyPath, createHistoryResult(0.93), { + recordedAt: "2026-01-02T00:00:00.000Z", + }); + + const listOutput = createOutput(); + const listExitCode = await runCli( + [ + "eval", + "history", + "list", + ".ignition/history.jsonl", + "--experiment", + "cli-definition-demo", + "--limit", + "1", + ], + { + cwd: workspace, + stdout: listOutput.stdout, + stderr: listOutput.stderr, + }, + ); + + expect(listExitCode).toBe(0); + expect(listOutput.err).toEqual([]); + const listStdout = listOutput.out.join("\n"); + expect(listStdout).toContain("Entries: 2"); + expect(listStdout).toContain("cli-definition-demo-2026-01-02T00-00-00-000Z"); + expect(listStdout).not.toContain("cli-definition-demo-2026-01-01T00-00-00-000Z"); + + const showOutput = createOutput(); + const showExitCode = await runCli( + [ + "eval", + "history", + "show", + ".ignition/history.jsonl", + "latest", + "--experiment", + "cli-definition-demo", + ], + { + cwd: workspace, + stdout: showOutput.stdout, + stderr: showOutput.stderr, + }, + ); + + expect(showExitCode).toBe(0); + expect(showOutput.err).toEqual([]); + const showStdout = showOutput.out.join("\n"); + expect(showStdout).toContain("History entry: cli-definition-demo-2026-01-02T00-00-00-000Z"); + expect(showStdout).toContain("Experiment: cli-definition-demo"); + expect(showStdout).toContain("1. strong-agent - score 0.930"); + }); + + it("records history and passes a regression check against the latest baseline", async () => { + const workspace = await mkdtemp(join(tmpdir(), "ignition-cli-regression-pass-")); + const historyPath = join(workspace, ".ignition", "history.jsonl"); + await appendExperimentHistory(historyPath, createHistoryResult(0.9), { + recordedAt: "2026-01-01T00:00:00.000Z", + }); + + const output = createOutput(); + const exitCode = await runCli( + [ + "eval", + "run", + "./experiment.ts", + "--history", + ".ignition/history.jsonl", + "--baseline", + "latest", + "--regression", + "--max-score-drop", + "0.2", + "--regression-markdown", + "reports/regression.md", + "--record-history", + ], + { + cwd: workspace, + stdout: output.stdout, + stderr: output.stderr, + fileExists: (absolutePath) => absolutePath === join(workspace, "experiment.ts"), + importModule: async () => ({ default: createCliExperimentDefinition() }), + }, + ); + + expect(exitCode).toBe(0); + expect(output.err).toEqual([]); + const stdout = output.out.join("\n"); + expect(stdout).toContain("Baseline: cli-definition-demo-2026-01-01T00-00-00-000Z"); + expect(stdout).toContain("Regression gate: pass"); + expect(stdout).toContain("Regression Markdown: reports/regression.md"); + expect(stdout).toContain("History entry: cli-definition-demo-"); + + const entries = await readExperimentHistory(historyPath); + expect(entries).toHaveLength(2); + expect(entries[1]?.metadata).toEqual({ + source: "cli", + experimentPath: "./experiment.ts", + }); + + const markdown = await readFile(join(workspace, "reports", "regression.md"), "utf8"); + expect(markdown).toContain("# Regression gate summary"); + expect(markdown).toContain("Result: pass"); + }); + + it("fails clearly when regression checks fail", async () => { + const workspace = await mkdtemp(join(tmpdir(), "ignition-cli-regression-fail-")); + const historyPath = join(workspace, ".ignition", "history.jsonl"); + await appendExperimentHistory(historyPath, createHistoryResult(1), { + recordedAt: "2026-01-01T00:00:00.000Z", + }); + + const output = createOutput(); + const exitCode = await runCli( + [ + "eval", + "run", + "./experiment.ts", + "--history", + ".ignition/history.jsonl", + "--baseline", + "latest", + "--regression", + ], + { + cwd: workspace, + stdout: output.stdout, + stderr: output.stderr, + fileExists: (absolutePath) => absolutePath === join(workspace, "experiment.ts"), + importModule: async () => ({ + default: createCliExperimentDefinition({ strongOutput: "wrong answer" }), + }), + }, + ); + + expect(exitCode).toBe(1); + expect(output.out.join("\n")).toContain("Regression gate: fail"); + expect(output.out.join("\n")).toContain("Result: fail"); + expect(output.err.join("\n")).toContain("Regression gate failed:"); + expect(output.err.join("\n")).toContain("Variant strong-agent score dropped"); + }); }); -function createCliExperimentDefinition(): ExperimentDefinition { +function createCliExperimentDefinition( + options: { strongOutput?: string; weakOutput?: string } = {}, +): ExperimentDefinition { const qualityReward: RewardFunction = { name: "quality", evaluate(run) { @@ -211,7 +472,7 @@ function createCliExperimentDefinition(): ExperimentDefinition { id: "strong-agent", name: "strong-agent", adapter: createMockAdapter({ - output: "correct answer", + output: options.strongOutput ?? "correct answer", trace: { steps: [] }, usage: { latencyMs: 100, costUsd: 0.001 }, }), @@ -220,7 +481,7 @@ function createCliExperimentDefinition(): ExperimentDefinition { id: "weak-agent", name: "weak-agent", adapter: createMockAdapter({ - output: "wrong answer", + output: options.weakOutput ?? "wrong answer", trace: { steps: [] }, usage: { latencyMs: 50, costUsd: 0.0005 }, }), @@ -230,6 +491,48 @@ function createCliExperimentDefinition(): ExperimentDefinition { }); } +function createHistoryResult(strongScore: number): ExperimentResult { + return { + name: "cli-definition-demo", + startedAt: "2026-01-01T00:00:00.000Z", + endedAt: "2026-01-01T00:00:01.000Z", + leaderboard: [ + variantSummary({ + id: "strong-agent", + score: strongScore, + latencyMs: 100, + costUsd: 0.001, + }), + variantSummary({ + id: "weak-agent", + score: 0, + latencyMs: 50, + costUsd: 0.0005, + }), + ], + cases: [], + failedCases: [], + }; +} + +function variantSummary(input: { + id: string; + score: number; + latencyMs: number; + costUsd: number; +}): VariantSummary { + return { + variantId: input.id, + name: input.id, + score: input.score, + totalCases: 1, + averageLatencyMs: input.latencyMs, + totalCostUsd: input.costUsd, + rewardAverages: { quality: input.score }, + failedCases: 0, + }; +} + function createOutput(): { out: string[]; err: string[]; diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index 282325a..e2a8d58 100644 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -6,7 +6,15 @@ import { dirname, isAbsolute, resolve } from "node:path"; import { pathToFileURL } from "node:url"; import { recommendVariant, type VariantRecommendation } from "@ignitionai/agent-trainer"; import type { ExperimentResult } from "@ignitionai/agent-trainer-core"; -import type { ExperimentDefinition } from "@ignitionai/agent-trainer-experiments"; +import { + appendExperimentHistory, + compareExperimentResults, + type ExperimentDefinition, + type ExperimentHistoryEntry, + getLatestExperimentHistoryEntry, + type RegressionGateOptions, + readExperimentHistory, +} from "@ignitionai/agent-trainer-experiments"; import { type ExperimentResultExportOptions, toJsonReport, @@ -20,9 +28,29 @@ export interface EvalRunCommand { jsonOutputPath?: string; markdownOutputPath?: string; bundleOutputDirectory?: string; + historyPath?: string; + recordHistory?: boolean; + baseline?: string; + regression?: boolean; + regressionMarkdownOutputPath?: string; + regressionOptions: RegressionGateOptions; +} + +export interface EvalHistoryListCommand { + kind: "eval-history-list"; + historyPath: string; + experimentName?: string; + limit?: number; } -export type CliCommand = EvalRunCommand; +export interface EvalHistoryShowCommand { + kind: "eval-history-show"; + historyPath: string; + selector: string; + experimentName?: string; +} + +export type CliCommand = EvalRunCommand | EvalHistoryListCommand | EvalHistoryShowCommand; export type ParseCliArgsResult = | { ok: true; command: CliCommand } @@ -52,12 +80,25 @@ const usage = `Ignition Agent Trainer CLI Usage: ignition-agent-trainer eval run [--json ] [--markdown ] [--bundle ] + ignition-agent-trainer eval history list [--experiment ] [--limit ] + ignition-agent-trainer eval history show [--experiment ] Options: - --json Write a JSON experiment report. - --markdown Write a Markdown experiment report. - --bundle Write a timestamped JSON/Markdown report bundle. - -h, --help Show this help message.`; + --json Write a JSON experiment report. + --markdown Write a Markdown experiment report. + --bundle Write a timestamped JSON/Markdown report bundle. + --history Read/write local experiment history JSONL. + --record-history Append the current result to --history after a passing run. + --baseline Compare the current result against a history baseline. + --regression Fail when the baseline comparison has regressions. + --max-score-drop Maximum allowed score drop. Defaults to 0. + --max-latency-increase-ms Maximum allowed latency increase. + --max-cost-increase-usd Maximum allowed cost increase. + --variant Restrict regression checks to a variant. Repeatable. + --regression-markdown Write regression comparison Markdown. + --experiment Filter history entries by experiment name. + --limit Limit history list output. + -h, --help Show this help message.`; export function parseCliArgs(args: string[]): ParseCliArgsResult { if (args.length === 0) { @@ -68,7 +109,20 @@ export function parseCliArgs(args: string[]): ParseCliArgsResult { return { ok: false, message: usage, exitCode: 0 }; } - if (args[0] !== "eval" || args[1] !== "run") { + if (args[0] !== "eval") { + return { + ok: false, + message: `Unknown command: ${args.join(" ")}`, + exitCode: 1, + showUsage: true, + }; + } + + if (args[1] === "history") { + return parseHistoryCommand(args); + } + + if (args[1] !== "run") { return { ok: false, message: `Unknown command: ${args.join(" ")}`, @@ -90,6 +144,7 @@ export function parseCliArgs(args: string[]): ParseCliArgsResult { const command: EvalRunCommand = { kind: "eval-run", experimentPath, + regressionOptions: {}, }; for (let index = 3; index < args.length; index += 1) { @@ -126,6 +181,83 @@ export function parseCliArgs(args: string[]): ParseCliArgsResult { continue; } + if (arg === "--history") { + const value = args[index + 1]; + if (value === undefined || value.startsWith("-")) { + return { ok: false, message: "Missing value for --history.", exitCode: 1 }; + } + command.historyPath = value; + index += 1; + continue; + } + + if (arg === "--record-history") { + command.recordHistory = true; + continue; + } + + if (arg === "--baseline") { + const value = args[index + 1]; + if (value === undefined || value.startsWith("-")) { + return { ok: false, message: "Missing value for --baseline.", exitCode: 1 }; + } + command.baseline = value; + index += 1; + continue; + } + + if (arg === "--regression") { + command.regression = true; + continue; + } + + if (arg === "--max-score-drop") { + const parsed = parseNumberOption(args[index + 1], "--max-score-drop"); + if (!parsed.ok) return parsed; + command.regressionOptions.maxScoreDrop = parsed.value; + index += 1; + continue; + } + + if (arg === "--max-latency-increase-ms") { + const parsed = parseNumberOption(args[index + 1], "--max-latency-increase-ms"); + if (!parsed.ok) return parsed; + command.regressionOptions.maxLatencyIncreaseMs = parsed.value; + index += 1; + continue; + } + + if (arg === "--max-cost-increase-usd") { + const parsed = parseNumberOption(args[index + 1], "--max-cost-increase-usd"); + if (!parsed.ok) return parsed; + command.regressionOptions.maxCostIncreaseUsd = parsed.value; + index += 1; + continue; + } + + if (arg === "--variant") { + const value = args[index + 1]; + if (value === undefined || value.startsWith("-")) { + return { ok: false, message: "Missing value for --variant.", exitCode: 1 }; + } + command.regressionOptions.variantIds = [ + ...(command.regressionOptions.variantIds ?? []), + value, + ]; + index += 1; + continue; + } + + if (arg === "--regression-markdown") { + const value = args[index + 1]; + if (value === undefined || value.startsWith("-")) { + return { ok: false, message: "Missing value for --regression-markdown.", exitCode: 1 }; + } + command.regressionMarkdownOutputPath = value; + index += 1; + continue; + } + if (arg === "-h" || arg === "--help") { return { ok: false, message: usage, exitCode: 0 }; } @@ -133,6 +265,9 @@ export function parseCliArgs(args: string[]): ParseCliArgsResult { return { ok: false, message: `Unknown option: ${arg}`, exitCode: 1 }; } + const validation = validateEvalRunCommand(command); + if (validation !== null) return validation; + return { ok: true, command }; } @@ -163,16 +298,29 @@ export async function runCli( } async function runCommand(command: CliCommand, env: ResolvedCliEnvironment): Promise { + if (command.kind === "eval-history-list") { + await printHistoryList(command, env); + return; + } + + if (command.kind === "eval-history-show") { + await printHistoryEntry(command, env); + return; + } + if (command.kind !== "eval-run") { - throw new Error(`Unsupported command: ${command.kind}`); + throw new Error(`Unsupported command: ${(command as { kind: string }).kind}`); } + const history = await readRequestedHistory(command, env); const definition = await loadExperimentDefinition(command.experimentPath, env); const result = await definition.run(); const recommendation = recommendVariant(result); printExperimentSummary(result, definition, recommendation, env.stdout); await writeRequestedReports(command, result, recommendation, env); + await compareAgainstBaseline(command, result, history, env); + await recordHistoryEntry(command, result, env); } async function loadExperimentDefinition( @@ -299,6 +447,134 @@ async function writeRequestedReports( } } +async function readRequestedHistory( + command: EvalRunCommand, + env: ResolvedCliEnvironment, +): Promise { + if (command.historyPath === undefined) return null; + return readExperimentHistory(resolveCliPath(command.historyPath, env)); +} + +async function compareAgainstBaseline( + command: EvalRunCommand, + result: ExperimentResult, + history: ExperimentHistoryEntry[] | null, + env: ResolvedCliEnvironment, +): Promise { + if (command.baseline === undefined) return; + if (history === null) { + throw new Error("--baseline requires --history."); + } + + const baseline = selectHistoryEntry(history, command.baseline, result.name); + if (baseline === null) { + throw new Error(`Baseline not found: ${command.baseline} for experiment ${result.name}.`); + } + + const comparison = compareExperimentResults(result, baseline.result, command.regressionOptions); + env.stdout(""); + env.stdout(`Baseline: ${baseline.id} (${baseline.recordedAt})`); + env.stdout(`Regression gate: ${comparison.passed ? "pass" : "fail"}`); + writeLines(env.stdout, comparison.markdown.trimEnd()); + + if (command.regressionMarkdownOutputPath !== undefined) { + await writeReport(command.regressionMarkdownOutputPath, comparison.markdown, env); + env.stdout(`Regression Markdown: ${command.regressionMarkdownOutputPath}`); + } + + if (command.regression === true && !comparison.passed) { + throw new Error( + `Regression gate failed:\n${comparison.failures + .map((failure) => `- ${failure.message}`) + .join("\n")}`, + ); + } +} + +async function recordHistoryEntry( + command: EvalRunCommand, + result: ExperimentResult, + env: ResolvedCliEnvironment, +): Promise { + if (command.recordHistory !== true) return; + if (command.historyPath === undefined) { + throw new Error("--record-history requires --history."); + } + + const entry = await appendExperimentHistory(resolveCliPath(command.historyPath, env), result, { + metadata: { + source: "cli", + experimentPath: command.experimentPath, + }, + }); + env.stdout(`History entry: ${entry.id}`); +} + +async function printHistoryList( + command: EvalHistoryListCommand, + env: ResolvedCliEnvironment, +): Promise { + const entries = await readExperimentHistory(resolveCliPath(command.historyPath, env)); + const filtered = filterHistoryEntries(entries, command.experimentName); + const visible = + command.limit === undefined + ? filtered + : filtered.slice(Math.max(0, filtered.length - command.limit)); + + env.stdout(`History: ${command.historyPath}`); + if (command.experimentName !== undefined) { + env.stdout(`Experiment filter: ${command.experimentName}`); + } + env.stdout(`Entries: ${filtered.length}`); + + if (filtered.length === 0) { + env.stdout("No history entries found."); + return; + } + + env.stdout(""); + for (const entry of [...visible].reverse()) { + const winner = entry.result.leaderboard[0]; + env.stdout( + `${entry.id} | ${entry.recordedAt} | ${entry.result.name} | ${formatWinner(winner)}`, + ); + } +} + +async function printHistoryEntry( + command: EvalHistoryShowCommand, + env: ResolvedCliEnvironment, +): Promise { + const entries = await readExperimentHistory(resolveCliPath(command.historyPath, env)); + const entry = selectHistoryEntry(entries, command.selector, command.experimentName); + if (entry === null) { + throw new Error(`History entry not found: ${command.selector}.`); + } + + env.stdout(`History entry: ${entry.id}`); + env.stdout(`Recorded at: ${entry.recordedAt}`); + env.stdout(`Experiment: ${entry.result.name}`); + env.stdout(`Cases: ${entry.result.cases.length}`); + env.stdout(`Failed cases: ${entry.result.failedCases.length}`); + if (entry.metadata !== undefined) { + env.stdout(`Metadata: ${JSON.stringify(entry.metadata)}`); + } + env.stdout(""); + env.stdout("Leaderboard:"); + if (entry.result.leaderboard.length === 0) { + env.stdout("No variants were reported."); + return; + } + + for (const [index, row] of entry.result.leaderboard.entries()) { + env.stdout( + `${index + 1}. ${row.name} - score ${row.score.toFixed(3)} (${row.totalCases} cases, ${ + row.failedCases + } failed)`, + ); + } +} + async function writeReport( outputPath: string, contents: string, @@ -310,7 +586,167 @@ async function writeReport( } function resolveOutputPath(outputPath: string, env: ResolvedCliEnvironment): string { - return isAbsolute(outputPath) ? outputPath : resolve(env.cwd, outputPath); + return resolveCliPath(outputPath, env); +} + +function resolveCliPath(inputPath: string, env: ResolvedCliEnvironment): string { + return isAbsolute(inputPath) ? inputPath : resolve(env.cwd, inputPath); +} + +function parseHistoryCommand(args: string[]): ParseCliArgsResult { + const action = args[2]; + const historyPath = args[3]; + + if (action !== "list" && action !== "show") { + return { + ok: false, + message: + "Missing history action. Expected: ignition-agent-trainer eval history ", + exitCode: 1, + showUsage: true, + }; + } + + if (historyPath === undefined || historyPath.startsWith("-")) { + return { + ok: false, + message: `Missing history path. Expected: ignition-agent-trainer eval history ${action} `, + exitCode: 1, + showUsage: true, + }; + } + + if (action === "list") { + const command: EvalHistoryListCommand = { + kind: "eval-history-list", + historyPath, + }; + for (let index = 4; index < args.length; index += 1) { + const arg = args[index]; + if (arg === "--experiment") { + const value = args[index + 1]; + if (value === undefined || value.startsWith("-")) { + return { ok: false, message: "Missing value for --experiment.", exitCode: 1 }; + } + command.experimentName = value; + index += 1; + continue; + } + if (arg === "--limit") { + const parsed = parseIntegerOption(args[index + 1], "--limit"); + if (!parsed.ok) return parsed; + command.limit = parsed.value; + index += 1; + continue; + } + return { ok: false, message: `Unknown option: ${arg}`, exitCode: 1 }; + } + return { ok: true, command }; + } + + const selector = args[4]; + if (selector === undefined || selector.startsWith("-")) { + return { + ok: false, + message: + "Missing history entry selector. Expected: ignition-agent-trainer eval history show ", + exitCode: 1, + showUsage: true, + }; + } + + const command: EvalHistoryShowCommand = { + kind: "eval-history-show", + historyPath, + selector, + }; + for (let index = 5; index < args.length; index += 1) { + const arg = args[index]; + if (arg === "--experiment") { + const value = args[index + 1]; + if (value === undefined || value.startsWith("-")) { + return { ok: false, message: "Missing value for --experiment.", exitCode: 1 }; + } + command.experimentName = value; + index += 1; + continue; + } + return { ok: false, message: `Unknown option: ${arg}`, exitCode: 1 }; + } + return { ok: true, command }; +} + +function validateEvalRunCommand(command: EvalRunCommand): ParseCliArgsResult | null { + if (command.recordHistory === true && command.historyPath === undefined) { + return { ok: false, message: "--record-history requires --history.", exitCode: 1 }; + } + if (command.baseline !== undefined && command.historyPath === undefined) { + return { ok: false, message: "--baseline requires --history.", exitCode: 1 }; + } + if (command.regression === true && command.baseline === undefined) { + return { ok: false, message: "--regression requires --baseline.", exitCode: 1 }; + } + if (command.regressionMarkdownOutputPath !== undefined && command.baseline === undefined) { + return { ok: false, message: "--regression-markdown requires --baseline.", exitCode: 1 }; + } + return null; +} + +function parseNumberOption( + value: string | undefined, + optionName: string, +): { ok: true; value: number } | { ok: false; message: string; exitCode: number } { + if (value === undefined || value.startsWith("-")) { + return { ok: false, message: `Missing value for ${optionName}.`, exitCode: 1 }; + } + const number = Number(value); + if (!Number.isFinite(number) || number < 0) { + return { ok: false, message: `Invalid value for ${optionName}: ${value}`, exitCode: 1 }; + } + return { ok: true, value: number }; +} + +function parseIntegerOption( + value: string | undefined, + optionName: string, +): { ok: true; value: number } | { ok: false; message: string; exitCode: number } { + const parsed = parseNumberOption(value, optionName); + if (!parsed.ok) return parsed; + if (!Number.isInteger(parsed.value) || parsed.value < 1) { + return { ok: false, message: `Invalid value for ${optionName}: ${value}`, exitCode: 1 }; + } + return parsed; +} + +function selectHistoryEntry( + entries: readonly ExperimentHistoryEntry[], + selector: string, + experimentName?: string, +): ExperimentHistoryEntry | null { + if (selector === "latest") { + return getLatestExperimentHistoryEntry(entries, experimentName); + } + + return ( + entries.find( + (entry) => + entry.id === selector && + (experimentName === undefined || entry.result.name === experimentName), + ) ?? null + ); +} + +function filterHistoryEntries( + entries: readonly ExperimentHistoryEntry[], + experimentName: string | undefined, +): ExperimentHistoryEntry[] { + if (experimentName === undefined) return [...entries]; + return entries.filter((entry) => entry.result.name === experimentName); +} + +function formatWinner(winner: ExperimentResult["leaderboard"][number] | undefined): string { + if (winner === undefined) return "no winner"; + return `winner ${winner.name} score ${winner.score.toFixed(3)}`; } function resolveEnvironment(environment: CliEnvironment): ResolvedCliEnvironment {