From 886a799327fea4963f858ef56593d419b5895cf3 Mon Sep 17 00:00:00 2001 From: salimlaimeche Date: Wed, 1 Jul 2026 00:53:11 +0200 Subject: [PATCH] feat(cli): add environment trajectory commands --- README.md | 9 + bun.lock | 2 + docs/ALPHA_READINESS.md | 3 +- docs/BACKLOG.md | 9 +- docs/PROJECT_AUDIT.md | 10 +- examples/rag-environment-episode/README.md | 13 + examples/rag-environment-episode/src/index.ts | 85 +++-- packages/cli/README.md | 31 +- packages/cli/package.json | 4 +- packages/cli/src/index.test.ts | 178 ++++++++++ packages/cli/src/index.ts | 306 +++++++++++++++++- packages/environment/README.md | 22 +- packages/environment/src/environment.test.ts | 59 ++++ packages/environment/src/environment.ts | 84 +++++ packages/rl/README.md | 9 + 15 files changed, 767 insertions(+), 57 deletions(-) diff --git a/README.md b/README.md index e4d97b8..59548e2 100644 --- a/README.md +++ b/README.md @@ -155,6 +155,15 @@ Run a deterministic RAG environment episode and trajectory report: bun run --filter './examples/rag-environment-episode' dev ``` +Run that episode through the local CLI and write trajectory reports: + +```bash +bun run --filter '@ignitionai/agent-trainer-cli' dev -- environment run ./examples/rag-environment-episode/src/index.ts \ + --json reports/rag-trajectory.json \ + --markdown reports/rag-trajectory.md \ + --offline-records +``` + --- ## Minimal usage diff --git a/bun.lock b/bun.lock index e032b8c..f5d9e12 100644 --- a/bun.lock +++ b/bun.lock @@ -205,8 +205,10 @@ "dependencies": { "@ignitionai/agent-trainer": "0.1.0-alpha.1", "@ignitionai/agent-trainer-core": "0.1.0-alpha.1", + "@ignitionai/agent-trainer-environment": "0.1.0-alpha.1", "@ignitionai/agent-trainer-experiments": "0.1.0-alpha.1", "@ignitionai/agent-trainer-exporters": "0.1.0-alpha.1", + "@ignitionai/agent-trainer-rl": "0.1.0-alpha.1", }, "devDependencies": { "tsup": "latest", diff --git a/docs/ALPHA_READINESS.md b/docs/ALPHA_READINESS.md index 9c663c5..eacb0a9 100644 --- a/docs/ALPHA_READINESS.md +++ b/docs/ALPHA_READINESS.md @@ -38,7 +38,7 @@ All packages declare `license: MIT`, matching the root `LICENSE` file. | `@ignitionai/agent-trainer-rl` | ready | ready | partial | ready | prototype | Deterministic policy helpers, trajectory recorder, fixed-strategy bandits, offline policy evaluation, GRPO-style selection and PPO interface skeletons are tested; deeper RL is intentionally absent. | | `@ignitionai/agent-trainer` | ready | ready | ready | ready | ready | Deterministic recommendation, candidate evaluation and grid search are tested. | -## Verified Through PR #49 +## Verified Through PR #50 - Package names use the `@ignitionai/*` scope. - Root and workspace package versions are aligned on `0.1.0-alpha.0`. @@ -56,6 +56,7 @@ All packages declare `license: MIT`, matching the root `LICENSE` file. - Ecosystem adapters have a deterministic grouped example covering LangChain, LangGraph, Mastra and Vercel AI SDK-style shapes. - Core runtime validation helpers cover datasets, variants, run results, usage metrics, traces, metric/reward results, normalized scores and JSON-compatible fields. - npm alpha publishing policy is manual-only for `v0.1.0-alpha.x`; automated publishing is disabled until a dedicated Trusted Publishing/OIDC PR exists. +- The CLI can run deterministic environment episode modules, export trajectory JSON/Markdown reports and print offline policy record counts. ## Known Work After Internal Alpha diff --git a/docs/BACKLOG.md b/docs/BACKLOG.md index ef29c1e..c307400 100644 --- a/docs/BACKLOG.md +++ b/docs/BACKLOG.md @@ -10,9 +10,8 @@ Status values: Current snapshot: -- Completed through PR #48. -- PR #49 is the current release-policy PR. -- PR #50 is the next planned CLI/environment ergonomics PR. +- Completed through PR #49. +- PR #50 is the current CLI/environment ergonomics PR. - The dogfood-driven policy optimization loop stays blocked until dogfood or representative trajectory fixtures exist. ## Stable PR sequence @@ -3041,7 +3040,7 @@ Next PR: Status: -- current +- completed Branch: @@ -3099,7 +3098,7 @@ Next PR: Status: -- planned +- current Branch: diff --git a/docs/PROJECT_AUDIT.md b/docs/PROJECT_AUDIT.md index 5cb05bf..54f6742 100644 --- a/docs/PROJECT_AUDIT.md +++ b/docs/PROJECT_AUDIT.md @@ -240,7 +240,7 @@ If a package exists but is intentionally narrow, minimal or untested, it is part | context engineering example | done | `examples/context-engineering` | Yes | Mocked strategy comparison plus CLI module. | | exporters | done | `packages/exporters` | Yes | Stable JSON/Markdown export shape and local report bundle writer. | | typed experiment definitions | done | `packages/experiments/src/definition.ts` | Yes | Used by the CLI example. | -| CLI runner | partial | `packages/cli` | Yes | Runs experiments, reports, bundles, local history, baseline selection and regression checks. | +| CLI runner | partial | `packages/cli` | Yes | Runs experiments, reports, bundles, local history, baseline selection, regression checks and environment episode trajectory reports. | | regression gates | done | `packages/experiments/src/regression-gates.ts` | Yes | Tested comparison helpers plus a copyable CI example. | | alpha dogfood workflow | done | `examples/alpha-dogfood` | Yes | Deterministic IgnitionRAG-style document assistant evaluation with report exports and a regression gate. | | ecosystem adapters | partial | `packages/adapter-*` | No | Structural adapters exist with tests/docs and a grouped mocked example; deeper framework internals remain out of scope. | @@ -250,9 +250,9 @@ If a package exists but is intentionally narrow, minimal or untested, it is part | IgnitionRAG evaluation bridge prototype | prototype | `examples/ignitionrag-evaluation-bridge` | No | Deterministic record mapping only; no database, hosted worker, auth or real provider calls. | | file-based history | done | `packages/experiments/src/history.ts` and `packages/cli` | Yes | JSONL local history helpers plus CLI list/show/record workflows. | | policy abstraction | partial | `packages/rl/src/policy.ts` | No | Deterministic static and score-based selection only; no training loop. | -| environment episodes | partial | `packages/environment` | No | Tested episode runner with max-step safety, seed forwarding, final state and metadata. | -| trajectory recorder | partial | `packages/rl/src/trajectory.ts` | No | Local state/action/reward/outcome records with deterministic summaries. | -| trajectory reports | partial | `packages/rl/src/episode-trajectory.ts` | No | Environment episodes can become trajectories, Markdown reports and offline policy records. | +| environment episodes | partial | `packages/environment` | No | Tested episode runner with max-step safety, seed forwarding, final state, metadata and reusable CLI-loadable episode definitions. | +| trajectory recorder | partial | `packages/rl/src/trajectory.ts` | No | Local state/action/reward/outcome records with deterministic summaries and CLI report export support. | +| trajectory reports | partial | `packages/rl/src/episode-trajectory.ts` | No | Environment episodes can become trajectories, Markdown reports, JSON reports and offline policy records. | | npm publishing policy | done | `docs/NPM_ALPHA_PUBLISHING.md` | Yes | Alpha publishing is manual-only with `--tag alpha`; future automation must use Trusted Publishing/OIDC and manual approval. | | bandit prototype | prototype | `packages/rl/src/strategy-bandit.ts` | No | Clearly experimental, fixed arms only, no PPO. | | contextual bandit prototype | prototype | `packages/rl/src/contextual-bandit.ts` | No | Deterministic fixed-feature scoring over task type, citation need, cost sensitivity, latency sensitivity and risk level. | @@ -270,7 +270,7 @@ If a package exists but is intentionally narrow, minimal or untested, it is part - IgnitionRAG integration is design-only. - Core still needs full experiment report schema validation before alpha-stable status. - Lightweight policy optimization remains blocked until dogfood or representative trajectory fixtures exist. -- Environment episodes still need real dogfood data and CLI ergonomics before stable status. +- Environment episodes still need real dogfood data before stable status. - Ecosystem adapters are minimal and structural. - CLI still lacks watch mode, remote execution and hosted history. - Bandit support is prototype-only. diff --git a/examples/rag-environment-episode/README.md b/examples/rag-environment-episode/README.md index 37c4654..47952e1 100644 --- a/examples/rag-environment-episode/README.md +++ b/examples/rag-environment-episode/README.md @@ -16,4 +16,17 @@ Each action returns a reward. The completed episode is converted into a trajecto bun run --filter './examples/rag-environment-episode' dev ``` +Run the same episode through the local CLI and write trajectory reports: + +```bash +bun run --filter '@ignitionai/agent-trainer-cli' dev -- environment run ./examples/rag-environment-episode/src/index.ts \ + --seed 7 \ + --max-steps 10 \ + --policy-id scripted-rag-policy \ + --trajectory-id rag-environment-episode \ + --json reports/rag-trajectory.json \ + --markdown reports/rag-trajectory.md \ + --offline-records +``` + The example is fully mocked. It does not call LLM providers, vector databases or external tools. diff --git a/examples/rag-environment-episode/src/index.ts b/examples/rag-environment-episode/src/index.ts index 624581c..d3d2e06 100644 --- a/examples/rag-environment-episode/src/index.ts +++ b/examples/rag-environment-episode/src/index.ts @@ -1,10 +1,11 @@ +import { pathToFileURL } from "node:url"; import { type AgentEnvironment, + defineEnvironmentEpisode, type EnvironmentAction, type EnvironmentState, type EnvironmentStepResult, type Policy, - runEpisode, } from "@ignitionai/agent-trainer-environment"; import { createOfflinePolicyRecordsFromTrajectories, @@ -16,7 +17,7 @@ import { type RagActionName = "search" | "rerank" | "verify" | "answer"; -class ScriptedRagPolicy implements Policy { +export class ScriptedRagPolicy implements Policy { private readonly sequence: RagActionName[] = ["search", "rerank", "verify", "answer"]; private index = 0; @@ -32,7 +33,7 @@ class ScriptedRagPolicy implements Policy { } } -class MockRagEnvironment implements AgentEnvironment { +export class MockRagEnvironment implements AgentEnvironment { private stage: RagActionName | "start" | "done" = "start"; async reset(seed?: number): Promise { @@ -116,38 +117,53 @@ class MockRagEnvironment implements AgentEnvironment { } } -const episode = await runEpisode(new MockRagEnvironment(), new ScriptedRagPolicy(), { - seed: 7, - policyId: "scripted-rag-policy", - metadata: { example: "rag-environment-episode" }, +const definition = defineEnvironmentEpisode({ + name: "rag-environment-episode", + environment: () => new MockRagEnvironment(), + policy: () => new ScriptedRagPolicy(), + options: { + seed: 7, + policyId: "scripted-rag-policy", + metadata: { example: "rag-environment-episode" }, + }, }); -const trajectory = recordEpisodeTrajectory(episode, { - id: "rag-environment-episode", - startedAt: "2026-06-30T00:00:00.000Z", - endedAt: "2026-06-30T00:00:04.000Z", -}); -const summary = summarizeTrajectory(trajectory); -const offlineRecords = createOfflinePolicyRecordsFromTrajectories([trajectory], { - experimentName: "rag-environment-episode", -}); -const report = exportTrajectoryReport(trajectory, { - generatedAt: "2026-06-30T00:00:05.000Z", -}); +export default definition; + +export async function runExample(): Promise { + const episode = await definition.run(); + + const trajectory = recordEpisodeTrajectory(episode, { + id: "rag-environment-episode", + startedAt: "2026-06-30T00:00:00.000Z", + endedAt: "2026-06-30T00:00:04.000Z", + }); + const summary = summarizeTrajectory(trajectory); + const offlineRecords = createOfflinePolicyRecordsFromTrajectories([trajectory], { + experimentName: "rag-environment-episode", + }); + const report = exportTrajectoryReport(trajectory, { + generatedAt: "2026-06-30T00:00:05.000Z", + }); -console.log("RAG environment episode"); -console.table( - episode.steps.map((step) => ({ - action: step.action.name, - reward: (step.reward.score * step.reward.weight).toFixed(3), - done: step.done, - })), -); -console.log(`Total reward: ${episode.totalReward.toFixed(3)}`); -console.log(`Trajectory steps: ${summary.stepCount}`); -console.log(`Offline records: ${offlineRecords.length}`); -console.log(""); -console.log(toMarkdownTrajectoryReport(report)); + console.log("RAG environment episode"); + console.table( + episode.steps.map((step) => ({ + action: step.action.name, + reward: (step.reward.score * step.reward.weight).toFixed(3), + done: step.done, + })), + ); + console.log(`Total reward: ${episode.totalReward.toFixed(3)}`); + console.log(`Trajectory steps: ${summary.stepCount}`); + console.log(`Offline records: ${offlineRecords.length}`); + console.log(""); + console.log(toMarkdownTrajectoryReport(report)); +} + +if (isMainModule()) { + await runExample(); +} function state( id: string, @@ -160,3 +176,8 @@ function state( ...(done ? { done: true } : {}), }; } + +function isMainModule(): boolean { + const entrypoint = process.argv[1]; + return entrypoint !== undefined && import.meta.url === pathToFileURL(entrypoint).href; +} diff --git a/packages/cli/README.md b/packages/cli/README.md index 1411348..e74c55d 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -114,6 +114,35 @@ bun run --filter '@ignitionai/agent-trainer-cli' dev -- eval run ./examples/cont Use `--variant ` one or more times when only specific variants should be checked. +## Run Environment Episodes + +Use `environment run` for deterministic `@ignitionai/agent-trainer-environment` episode modules: + +```bash +bun run --filter '@ignitionai/agent-trainer-cli' dev -- environment run ./examples/rag-environment-episode/src/index.ts \ + --seed 7 \ + --max-steps 10 \ + --policy-id scripted-rag-policy \ + --trajectory-id rag-environment-episode \ + --json reports/rag-trajectory.json \ + --markdown reports/rag-trajectory.md \ + --offline-records +``` + +The episode file must default export an `EnvironmentEpisodeDefinition` created with `defineEnvironmentEpisode()`: + +```ts +import { defineEnvironmentEpisode } from "@ignitionai/agent-trainer-environment"; + +export default defineEnvironmentEpisode({ + name: "rag-environment-episode", + environment: () => environment, + policy: () => policy, +}); +``` + +The command prints step rewards, total reward, average reward and trajectory summary. `--json` writes the stable `ignition.trajectory-report.v1` shape. `--markdown` writes the Markdown trajectory report. `--offline-records` prints how many offline policy records can be created from the trajectory. + ## Non-goals -The CLI does not implement watch mode, remote execution, hosted dashboards, auth, provider keys, hosted history or provider-backed regression scoring. +The CLI does not implement watch mode, remote execution, hosted dashboards, auth, provider keys, hosted history, policy optimization, model training or provider-backed regression scoring. diff --git a/packages/cli/package.json b/packages/cli/package.json index 90f4bdc..76962c2 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -32,8 +32,10 @@ "dependencies": { "@ignitionai/agent-trainer": "0.1.0-alpha.1", "@ignitionai/agent-trainer-core": "0.1.0-alpha.1", + "@ignitionai/agent-trainer-environment": "0.1.0-alpha.1", "@ignitionai/agent-trainer-experiments": "0.1.0-alpha.1", - "@ignitionai/agent-trainer-exporters": "0.1.0-alpha.1" + "@ignitionai/agent-trainer-exporters": "0.1.0-alpha.1", + "@ignitionai/agent-trainer-rl": "0.1.0-alpha.1" }, "devDependencies": { "tsup": "latest", diff --git a/packages/cli/src/index.test.ts b/packages/cli/src/index.test.ts index 0454162..6352a0a 100644 --- a/packages/cli/src/index.test.ts +++ b/packages/cli/src/index.test.ts @@ -8,6 +8,13 @@ import { type RewardFunction, type VariantSummary, } from "@ignitionai/agent-trainer-core"; +import { + type AgentEnvironment, + defineEnvironmentEpisode, + type EnvironmentAction, + type EnvironmentState, + type Policy, +} from "@ignitionai/agent-trainer-environment"; import { appendExperimentHistory, defineExperiment, @@ -130,6 +137,45 @@ describe("parseCliArgs", () => { }); }); + it("parses environment episode run commands", () => { + expect( + parseCliArgs([ + "environment", + "run", + "./episode.ts", + "--seed", + "0", + "--max-steps", + "4", + "--policy-id", + "cli-policy", + "--trajectory-id", + "cli-trajectory", + "--metadata-json", + '{"suite":"cli"}', + "--json", + "reports/trajectory.json", + "--markdown", + "reports/trajectory.md", + "--offline-records", + ]), + ).toEqual({ + ok: true, + command: { + kind: "environment-run", + episodePath: "./episode.ts", + seed: 0, + maxSteps: 4, + policyId: "cli-policy", + trajectoryId: "cli-trajectory", + metadata: { suite: "cli" }, + jsonOutputPath: "reports/trajectory.json", + markdownOutputPath: "reports/trajectory.md", + printOfflineRecords: true, + }, + }); + }); + it("rejects regression flags that cannot select a baseline", () => { expect(parseCliArgs(["eval", "run", "./experiment.ts", "--regression"])).toEqual({ ok: false, @@ -141,6 +187,11 @@ describe("parseCliArgs", () => { message: "--baseline requires --history.", exitCode: 1, }); + expect(parseCliArgs(["environment", "run", "./episode.ts", "--metadata-json", "[]"])).toEqual({ + ok: false, + message: "--metadata-json must be a JSON object.", + exitCode: 1, + }); }); it("reports an invalid command clearly", () => { @@ -448,6 +499,82 @@ describe("runCli", () => { expect(output.err.join("\n")).toContain("Regression gate failed:"); expect(output.err.join("\n")).toContain("Variant strong-agent score dropped"); }); + + it("runs an environment episode and writes trajectory reports", async () => { + const workspace = await mkdtemp(join(tmpdir(), "ignition-cli-environment-")); + const output = createOutput(); + + const exitCode = await runCli( + [ + "environment", + "run", + "./episode.ts", + "--seed", + "9", + "--max-steps", + "2", + "--policy-id", + "cli-policy", + "--trajectory-id", + "cli-trajectory", + "--metadata-json", + '{"suite":"cli"}', + "--json", + "reports/trajectory.json", + "--markdown", + "reports/trajectory.md", + "--offline-records", + ], + { + cwd: workspace, + stdout: output.stdout, + stderr: output.stderr, + fileExists: (absolutePath) => absolutePath === join(workspace, "episode.ts"), + importModule: async () => ({ default: createCliEnvironmentEpisodeDefinition() }), + }, + ); + + expect(exitCode).toBe(0); + expect(output.err).toEqual([]); + const stdout = output.out.join("\n"); + expect(stdout).toContain("Environment episode: cli-environment-demo"); + expect(stdout).toContain("Policy: cli-policy"); + expect(stdout).toContain("1. answer - reward 1.000 - done yes"); + expect(stdout).toContain("Total reward: 1.000"); + expect(stdout).toContain("Trajectory steps: 1"); + expect(stdout).toContain("Offline records: 1"); + expect(stdout).toContain("Trajectory JSON report: reports/trajectory.json"); + expect(stdout).toContain("Trajectory Markdown report: reports/trajectory.md"); + + const json = JSON.parse(await readFile(join(workspace, "reports", "trajectory.json"), "utf8")); + expect(json).toMatchObject({ + schemaVersion: "ignition.trajectory-report.v1", + trajectory: { + id: "cli-trajectory", + policyId: "cli-policy", + }, + summary: { + stepCount: 1, + totalReward: 1, + }, + steps: [ + { + state: { + observation: { + seed: 9, + }, + }, + action: { + name: "answer", + }, + }, + ], + }); + + const markdown = await readFile(join(workspace, "reports", "trajectory.md"), "utf8"); + expect(markdown).toContain("# Trajectory report: cli-trajectory"); + expect(markdown).toContain("| 0 | answer | 1.000 | yes |"); + }); }); function createCliExperimentDefinition( @@ -515,6 +642,57 @@ function createHistoryResult(strongScore: number): ExperimentResult { }; } +function createCliEnvironmentEpisodeDefinition() { + return defineEnvironmentEpisode({ + name: "cli-environment-demo", + environment: () => new CliEpisodeEnvironment(), + policy: new FirstActionPolicy(), + options: { + metadata: { defaultOption: true }, + }, + }); +} + +class CliEpisodeEnvironment implements AgentEnvironment { + async reset(seed?: number): Promise { + return { + id: "start", + observation: { seed: seed ?? 0 }, + }; + } + + async actions(): Promise { + return [{ name: "answer", input: { mode: "direct" } }]; + } + + async step(state: EnvironmentState, action: EnvironmentAction) { + return { + state: { + id: "answered", + observation: { + seed: typeof state.observation.seed === "number" ? state.observation.seed : 0, + action: action.name, + }, + done: true, + }, + reward: { + name: "quality", + score: 1, + weight: 1, + }, + done: true, + }; + } +} + +class FirstActionPolicy implements Policy { + async chooseAction(_state: EnvironmentState, actions: EnvironmentAction[]) { + const action = actions[0]; + if (action === undefined) throw new Error("No action available."); + return action; + } +} + function variantSummary(input: { id: string; score: number; diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index e2a8d58..87fa0a4 100644 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -5,7 +5,11 @@ import { mkdir, writeFile } from "node:fs/promises"; import { dirname, isAbsolute, resolve } from "node:path"; import { pathToFileURL } from "node:url"; import { recommendVariant, type VariantRecommendation } from "@ignitionai/agent-trainer"; -import type { ExperimentResult } from "@ignitionai/agent-trainer-core"; +import type { ExperimentResult, Metadata } from "@ignitionai/agent-trainer-core"; +import type { + EnvironmentEpisodeDefinition, + RunEpisodeOptions, +} from "@ignitionai/agent-trainer-environment"; import { appendExperimentHistory, compareExperimentResults, @@ -21,6 +25,13 @@ import { toMarkdownReport, writeReportBundle, } from "@ignitionai/agent-trainer-exporters"; +import { + createOfflinePolicyRecordsFromTrajectories, + exportTrajectoryReport, + recordEpisodeTrajectory, + summarizeTrajectory, + toMarkdownTrajectoryReport, +} from "@ignitionai/agent-trainer-rl"; export interface EvalRunCommand { kind: "eval-run"; @@ -50,7 +61,24 @@ export interface EvalHistoryShowCommand { experimentName?: string; } -export type CliCommand = EvalRunCommand | EvalHistoryListCommand | EvalHistoryShowCommand; +export interface EnvironmentRunCommand { + kind: "environment-run"; + episodePath: string; + seed?: number; + maxSteps?: number; + policyId?: string; + trajectoryId?: string; + metadata?: Metadata; + jsonOutputPath?: string; + markdownOutputPath?: string; + printOfflineRecords?: boolean; +} + +export type CliCommand = + | EvalRunCommand + | EvalHistoryListCommand + | EvalHistoryShowCommand + | EnvironmentRunCommand; export type ParseCliArgsResult = | { ok: true; command: CliCommand } @@ -82,6 +110,7 @@ Usage: ignition-agent-trainer eval run [--json ] [--markdown ] [--bundle ] ignition-agent-trainer eval history list [--experiment ] [--limit ] ignition-agent-trainer eval history show [--experiment ] + ignition-agent-trainer environment run [--json ] [--markdown ] Options: --json Write a JSON experiment report. @@ -98,6 +127,12 @@ Options: --regression-markdown Write regression comparison Markdown. --experiment Filter history entries by experiment name. --limit Limit history list output. + --seed Seed passed to environment.reset(seed). + --max-steps Maximum environment episode steps. + --policy-id Policy id copied onto the episode result. + --trajectory-id Trajectory id used for exported reports. + --metadata-json Metadata object merged into the episode options. + --offline-records Print offline policy record count. -h, --help Show this help message.`; export function parseCliArgs(args: string[]): ParseCliArgsResult { @@ -109,6 +144,10 @@ export function parseCliArgs(args: string[]): ParseCliArgsResult { return { ok: false, message: usage, exitCode: 0 }; } + if (args[0] === "environment") { + return parseEnvironmentCommand(args); + } + if (args[0] !== "eval") { return { ok: false, @@ -308,6 +347,11 @@ async function runCommand(command: CliCommand, env: ResolvedCliEnvironment): Pro return; } + if (command.kind === "environment-run") { + await runEnvironmentEpisode(command, env); + return; + } + if (command.kind !== "eval-run") { throw new Error(`Unsupported command: ${(command as { kind: string }).kind}`); } @@ -327,7 +371,7 @@ async function loadExperimentDefinition( experimentPath: string, env: ResolvedCliEnvironment, ): Promise { - const absolutePath = await resolveExistingExperimentPath(experimentPath, env); + const absolutePath = await resolveExistingModulePath(experimentPath, env); if (absolutePath === null) { throw new Error(`Experiment file not found: ${experimentPath}`); } @@ -349,17 +393,17 @@ async function loadExperimentDefinition( return definition; } -async function resolveExistingExperimentPath( - experimentPath: string, +async function resolveExistingModulePath( + modulePath: string, env: ResolvedCliEnvironment, ): Promise { - if (isAbsolute(experimentPath)) { - return (await env.fileExists(experimentPath)) ? experimentPath : null; + if (isAbsolute(modulePath)) { + return (await env.fileExists(modulePath)) ? modulePath : null; } let currentDirectory = env.cwd; while (true) { - const candidate = resolve(currentDirectory, experimentPath); + const candidate = resolve(currentDirectory, modulePath); if (await env.fileExists(candidate)) return candidate; const parentDirectory = dirname(currentDirectory); @@ -378,6 +422,46 @@ function readExperimentDefinition(module: unknown): ExperimentDefinition | null return candidate as unknown as ExperimentDefinition; } +async function loadEnvironmentEpisodeDefinition( + episodePath: string, + env: ResolvedCliEnvironment, +): Promise { + const absolutePath = await resolveExistingModulePath(episodePath, env); + if (absolutePath === null) { + throw new Error(`Environment episode file not found: ${episodePath}`); + } + + let module: unknown; + try { + module = await env.importModule(pathToFileURL(absolutePath).href); + } catch (error) { + throw new Error( + `Failed to load environment episode file ${episodePath}: ${errorMessage(error)}`, + ); + } + + const definition = readEnvironmentEpisodeDefinition(module); + if (definition === null) { + throw new Error( + `Environment episode file must default export an EnvironmentEpisodeDefinition from defineEnvironmentEpisode(): ${episodePath}`, + ); + } + + return definition; +} + +function readEnvironmentEpisodeDefinition(module: unknown): EnvironmentEpisodeDefinition | null { + if (!isRecord(module)) return null; + const candidate = module.default; + if (!isRecord(candidate)) return null; + if (candidate.kind !== "ignition.environment-episode-definition") return null; + if (typeof candidate.name !== "string") return null; + if (typeof candidate.createEnvironment !== "function") return null; + if (typeof candidate.createPolicy !== "function") return null; + if (typeof candidate.run !== "function") return null; + return candidate as unknown as EnvironmentEpisodeDefinition; +} + function printExperimentSummary( result: ExperimentResult, definition: ExperimentDefinition, @@ -575,6 +659,80 @@ async function printHistoryEntry( } } +async function runEnvironmentEpisode( + command: EnvironmentRunCommand, + env: ResolvedCliEnvironment, +): Promise { + const definition = await loadEnvironmentEpisodeDefinition(command.episodePath, env); + const episodeOptions: RunEpisodeOptions = {}; + if (command.seed !== undefined) episodeOptions.seed = command.seed; + if (command.maxSteps !== undefined) episodeOptions.maxSteps = command.maxSteps; + if (command.policyId !== undefined) episodeOptions.policyId = command.policyId; + if (command.metadata !== undefined) episodeOptions.metadata = command.metadata; + + const episode = await definition.run(episodeOptions); + const trajectoryId = command.trajectoryId ?? definition.name; + const trajectory = recordEpisodeTrajectory(episode, { + id: trajectoryId, + metadata: { + source: "cli", + episodePath: command.episodePath, + ...(command.metadata ?? {}), + }, + }); + const summary = summarizeTrajectory(trajectory); + const report = exportTrajectoryReport(trajectory); + const offlineRecords = createOfflinePolicyRecordsFromTrajectories([trajectory], { + experimentName: definition.name, + }); + + printEnvironmentEpisodeSummary(definition.name, episode, summary, env.stdout); + + if (command.printOfflineRecords === true) { + env.stdout(`Offline records: ${offlineRecords.length}`); + } + + if (command.jsonOutputPath !== undefined) { + await writeReport(command.jsonOutputPath, `${JSON.stringify(report, null, 2)}\n`, env); + env.stdout(`Trajectory JSON report: ${command.jsonOutputPath}`); + } + + if (command.markdownOutputPath !== undefined) { + await writeReport(command.markdownOutputPath, toMarkdownTrajectoryReport(report), env); + env.stdout(`Trajectory Markdown report: ${command.markdownOutputPath}`); + } +} + +function printEnvironmentEpisodeSummary( + name: string, + episode: Awaited>, + summary: ReturnType, + write: (line: string) => void, +): void { + write(`Environment episode: ${name}`); + write(`Policy: ${episode.policyId ?? "n/a"}`); + write(`Done: ${episode.done ? "yes" : "no"}`); + write(""); + write("Steps:"); + if (episode.steps.length === 0) { + write("No steps were recorded."); + } else { + for (const [index, step] of episode.steps.entries()) { + const reward = step.reward.score * step.reward.weight; + write( + `${index + 1}. ${step.action.name} - reward ${reward.toFixed(3)} - done ${ + step.done ? "yes" : "no" + }`, + ); + } + } + write(""); + write(`Total reward: ${episode.totalReward.toFixed(3)}`); + write(`Average reward: ${episode.averageReward.toFixed(3)}`); + write(`Trajectory: ${summary.trajectoryId}`); + write(`Trajectory steps: ${summary.stepCount}`); +} + async function writeReport( outputPath: string, contents: string, @@ -676,6 +834,112 @@ function parseHistoryCommand(args: string[]): ParseCliArgsResult { return { ok: true, command }; } +function parseEnvironmentCommand(args: string[]): ParseCliArgsResult { + if (args[1] !== "run") { + return { + ok: false, + message: + "Missing environment action. Expected: ignition-agent-trainer environment run ", + exitCode: 1, + showUsage: true, + }; + } + + const episodePath = args[2]; + if (episodePath === undefined || episodePath.startsWith("-")) { + return { + ok: false, + message: + "Missing episode path. Expected: ignition-agent-trainer environment run ", + exitCode: 1, + showUsage: true, + }; + } + + const command: EnvironmentRunCommand = { + kind: "environment-run", + episodePath, + }; + + for (let index = 3; index < args.length; index += 1) { + const arg = args[index]; + if (arg === undefined) break; + + if (arg === "--seed") { + const parsed = parseIntegerOption(args[index + 1], "--seed", { min: 0 }); + if (!parsed.ok) return parsed; + command.seed = parsed.value; + index += 1; + continue; + } + + if (arg === "--max-steps") { + const parsed = parseIntegerOption(args[index + 1], "--max-steps"); + if (!parsed.ok) return parsed; + command.maxSteps = parsed.value; + index += 1; + continue; + } + + if (arg === "--policy-id") { + const value = args[index + 1]; + if (value === undefined || value.startsWith("-")) { + return { ok: false, message: "Missing value for --policy-id.", exitCode: 1 }; + } + command.policyId = value; + index += 1; + continue; + } + + if (arg === "--trajectory-id") { + const value = args[index + 1]; + if (value === undefined || value.startsWith("-")) { + return { ok: false, message: "Missing value for --trajectory-id.", exitCode: 1 }; + } + command.trajectoryId = value; + index += 1; + continue; + } + + if (arg === "--metadata-json") { + const parsed = parseMetadataJsonOption(args[index + 1], "--metadata-json"); + if (!parsed.ok) return parsed; + command.metadata = parsed.value; + index += 1; + continue; + } + + if (arg === "--json") { + const value = args[index + 1]; + if (value === undefined || value.startsWith("-")) { + return { ok: false, message: "Missing value for --json.", exitCode: 1 }; + } + command.jsonOutputPath = value; + index += 1; + continue; + } + + if (arg === "--markdown") { + const value = args[index + 1]; + if (value === undefined || value.startsWith("-")) { + return { ok: false, message: "Missing value for --markdown.", exitCode: 1 }; + } + command.markdownOutputPath = value; + index += 1; + continue; + } + + if (arg === "--offline-records") { + command.printOfflineRecords = true; + continue; + } + + return { ok: false, message: `Unknown option: ${arg}`, exitCode: 1 }; + } + + return { ok: true, command }; +} + function validateEvalRunCommand(command: EvalRunCommand): ParseCliArgsResult | null { if (command.recordHistory === true && command.historyPath === undefined) { return { ok: false, message: "--record-history requires --history.", exitCode: 1 }; @@ -709,15 +973,39 @@ function parseNumberOption( function parseIntegerOption( value: string | undefined, optionName: string, + options: { min?: number } = {}, ): { ok: true; value: number } | { ok: false; message: string; exitCode: number } { const parsed = parseNumberOption(value, optionName); if (!parsed.ok) return parsed; - if (!Number.isInteger(parsed.value) || parsed.value < 1) { + const min = options.min ?? 1; + if (!Number.isInteger(parsed.value) || parsed.value < min) { return { ok: false, message: `Invalid value for ${optionName}: ${value}`, exitCode: 1 }; } return parsed; } +function parseMetadataJsonOption( + value: string | undefined, + optionName: string, +): { ok: true; value: Metadata } | { ok: false; message: string; exitCode: number } { + if (value === undefined || value.startsWith("-")) { + return { ok: false, message: `Missing value for ${optionName}.`, exitCode: 1 }; + } + try { + const parsed = JSON.parse(value) as unknown; + if (!isRecord(parsed) || Array.isArray(parsed)) { + return { ok: false, message: `${optionName} must be a JSON object.`, exitCode: 1 }; + } + return { ok: true, value: parsed }; + } catch (error) { + return { + ok: false, + message: `Invalid JSON for ${optionName}: ${errorMessage(error)}`, + exitCode: 1, + }; + } +} + function selectHistoryEntry( entries: readonly ExperimentHistoryEntry[], selector: string, diff --git a/packages/environment/README.md b/packages/environment/README.md index 6c3213b..97b87c0 100644 --- a/packages/environment/README.md +++ b/packages/environment/README.md @@ -7,7 +7,7 @@ Use this package when modeling deterministic agent strategy episodes before deep ## Current API ```ts -import { runEpisode } from "@ignitionai/agent-trainer-environment"; +import { defineEnvironmentEpisode, runEpisode } from "@ignitionai/agent-trainer-environment"; ``` Main exports: @@ -15,7 +15,8 @@ Main exports: - environment types: `AgentEnvironment`, `EnvironmentState`, `EnvironmentAction`, `EnvironmentStepResult`, - policy type: `Policy`, - episode types: `EpisodeStep`, `EpisodeResult`, -- runner helper: `runEpisode`. +- runner helper: `runEpisode`, +- reusable module helper: `defineEnvironmentEpisode`. `runEpisode(environment, policy, options)` supports: @@ -43,6 +44,21 @@ search -> rerank -> verify -> answer That example records an episode trajectory through `@ignitionai/agent-trainer-rl` and converts it into offline policy records. +Episode modules can be exported for the CLI: + +```ts +export default defineEnvironmentEpisode({ + name: "rag-environment-episode", + environment: () => environment, + policy: () => policy, + options: { + seed: 7, + maxSteps: 10, + policyId: "scripted-rag-policy", + }, +}); +``` + ## Alpha Readiness Status This package is partial alpha-level. @@ -52,7 +68,7 @@ Known gaps: - no production environment implementation, - no durable rollout store, - no automatic policy optimization loop, -- no CLI integration for environment episodes. +- CLI integration is local-only and loads deterministic episode modules. ## Non-goals diff --git a/packages/environment/src/environment.test.ts b/packages/environment/src/environment.test.ts index 01b507e..a80f660 100644 --- a/packages/environment/src/environment.test.ts +++ b/packages/environment/src/environment.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from "vitest"; import { type AgentEnvironment, + defineEnvironmentEpisode, type EnvironmentAction, type EnvironmentState, type Policy, @@ -118,6 +119,64 @@ describe("runEpisode", () => { "Environment reward score must be finite for action answer.", ); }); + + it("defines reusable environment episode modules with overridable options", async () => { + const definition = defineEnvironmentEpisode({ + name: "scripted-environment", + environment: () => + new ScriptedEnvironment([ + { + action: "answer", + score: 1, + weight: 1, + nextState: "answered", + done: true, + }, + ]), + policy: () => new OrderedPolicy(), + options: { + seed: 1, + policyId: "default-policy", + metadata: { defaultOption: true }, + }, + metadata: { source: "unit-test" }, + }); + + const episode = await definition.run({ + seed: 2, + policyId: "override-policy", + metadata: { overrideOption: true }, + }); + + expect(definition.kind).toBe("ignition.environment-episode-definition"); + expect(episode.policyId).toBe("override-policy"); + expect(episode.metadata).toEqual({ + source: "unit-test", + defaultOption: true, + overrideOption: true, + }); + expect(episode.finalState.id).toBe("answered"); + }); + + it("rejects invalid environment episode definitions clearly", async () => { + expect(() => + defineEnvironmentEpisode({ + name: " ", + environment: new ScriptedEnvironment([]), + policy: new OrderedPolicy(), + }), + ).toThrow("Environment episode name is required."); + + const definition = defineEnvironmentEpisode({ + name: "bad-environment", + environment: () => ({}) as AgentEnvironment, + policy: new OrderedPolicy(), + }); + + await expect(Promise.resolve().then(() => definition.run())).rejects.toThrow( + "Environment episode environment must implement reset, actions and step.", + ); + }); }); interface ScriptedTransition { diff --git a/packages/environment/src/environment.ts b/packages/environment/src/environment.ts index ad2f41d..bc7cb3b 100644 --- a/packages/environment/src/environment.ts +++ b/packages/environment/src/environment.ts @@ -28,6 +28,24 @@ export interface Policy { chooseAction(state: EnvironmentState, actions: EnvironmentAction[]): Promise; } +export type EnvironmentFactory = AgentEnvironment | (() => AgentEnvironment); +export type PolicyFactory = Policy | (() => Policy); + +export interface EnvironmentEpisodeConfig { + name: string; + environment: EnvironmentFactory; + policy: PolicyFactory; + options?: RunEpisodeOptions; + metadata?: Metadata; +} + +export interface EnvironmentEpisodeDefinition extends EnvironmentEpisodeConfig { + readonly kind: "ignition.environment-episode-definition"; + createEnvironment(): AgentEnvironment; + createPolicy(): Policy; + run(options?: RunEpisodeOptions): Promise; +} + export interface EpisodeStep { state: EnvironmentState; action: EnvironmentAction; @@ -54,6 +72,30 @@ export interface RunEpisodeOptions { metadata?: Metadata; } +export function defineEnvironmentEpisode( + config: EnvironmentEpisodeConfig, +): EnvironmentEpisodeDefinition { + validateEnvironmentEpisodeConfig(config); + + return { + ...config, + kind: "ignition.environment-episode-definition", + createEnvironment() { + return resolveEnvironment(config.environment); + }, + createPolicy() { + return resolvePolicy(config.policy); + }, + run(options = {}) { + return runEpisode( + this.createEnvironment(), + this.createPolicy(), + mergeOptions(config, options), + ); + }, + }; +} + export async function runEpisode( environment: AgentEnvironment, policy: Policy, @@ -110,6 +152,48 @@ function validateMaxSteps(maxSteps: number): void { } } +function validateEnvironmentEpisodeConfig(config: EnvironmentEpisodeConfig): void { + if (!config.name.trim()) { + throw new Error("Environment episode name is required."); + } +} + +function resolveEnvironment(value: EnvironmentFactory): AgentEnvironment { + const environment = typeof value === "function" ? value() : value; + if ( + typeof environment.reset !== "function" || + typeof environment.actions !== "function" || + typeof environment.step !== "function" + ) { + throw new Error("Environment episode environment must implement reset, actions and step."); + } + return environment; +} + +function resolvePolicy(value: PolicyFactory): Policy { + const policy = typeof value === "function" ? value() : value; + if (typeof policy.chooseAction !== "function") { + throw new Error("Environment episode policy must implement chooseAction."); + } + return policy; +} + +function mergeOptions( + config: EnvironmentEpisodeConfig, + options: RunEpisodeOptions, +): RunEpisodeOptions { + const metadata = { + ...(config.metadata ?? {}), + ...(config.options?.metadata ?? {}), + ...(options.metadata ?? {}), + }; + const merged = { + ...(config.options ?? {}), + ...options, + }; + return Object.keys(metadata).length === 0 ? merged : { ...merged, metadata }; +} + function validateReward(reward: RewardResult, action: EnvironmentAction): void { if (!Number.isFinite(reward.score)) { throw new Error(`Environment reward score must be finite for action ${action.name}.`); diff --git a/packages/rl/README.md b/packages/rl/README.md index 89735f9..d0b8b96 100644 --- a/packages/rl/README.md +++ b/packages/rl/README.md @@ -69,6 +69,15 @@ See `examples/rag-environment-episode` for a complete mocked RAG episode that ru search -> rerank -> verify -> answer ``` +The same episode module can be run through the local CLI: + +```bash +bun run --filter '@ignitionai/agent-trainer-cli' dev -- environment run ./examples/rag-environment-episode/src/index.ts \ + --json reports/rag-trajectory.json \ + --markdown reports/rag-trajectory.md \ + --offline-records +``` + ## Experimental Fixed-Strategy Bandit Use `ExperimentalBanditStrategySelector` to choose among fixed, developer-supplied strategies and update their rewards from observed experiment outcomes.