From 372fd66a883e87079f9ae7be9f26b2987da2d538 Mon Sep 17 00:00:00 2001 From: salimlaimeche Date: Wed, 1 Jul 2026 00:35:43 +0200 Subject: [PATCH] feat(core): add runtime validation helpers --- docs/ALPHA_READINESS.md | 2 +- docs/BACKLOG.md | 193 ++++++++++++++++- docs/PROJECT_AUDIT.md | 4 +- packages/core/README.md | 7 +- packages/core/src/adapter.ts | 8 +- packages/core/src/dataset.ts | 23 +- packages/core/src/index.test.ts | 86 ++++++++ packages/core/src/index.ts | 1 + packages/core/src/validation.ts | 357 ++++++++++++++++++++++++++++++++ 9 files changed, 650 insertions(+), 31 deletions(-) create mode 100644 packages/core/src/validation.ts diff --git a/docs/ALPHA_READINESS.md b/docs/ALPHA_READINESS.md index 2a9807d..09c07f0 100644 --- a/docs/ALPHA_READINESS.md +++ b/docs/ALPHA_READINESS.md @@ -28,7 +28,7 @@ All packages declare `license: MIT`, matching the root `LICENSE` file. | `@ignitionai/agent-trainer-adapter-mastra` | ready | ready | ready | ready | partial | Structural adapter with mocked example coverage; no memory, tool or full Mastra coverage. | | `@ignitionai/agent-trainer-adapter-vercel-ai` | ready | ready | ready | ready | partial | Structural adapter with mocked example coverage; no streaming, tools or live provider calls. | | `@ignitionai/agent-trainer-cli` | ready | ready | ready | ready | partial | Runs typed experiments, writes reports/bundles, records local history, selects baselines and runs regression checks; no watch mode or remote execution. | -| `@ignitionai/agent-trainer-core` | ready | ready | ready | ready | partial | Foundational helpers have dedicated tests; runtime schema validation remains outside the current helper surface. | +| `@ignitionai/agent-trainer-core` | ready | ready | ready | ready | partial | Foundational helpers and runtime validation helpers have dedicated tests; full experiment report schema validation remains outside the current helper surface. | | `@ignitionai/agent-trainer-environment` | ready | ready | ready | ready | partial | Tested episode runner with safety guards and a deterministic RAG episode example; no production runtime or optimization loop. | | `@ignitionai/agent-trainer-evals` | ready | ready | ready | ready | partial | Current rewards are tested; RAG presets and richer scoring are still missing. | | `@ignitionai/agent-trainer-experiments` | ready | ready | ready | ready | ready | Local runner, definitions, gates and JSONL history are tested and documented. | diff --git a/docs/BACKLOG.md b/docs/BACKLOG.md index 71dc72e..d47dd21 100644 --- a/docs/BACKLOG.md +++ b/docs/BACKLOG.md @@ -11,9 +11,9 @@ Status values: Current snapshot: - Completed through PR #47. -- No unblocked implementation PR is currently selected. -- PR #48 is the next RL/product loop, but it stays blocked until dogfood or representative trajectory fixtures exist. -- Useful unblocked maintenance can still happen as small focused PRs for docs, publish automation decisions or issues found while dogfooding. +- PR #48 is the current unblocked core validation PR. +- PR #49 and PR #50 are the next planned release-policy and CLI/environment ergonomics PRs. +- The dogfood-driven policy optimization loop stays blocked until dogfood or representative trajectory fixtures exist. ## Stable PR sequence @@ -2972,9 +2972,192 @@ Definition of done: Next PR: -- PR #48 - `feat: add dogfood-driven policy optimization loop` +- PR #48 - `feat(core): add runtime validation helpers` -### PR #48 - `feat: add dogfood-driven policy optimization loop` +### PR #48 - `feat(core): add runtime validation helpers` + +Status: + +- current + +Branch: + +```txt +feat/core-runtime-validation-helpers +``` + +Goal: + +Move `@ignitionai/agent-trainer-core` closer to alpha-stable status by validating core runtime shapes. + +Scope: + +- add runtime assertions and non-throwing validators for datasets, dataset items, variants, adapters, run results, usage metrics, traces, metric results, reward results, normalized scores and JSON-compatible fields, +- wire high-risk core entry points through the validators where compatible, +- document the validation surface in `packages/core/README.md`, +- update audit/readiness docs to remove the stale "no runtime schema validation" limitation for the covered core surface, +- add focused tests for positive and failure paths. + +Out of scope: + +- external schema libraries, +- full experiment report validation, +- public API redesign, +- provider calls, +- database or hosted validation services. + +Required APIs / files: + +- `packages/core/src/validation.ts`, +- `packages/core/src/dataset.ts`, +- `packages/core/src/adapter.ts`, +- `packages/core/src/index.ts`, +- `packages/core/src/index.test.ts`, +- `packages/core/README.md`, +- alpha readiness and project audit docs. + +Acceptance: + +```bash +bun run lint +bun run typecheck +bun run test +bun run build +bun run pack:check +``` + +Definition of done: + +- developers can validate or assert common core runtime values without adding dependencies, +- invalid usage, traces, scores and serialized JSON fields fail with clear errors, +- existing public helpers keep working, +- docs explain the validation limits. + +Next PR: + +- PR #49 - `docs(release): decide npm publish automation policy` + +### PR #49 - `docs(release): decide npm publish automation policy` + +Status: + +- planned + +Branch: + +```txt +docs/npm-publish-automation-policy +``` + +Goal: + +Make the alpha npm publishing policy explicit before any future public release automation. + +Scope: + +- decide manual vs GitHub Actions publication policy, +- document dist-tag policy, especially `alpha` and no accidental `latest`, +- document npm provenance posture, +- document OTP/2FA and org permission expectations, +- document what must pass before any publish attempt. + +Out of scope: + +- actually publishing packages, +- introducing automatic npm publish in this PR, +- changing package names or versions unless required by the policy doc, +- release dashboard or hosted workflow. + +Required APIs / files: + +- `docs/NPM_ALPHA_PUBLISHING.md`, +- `docs/ALPHA_RELEASE.md`, +- `docs/CODEX_RUNBOOK.md` if runbook steps change, +- alpha readiness/backlog docs. + +Acceptance: + +```bash +bun run lint +bun run typecheck +bun run test +bun run build +bun run pack:check +``` + +Definition of done: + +- the repo clearly says whether alpha publishing is manual or automated, +- npm tag, provenance, OTP and permission rules are explicit, +- no workflow can publish accidentally. + +Next PR: + +- PR #50 - `feat(cli): add environment episode trajectory commands` + +### PR #50 - `feat(cli): add environment episode trajectory commands` + +Status: + +- planned + +Branch: + +```txt +feat/cli-environment-trajectory-commands +``` + +Goal: + +Expose environment episode and trajectory report ergonomics through the CLI without adding model training. + +Scope: + +- add a CLI command that loads a deterministic environment episode module, +- run `runEpisode()` with seed, max steps, policy id and metadata options, +- print episode steps, total reward and trajectory summary, +- optionally write JSON and Markdown trajectory reports, +- optionally print offline policy record counts, +- add focused CLI parser/runtime tests and docs. + +Out of scope: + +- PPO, GRPO or neural policy training, +- live provider calls, +- production routing, +- database or hosted trajectory store, +- policy optimization loop. + +Required APIs / files: + +- `packages/cli`, +- `packages/environment`, +- `packages/rl`, +- `examples/rag-environment-episode` if a reusable module export is needed, +- CLI/environment/rl READMEs and audit docs. + +Acceptance: + +```bash +bun run lint +bun run typecheck +bun run test +bun run build +bun run pack:check +``` + +Definition of done: + +- a developer can run an episode module from the CLI, +- JSON and Markdown trajectory reports are writable from CLI flags, +- offline record count is visible, +- docs explicitly say this is not a training loop. + +Next PR: + +- PR #51 - `feat: add dogfood-driven policy optimization loop` + +### PR #51 - `feat: add dogfood-driven policy optimization loop` Status: diff --git a/docs/PROJECT_AUDIT.md b/docs/PROJECT_AUDIT.md index f25bb84..eabc28d 100644 --- a/docs/PROJECT_AUDIT.md +++ b/docs/PROJECT_AUDIT.md @@ -229,7 +229,7 @@ If a package exists but is intentionally narrow, minimal or untested, it is part | Capability | Status | Package/File | Stable? | Notes | |---|---|---|---|---| -| core primitives | partial | `packages/core` | No | Core types and helpers have dedicated tests; runtime schema validation remains out of scope. | +| core primitives | partial | `packages/core` | No | Core types, helpers and runtime validation helpers have dedicated tests; full experiment report schema validation remains out of scope. | | IgnitionRAG adapter contract | partial | `packages/adapter-ignitionrag` | No | Type-level contract only; no runtime IgnitionRAG integration. | | evals/rewards | partial | `packages/evals` | No | Tests and README exist, but reward set is intentionally small. | | RAG presets | partial | `packages/preset-rag` | No | Deterministic presets compose text, citation, latency, cost and tool-use rewards. | @@ -267,7 +267,7 @@ If a package exists but is intentionally narrow, minimal or untested, it is part - No real LLM calls by default. - No hosted IgnitionRAG integration code. - IgnitionRAG integration is design-only. -- Core still needs runtime schema validation before alpha-stable status. +- Core still needs full experiment report schema validation before alpha-stable status. - Lightweight policy optimization remains blocked until dogfood or representative trajectory fixtures exist. - Environment episodes still need real dogfood data and CLI ergonomics before stable status. - Ecosystem adapters are minimal and structural. diff --git a/packages/core/README.md b/packages/core/README.md index 389555b..f3dc79b 100644 --- a/packages/core/README.md +++ b/packages/core/README.md @@ -8,10 +8,12 @@ Use this package when defining datasets, agent adapters, traces, usage metrics, ```ts import { + assertRunResult, createDataset, createMockAdapter, normalizeRunResult, toAgentInput, + validateDataset, weightedAverage, } from "@ignitionai/agent-trainer-core"; ``` @@ -19,17 +21,20 @@ import { Main exports: - dataset helpers: `createDataset`, `assertDatasetItem`, +- runtime validation helpers: `assertDataset`, `validateDataset`, `assertAgentVariant`, `validateAgentVariant`, `assertRunResult`, `validateRunResult`, `assertUsageMetrics`, `validateUsageMetrics`, `assertTrace`, `validateTrace`, `assertMetricResult`, `validateMetricResult`, `assertRewardResult`, `assertNormalizedScore`, `assertJsonValue`, - adapter helpers: `createMockAdapter`, `normalizeRunResult`, `toAgentInput`, - score helpers: `clampScore`, `weightedAverage`, - shared types for datasets, adapters, traces, rewards, cases, leaderboards and experiment reports. +The `assert*` helpers throw clear errors and narrow TypeScript types. The `validate*` helpers return `{ ok: true, value }` or `{ ok: false, error }` when callers need non-throwing validation. + ## Alpha Readiness Status This package is foundational and covered by dedicated package-level tests for the current helper surface. Known gaps: -- no runtime schema validation for serialized reports, +- runtime validation covers core datasets, variants, run results, usage, traces and scores; full serialized experiment report schema validation remains outside the current helper surface, - no compatibility policy beyond the current monorepo usage. ## Non-goals diff --git a/packages/core/src/adapter.ts b/packages/core/src/adapter.ts index 1c2076b..2a35181 100644 --- a/packages/core/src/adapter.ts +++ b/packages/core/src/adapter.ts @@ -11,6 +11,7 @@ import type { Trace, UsageMetrics, } from "./types"; +import { assertRunResult, assertUsageMetrics } from "./validation"; export type MockAdapterHandler = | AgentAdapterResult @@ -26,6 +27,9 @@ export function createMockAdapter( handler: MockAdapterHandler, options: MockAdapterOptions = {}, ): AgentAdapter { + if (options.trace !== undefined) assertRunResult({ output: "", trace: options.trace }); + if (options.usage !== undefined) assertUsageMetrics(options.usage); + const adapter: AgentAdapter = { async run(input, context) { const value = typeof handler === "function" ? await handler(input, context) : handler; @@ -56,12 +60,14 @@ export function toAgentInput(item: DatasetItem): AgentInput { export function normalizeRunResult(value: AgentAdapterResult): RunResult { if (isRunResultLike(value)) { - return { + const result = { output: value.output, trace: value.trace ?? { steps: [] }, ...(value.usage !== undefined ? { usage: value.usage } : {}), ...(value.metadata !== undefined ? { metadata: value.metadata } : {}), }; + assertRunResult(result); + return result; } return { diff --git a/packages/core/src/dataset.ts b/packages/core/src/dataset.ts index ae71327..983c766 100644 --- a/packages/core/src/dataset.ts +++ b/packages/core/src/dataset.ts @@ -1,31 +1,12 @@ import type { Dataset, DatasetItem } from "./types"; +import { assertDataset } from "./validation"; export function createDataset(items: DatasetItem[]): Dataset; export function createDataset(input: Dataset): Dataset; export function createDataset(input: Dataset | DatasetItem[]): Dataset { const dataset = Array.isArray(input) ? { name: "dataset", items: input } : input; - if (!dataset.name.trim()) { - throw new Error("Dataset name is required."); - } - - const seen = new Set(); - for (const item of dataset.items) { - assertDatasetItem(item); - if (seen.has(item.id)) { - throw new Error(`Duplicate dataset item id: ${item.id}`); - } - seen.add(item.id); - } + assertDataset(dataset); return dataset; } - -export function assertDatasetItem(item: DatasetItem): void { - if (!item.id.trim()) { - throw new Error("Dataset item id is required."); - } - if (!item.input.trim()) { - throw new Error(`Dataset item ${item.id} input is required.`); - } -} diff --git a/packages/core/src/index.test.ts b/packages/core/src/index.test.ts index 20e2feb..610d0fe 100644 --- a/packages/core/src/index.test.ts +++ b/packages/core/src/index.test.ts @@ -1,11 +1,21 @@ import { describe, expect, it } from "vitest"; import { + assertAgentVariant, assertDatasetItem, + assertJsonValue, + assertMetricResult, + assertNormalizedScore, + assertRunResult, + assertTrace, + assertUsageMetrics, clampScore, createDataset, createMockAdapter, normalizeRunResult, toAgentInput, + validateAgentVariant, + validateDataset, + validateRunResult, weightedAverage, } from "./index"; @@ -62,6 +72,28 @@ describe("@ignitionai/agent-trainer-core", () => { "Dataset item case-1 input is required.", ); }); + + it("validates dataset shapes without throwing when requested", () => { + const valid = validateDataset({ + name: "runtime-dataset", + items: [{ id: "case-1", input: "Question?", expected: { json: { ok: true } } }], + }); + + expect(valid.ok).toBe(true); + if (!valid.ok) return; + expect(valid.value.items[0]?.id).toBe("case-1"); + + const invalid = validateDataset({ + name: "runtime-dataset", + items: [{ id: "case-1", input: "Question?", expected: { json: { bad: undefined } } }], + }); + + expect(invalid.ok).toBe(false); + if (invalid.ok) return; + expect(invalid.error.message).toContain( + "Dataset item 0 expected json.bad must be JSON-compatible.", + ); + }); }); describe("agent adapters", () => { @@ -151,6 +183,42 @@ describe("@ignitionai/agent-trainer-core", () => { usage: { inputTokens: 4, outputTokens: 3 }, }); }); + + it("validates variants and run results at runtime", () => { + const adapter = createMockAdapter("answer"); + + expect(() => assertAgentVariant({ name: "valid-agent", adapter })).not.toThrow(); + expect(validateAgentVariant({ name: "broken-agent" })).toMatchObject({ + ok: false, + error: { message: "Agent variant requires an adapter or run function." }, + }); + + const validRun = validateRunResult({ + output: "answer", + trace: { steps: [{ type: "message", role: "assistant", content: "" }] }, + usage: { inputTokens: 1, outputTokens: 2, totalTokens: 3, latencyMs: 10, costUsd: 0 }, + }); + + expect(validRun.ok).toBe(true); + expect(() => + assertRunResult({ + output: "answer", + trace: { steps: [{ type: "decision", action: "answer", confidence: 1.2 }] }, + }), + ).toThrow("Trace step 0 confidence must be between 0 and 1."); + }); + + it("rejects unsafe trace and usage metrics", () => { + expect(() => assertTrace({ steps: [{ type: "tool_call", name: "" }] })).toThrow( + "Trace step 0 name is required.", + ); + expect(() => assertUsageMetrics({ inputTokens: 1.5 })).toThrow( + "Usage metrics inputTokens must be a non-negative integer.", + ); + expect(() => createMockAdapter("answer", { usage: { latencyMs: -1 } })).toThrow( + "Usage metrics latencyMs must be a non-negative finite number.", + ); + }); }); describe("scores", () => { @@ -174,5 +242,23 @@ describe("@ignitionai/agent-trainer-core", () => { it("returns zero when total score weight is zero", () => { expect(weightedAverage([{ name: "disabled", score: 1, weight: 0 }])).toBe(0); }); + + it("asserts normalized score and metric result boundaries", () => { + expect(() => assertNormalizedScore(1, "Reward score")).not.toThrow(); + expect(() => assertNormalizedScore(1.1, "Reward score")).toThrow( + "Reward score must be between 0 and 1.", + ); + expect(() => assertMetricResult({ name: "quality", score: 0.8, weight: 1 })).not.toThrow(); + expect(() => assertMetricResult({ name: "quality", score: Number.NaN })).toThrow( + "Metric result score must be a finite number.", + ); + }); + + it("asserts JSON-compatible values for serialized fields", () => { + expect(() => assertJsonValue({ ok: true, nested: [1, "two", null] })).not.toThrow(); + expect(() => assertJsonValue({ bad: Number.POSITIVE_INFINITY })).toThrow( + "JSON value.bad number must be finite.", + ); + }); }); }); diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 2cab926..a64b4bb 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -2,3 +2,4 @@ export * from "./adapter"; export * from "./dataset"; export * from "./score"; export * from "./types"; +export * from "./validation"; diff --git a/packages/core/src/validation.ts b/packages/core/src/validation.ts new file mode 100644 index 0000000..42aed9c --- /dev/null +++ b/packages/core/src/validation.ts @@ -0,0 +1,357 @@ +import type { + AgentAdapter, + AgentVariant, + Dataset, + DatasetItem, + ExpectedOutput, + JsonValue, + Metadata, + MetricResult, + RewardResult, + RunResult, + Trace, + TraceStep, + UsageMetrics, +} from "./types"; + +export type RuntimeValidationResult = { ok: true; value: T } | { ok: false; error: Error }; + +export function validateDataset(value: unknown): RuntimeValidationResult { + return captureValidation(value, assertDataset); +} + +export function validateAgentVariant(value: unknown): RuntimeValidationResult { + return captureValidation(value, assertAgentVariant); +} + +export function validateRunResult(value: unknown): RuntimeValidationResult { + return captureValidation(value, assertRunResult); +} + +export function validateUsageMetrics(value: unknown): RuntimeValidationResult { + return captureValidation(value, assertUsageMetrics); +} + +export function validateTrace(value: unknown): RuntimeValidationResult { + return captureValidation(value, assertTrace); +} + +export function validateMetricResult(value: unknown): RuntimeValidationResult { + return captureValidation(value, assertMetricResult); +} + +export function assertDataset(value: unknown): asserts value is Dataset { + assertRecord(value, "Dataset"); + assertRequiredString(value.name, "Dataset name"); + assertOptionalString(value.description, "Dataset description"); + assertOptionalMetadata(value.metadata, "Dataset metadata"); + + if (!Array.isArray(value.items)) { + throw new Error("Dataset items must be an array."); + } + + const seen = new Set(); + for (const [index, item] of value.items.entries()) { + assertDatasetItem(item, `Dataset item ${index}`); + if (seen.has(item.id)) { + throw new Error(`Duplicate dataset item id: ${item.id}`); + } + seen.add(item.id); + } +} + +export function assertDatasetItem( + value: unknown, + path = "Dataset item", +): asserts value is DatasetItem { + assertRecord(value, path); + assertRequiredString(value.id, `${path} id`); + assertRequiredString(value.input, `${path} ${String(value.id)} input`); + assertExpectedOutput(value.expected, `${path} expected`); + assertOptionalMetadata(value.metadata, `${path} metadata`); +} + +export function assertAgentVariant( + value: unknown, + path = "Agent variant", +): asserts value is AgentVariant { + assertRecord(value, path); + assertOptionalString(value.id, `${path} id`, { allowBlank: false }); + assertRequiredString(value.name, `${path} name`); + assertOptionalString(value.description, `${path} description`); + assertOptionalMetadata(value.config, `${path} config`); + + if (value.adapter !== undefined) { + assertAgentAdapter(value.adapter, `${path} adapter`); + } + if (value.run !== undefined && typeof value.run !== "function") { + throw new Error(`${path} run must be a function.`); + } + if (value.adapter === undefined && value.run === undefined) { + throw new Error(`${path} requires an adapter or run function.`); + } +} + +export function assertAgentAdapter( + value: unknown, + path = "Agent adapter", +): asserts value is AgentAdapter { + assertRecord(value, path); + assertOptionalString(value.name, `${path} name`, { allowBlank: false }); + if (typeof value.run !== "function") { + throw new Error(`${path} run must be a function.`); + } +} + +export function assertRunResult(value: unknown): asserts value is RunResult { + assertRecord(value, "Run result"); + if (!("output" in value)) { + throw new Error("Run result output is required."); + } + assertTrace(value.trace); + if (value.usage !== undefined) { + assertUsageMetrics(value.usage); + } + assertOptionalMetadata(value.metadata, "Run result metadata"); +} + +export function assertUsageMetrics(value: unknown): asserts value is UsageMetrics { + assertRecord(value, "Usage metrics"); + assertOptionalNonNegativeInteger(value.inputTokens, "Usage metrics inputTokens"); + assertOptionalNonNegativeInteger(value.outputTokens, "Usage metrics outputTokens"); + assertOptionalNonNegativeInteger(value.totalTokens, "Usage metrics totalTokens"); + assertOptionalNonNegativeNumber(value.costUsd, "Usage metrics costUsd"); + assertOptionalNonNegativeNumber(value.latencyMs, "Usage metrics latencyMs"); +} + +export function assertTrace(value: unknown): asserts value is Trace { + assertRecord(value, "Trace"); + if (!Array.isArray(value.steps)) { + throw new Error("Trace steps must be an array."); + } + for (const [index, step] of value.steps.entries()) { + assertTraceStep(step, `Trace step ${index}`); + } + assertOptionalMetadata(value.metadata, "Trace metadata"); +} + +export function assertTraceStep(value: unknown, path = "Trace step"): asserts value is TraceStep { + assertRecord(value, path); + + if (value.type === "message") { + assertMessageTraceStep(value, path); + return; + } + if (value.type === "tool_call") { + assertToolCallTraceStep(value, path); + return; + } + if (value.type === "decision") { + assertDecisionTraceStep(value, path); + return; + } + if (value.type === "custom") { + assertCustomTraceStep(value, path); + return; + } + + throw new Error(`${path} type must be one of message, tool_call, decision or custom.`); +} + +export function assertMetricResult( + value: unknown, + path = "Metric result", +): asserts value is MetricResult { + assertRecord(value, path); + assertRequiredString(value.name, `${path} name`); + assertNormalizedScore(value.score, `${path} score`); + assertOptionalBoolean(value.passed, `${path} passed`); + assertOptionalNonNegativeNumber(value.weight, `${path} weight`); + assertOptionalString(value.reason, `${path} reason`); + assertOptionalMetadata(value.metadata, `${path} metadata`); +} + +export function assertRewardResult( + value: unknown, + path = "Reward result", +): asserts value is RewardResult { + assertMetricResult(value, path); + if (!Number.isFinite(value.weight)) { + throw new Error(`${path} weight must be finite.`); + } +} + +export function assertNormalizedScore(value: unknown, path = "Score"): asserts value is number { + if (typeof value !== "number" || !Number.isFinite(value)) { + throw new Error(`${path} must be a finite number.`); + } + if (value < 0 || value > 1) { + throw new Error(`${path} must be between 0 and 1.`); + } +} + +export function assertJsonValue(value: unknown, path = "JSON value"): asserts value is JsonValue { + if (value === null) return; + if (typeof value === "string" || typeof value === "boolean") return; + if (typeof value === "number") { + if (!Number.isFinite(value)) { + throw new Error(`${path} number must be finite.`); + } + return; + } + if (Array.isArray(value)) { + for (const [index, item] of value.entries()) { + assertJsonValue(item, `${path}[${index}]`); + } + return; + } + if (isRecord(value)) { + for (const [key, item] of Object.entries(value)) { + assertJsonValue(item, `${path}.${key}`); + } + return; + } + + throw new Error(`${path} must be JSON-compatible.`); +} + +function assertExpectedOutput( + value: unknown, + path: string, +): asserts value is ExpectedOutput | undefined { + if (value === undefined) return; + assertRecord(value, path); + assertOptionalString(value.exact, `${path} exact`); + assertOptionalStringArray(value.contains, `${path} contains`); + assertOptionalStringArray(value.citations, `${path} citations`); + if (value.json !== undefined) { + assertJsonValue(value.json, `${path} json`); + } + assertOptionalMetadata(value.metadata, `${path} metadata`); +} + +function assertMessageTraceStep(value: Record, path: string): void { + const roles = new Set(["system", "user", "assistant", "tool"]); + if (typeof value.role !== "string" || !roles.has(value.role)) { + throw new Error(`${path} role must be one of system, user, assistant or tool.`); + } + assertRequiredString(value.content, `${path} content`, { allowBlank: true }); + assertOptionalMetadata(value.metadata, `${path} metadata`); +} + +function assertToolCallTraceStep(value: Record, path: string): void { + assertRequiredString(value.name, `${path} name`); + assertOptionalString(value.startedAt, `${path} startedAt`); + assertOptionalString(value.endedAt, `${path} endedAt`); + assertOptionalNonNegativeNumber(value.latencyMs, `${path} latencyMs`); + assertOptionalMetadata(value.metadata, `${path} metadata`); +} + +function assertDecisionTraceStep(value: Record, path: string): void { + assertRequiredString(value.action, `${path} action`); + assertOptionalString(value.reason, `${path} reason`); + if (value.confidence !== undefined) { + assertNormalizedScore(value.confidence, `${path} confidence`); + } + assertOptionalMetadata(value.metadata, `${path} metadata`); +} + +function assertCustomTraceStep(value: Record, path: string): void { + assertRequiredString(value.name, `${path} name`); + assertOptionalMetadata(value.metadata, `${path} metadata`); +} + +function captureValidation( + value: unknown, + assertValue: (input: unknown) => asserts input is T, +): RuntimeValidationResult { + try { + assertValue(value); + return { ok: true, value }; + } catch (error) { + return { ok: false, error: error instanceof Error ? error : new Error(String(error)) }; + } +} + +function assertRecord(value: unknown, path: string): asserts value is Record { + if (!isRecord(value)) { + throw new Error(`${path} must be an object.`); + } +} + +function assertRequiredString( + value: unknown, + path: string, + options: { allowBlank?: boolean } = {}, +): asserts value is string { + if (typeof value !== "string") { + throw new Error(`${path} is required.`); + } + if (options.allowBlank !== true && !value.trim()) { + throw new Error(`${path} is required.`); + } +} + +function assertOptionalString( + value: unknown, + path: string, + options: { allowBlank?: boolean } = {}, +): asserts value is string | undefined { + if (value === undefined) return; + if (typeof value !== "string") { + throw new Error(`${path} must be a string.`); + } + if (options.allowBlank === false && !value.trim()) { + throw new Error(`${path} must be a non-empty string.`); + } +} + +function assertOptionalStringArray( + value: unknown, + path: string, +): asserts value is string[] | undefined { + if (value === undefined) return; + if (!Array.isArray(value) || value.some((item) => typeof item !== "string")) { + throw new Error(`${path} must be an array of strings.`); + } +} + +function assertOptionalBoolean(value: unknown, path: string): asserts value is boolean | undefined { + if (value !== undefined && typeof value !== "boolean") { + throw new Error(`${path} must be a boolean.`); + } +} + +function assertOptionalNonNegativeInteger( + value: unknown, + path: string, +): asserts value is number | undefined { + if (value === undefined) return; + if (typeof value !== "number" || !Number.isInteger(value) || value < 0) { + throw new Error(`${path} must be a non-negative integer.`); + } +} + +function assertOptionalNonNegativeNumber( + value: unknown, + path: string, +): asserts value is number | undefined { + if (value === undefined) return; + if (typeof value !== "number" || !Number.isFinite(value) || value < 0) { + throw new Error(`${path} must be a non-negative finite number.`); + } +} + +function assertOptionalMetadata( + value: unknown, + path: string, +): asserts value is Metadata | undefined { + if (value === undefined) return; + if (!isRecord(value)) { + throw new Error(`${path} must be an object.`); + } +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +}