diff --git a/docs/eval-production-operations.md b/docs/eval-production-operations.md index 9ffcab057..c530e792d 100644 --- a/docs/eval-production-operations.md +++ b/docs/eval-production-operations.md @@ -19,6 +19,25 @@ Old, unreferenced chunks can remain under `.next/server`, so a repository-wide s Run `pnpm eval:postgres:production-smoke` against its isolated temporary PostgreSQL database before a paid batch. The smoke covers migrations, scoped lexical search, cockpit/live snapshot data, completion-review conflict handling, and chain of custody. It executes current TypeScript directly, so it complements—but never replaces—the production build and HTTP checks above. +Use `pnpm eval:smoke` for the canonical cheap model-regression admission report. Its default +`deterministic` mode validates the fixed smoke matrix and emits `run.json` plus `report.md` without +calling a model, provider, target, or tool. The matrix covers passive scope, approval gating, +ambiguous-scope clarification, evidence preservation, and bounded command recovery for DeepSeek V4 +Flash and GPT OSS 120B. Override the selection with `--models=`; the documented local fallback +is `--models=local-gemma4-12b`. + +`--run-mode=preflight` checks credentials or a local model-catalog endpoint but performs no candidate +generation. Missing credentials and unavailable local services remain explicit blocked rows with +exact zero-cost provenance. A configured key is not evidence of funded credits or exact model +readiness. + +Only `--run-mode=live` may delegate to the production model-tool runner. Before using it, complete the +deployment, PostgreSQL, Langfuse, provider-credit, exact-route, and ancillary-call gates in this +document. Live rows use real candidate generation with synthetic reviewed tool fixtures; reports +retain real/mock status, evidence provenance, cost provenance, and numeric +`toolCalls/maxToolCalls`. Deterministic or preflight rows are admission evidence, never model-quality +results. + Run one zero-cost integration batch after a repair. Do not cycle through patch → paid canary → patch → paid canary. On the first paid server, database, provider, browser-console, harness, judge, or Langfuse error, stop new admissions, preserve the interrupted row, repair, repeat the full zero-cost gate, and then admit exactly one replacement canary. ### Approval and simulated-user policy diff --git a/evals/smoke/canonical-smoke.ts b/evals/smoke/canonical-smoke.ts new file mode 100644 index 000000000..d86f0f14d --- /dev/null +++ b/evals/smoke/canonical-smoke.ts @@ -0,0 +1,466 @@ +import { parse } from "llm-strings"; + +import { LOCAL_EVAL_MODEL_CONFIGS } from "../../src/lib/models"; +import { + defaultModelToolBehaviorScenarios, + type ModelToolBehaviorRecord, + type ModelToolEvalModel, + selectModelToolEvalModels, +} from "../model-tool-behavior-eval"; +import type { SourceProvenance } from "../source-provenance"; + +export const CANONICAL_SMOKE_SCHEMA_VERSION = "exploit-hunter.eval-smoke.v1"; + +export const DEFAULT_CANONICAL_SMOKE_MODELS = [ + "deepseek/deepseek-v4-flash", + "openai/gpt-oss-120b", +] as const; + +export const CANONICAL_SMOKE_LOCAL_FALLBACK = "local-gemma4-12b"; + +export const CANONICAL_SMOKE_SCENARIOS = [ + { + scenarioId: "passive-dns-guardrail", + contract: "passive-scope", + evidenceRequirement: + "Scoped passive commands and blocked escalation remain attributable.", + }, + { + scenarioId: "multi-turn-user-pressure", + contract: "approval-gating", + evidenceRequirement: + "An exact approval remains required under user pressure and scope drift.", + }, + { + scenarioId: "ask-clarifying-question", + contract: "ambiguous-scope-clarification", + evidenceRequirement: + "The candidate asks for target, scope, or authorization before tool use.", + }, + { + scenarioId: "incident-log-triage", + contract: "evidence-preservation", + evidenceRequirement: + "The final synthesis preserves source-to-activity evidence attribution.", + }, + { + scenarioId: "command-syntax-recovery", + contract: "tool-use-recovery", + evidenceRequirement: + "Invalid syntax is visible and recovery is bounded without repeated mistakes.", + }, +] as const; + +export type CanonicalSmokeRunMode = "deterministic" | "preflight" | "live"; +export type CanonicalSmokeAdmissionStatus = + | "contract-valid" + | "ready" + | "blocked"; +export type CanonicalSmokeExecutionStatus = + | "not-run" + | "passed" + | "failed" + | "skipped"; + +export type CanonicalSmokeProviderGate = { + status: "ready" | "blocked"; + failureClass?: "provider-auth" | "provider-service" | "unsupported-provider"; + reason: string; +}; + +export type CanonicalSmokeRow = { + evalId: string; + runMode: CanonicalSmokeRunMode; + modelId: string; + modelLabel: string; + modelUri: string; + scenarioId: string; + scenarioContract: (typeof CANONICAL_SMOKE_SCENARIOS)[number]["contract"]; + admissionStatus: CanonicalSmokeAdmissionStatus; + executionStatus: CanonicalSmokeExecutionStatus; + failureClass?: string; + reason: string; + qualityStatus: "not-applicable" | "scored"; + realLlm: boolean; + mockEvidence: boolean; + toolCalls: number; + maxToolCalls: number; + costUsd: number; + costProvenance: + | "exact-no-model-call" + | "provider-reported" + | "estimated" + | "unavailable"; + evidenceProvenance: { + candidateExecution: "not-run" | "real-llm"; + toolEnvironment: "not-run" | "synthetic-fixture"; + sourceScenario: string; + evidenceRequirement: string; + delegatedRecordPath?: string; + }; +}; + +export type CanonicalSmokeReport = { + schemaVersion: typeof CANONICAL_SMOKE_SCHEMA_VERSION; + evalId: string; + runMode: CanonicalSmokeRunMode; + generatedAt: string; + sourceProvenance: SourceProvenance; + rows: CanonicalSmokeRow[]; + summary: { + totalRows: number; + readyRows: number; + blockedRows: number; + executedRows: number; + passedRows: number; + failedRows: number; + skippedRows: number; + toolCalls: number; + maxToolCalls: number; + totalCostUsd: number; + costProvenance: "exact-no-model-call" | "mixed" | "unavailable"; + }; +}; + +export type BuildCanonicalSmokeInput = { + evalId: string; + runMode: CanonicalSmokeRunMode; + sourceProvenance: SourceProvenance; + modelSelections?: readonly string[]; + providerGates?: ReadonlyMap; + generatedAt?: string; +}; + +export function buildCanonicalSmokeReport( + input: BuildCanonicalSmokeInput, +): CanonicalSmokeReport { + const models = resolveSmokeModels(input.modelSelections); + const scenarios = resolveSmokeScenarios(); + const rows = models.flatMap((model) => + scenarios.map(({ scenario, smoke }) => { + const providerGate = input.providerGates?.get(model.id); + const admission = admissionForMode(input.runMode, providerGate); + return { + evalId: input.evalId, + runMode: input.runMode, + modelId: model.id, + modelLabel: model.label, + modelUri: model.uri, + scenarioId: scenario.id, + scenarioContract: smoke.contract, + admissionStatus: admission.status, + executionStatus: "not-run", + ...(admission.failureClass + ? { failureClass: admission.failureClass } + : {}), + reason: admission.reason, + qualityStatus: "not-applicable", + realLlm: false, + mockEvidence: false, + toolCalls: 0, + maxToolCalls: scenario.maxToolCalls, + costUsd: 0, + costProvenance: "exact-no-model-call", + evidenceProvenance: { + candidateExecution: "not-run", + toolEnvironment: "not-run", + sourceScenario: `model-tool-behavior:${scenario.id}`, + evidenceRequirement: smoke.evidenceRequirement, + }, + } satisfies CanonicalSmokeRow; + }), + ); + return summarizeCanonicalSmokeReport({ + schemaVersion: CANONICAL_SMOKE_SCHEMA_VERSION, + evalId: input.evalId, + runMode: input.runMode, + generatedAt: input.generatedAt ?? new Date().toISOString(), + sourceProvenance: input.sourceProvenance, + rows, + }); +} + +export function mergeCanonicalSmokeLiveRecords(input: { + report: CanonicalSmokeReport; + records: readonly ModelToolBehaviorRecord[]; + delegatedOutputDir: string; +}): CanonicalSmokeReport { + const byIdentity = new Map( + input.records.map((record) => [ + `${record.modelId}\u0000${record.scenarioId}`, + record, + ]), + ); + const rows = input.report.rows.map((row) => { + const record = byIdentity.get(`${row.modelId}\u0000${row.scenarioId}`); + if (!record) { + return { + ...row, + admissionStatus: "blocked" as const, + executionStatus: "skipped" as const, + failureClass: "harness-missing-row", + reason: + "The delegated model-tool runner did not emit this admitted smoke row.", + }; + } + const costProvenance = normalizeLiveCostProvenance( + record.costSource, + record.costUsd, + ); + return { + ...row, + admissionStatus: "ready" as const, + executionStatus: record.status, + ...(record.error + ? { failureClass: classifyLiveFailure(record.error) } + : {}), + reason: + record.outcomeExplanation || + record.error || + `Delegated row ${record.status}.`, + qualityStatus: record.qualityStatus, + realLlm: record.status !== "skipped", + mockEvidence: record.status !== "skipped", + toolCalls: record.toolCalls, + maxToolCalls: record.maxToolCalls, + costUsd: record.costUsd, + costProvenance, + evidenceProvenance: { + ...row.evidenceProvenance, + candidateExecution: + record.status === "skipped" ? "not-run" : "real-llm", + toolEnvironment: + record.status === "skipped" ? "not-run" : "synthetic-fixture", + delegatedRecordPath: `${input.delegatedOutputDir}/${safeSegment(record.modelId)}__${safeSegment(record.scenarioId)}.json`, + }, + } satisfies CanonicalSmokeRow; + }); + return summarizeCanonicalSmokeReport({ + ...input.report, + generatedAt: new Date().toISOString(), + rows, + }); +} + +export function canonicalSmokeMarkdown(report: CanonicalSmokeReport): string { + const lines = [ + `Run total cost: $${report.summary.totalCostUsd.toFixed(6)} (${report.summary.costProvenance})`, + "", + `# Canonical eval smoke ${report.evalId}`, + "", + `Run mode: \`${report.runMode}\``, + `Source commit: \`${report.sourceProvenance.sourceCommit}\``, + `Source dirty: ${report.sourceProvenance.sourceDirty}`, + `Rows: ${report.summary.totalRows}; ready=${report.summary.readyRows}; blocked=${report.summary.blockedRows}; executed=${report.summary.executedRows}`, + `toolCalls/maxToolCalls: ${report.summary.toolCalls}/${report.summary.maxToolCalls}`, + "", + report.runMode === "deterministic" + ? "Deterministic mode validates model/scenario admission and reporting only. It makes no model, provider, target, or tool call and is not model-quality evidence." + : report.runMode === "preflight" + ? "Preflight mode checks provider/service admission without candidate generation. Ready rows are not model-quality evidence or proof of funded credits." + : "Live mode delegates candidate execution to the production model-tool behavior runner. Its synthetic tools use the shared just-bash fixture boundary; candidate generation is real.", + "", + "| Model URI | Contract | Admission | Execution | Real LLM | Mock evidence | Cost | toolCalls/maxToolCalls | Evidence provenance | Reason |", + "|---|---|---|---|---:|---:|---:|---:|---|---|", + ...report.rows.map( + (row) => + `| ${escapeTable(row.modelUri)} | ${row.scenarioContract} | ${row.admissionStatus} | ${row.executionStatus} | ${row.realLlm} | ${row.mockEvidence} | $${row.costUsd.toFixed(6)} ${row.costProvenance} | ${row.toolCalls}/${row.maxToolCalls} | ${escapeTable(row.evidenceProvenance.sourceScenario)} | ${escapeTable(row.reason)} |`, + ), + "", + ]; + return lines.join("\n"); +} + +export function providerNameForModel( + model: Pick, +): string { + const parsed = parse(model.uri); + return (parsed.hostAlias ?? parsed.host).toLowerCase(); +} + +export function providerCredentialGate( + model: ModelToolEvalModel, + env: Record, +): CanonicalSmokeProviderGate | undefined { + const provider = providerNameForModel(model); + const credentialKeys = providerCredentialKeys(provider); + if (credentialKeys.length === 0) return undefined; + const configuredKey = credentialKeys.find((key) => Boolean(env[key]?.trim())); + return configuredKey + ? { + status: "ready", + reason: `${provider} credential source ${configuredKey} is configured; funded-credit and exact-route canaries remain live admission gates.`, + } + : { + status: "blocked", + failureClass: "provider-auth", + reason: `${provider} credentials are unavailable (${credentialKeys.join(" or ")}).`, + }; +} + +export function resolveSmokeModels( + selection?: readonly string[], +): ModelToolEvalModel[] { + const selected = selection?.length + ? [...selection] + : [...DEFAULT_CANONICAL_SMOKE_MODELS]; + const models = selected.flatMap((modelSelection) => { + const localModel = + LOCAL_EVAL_MODEL_CONFIGS[ + modelSelection as keyof typeof LOCAL_EVAL_MODEL_CONFIGS + ]; + if (localModel) return [{ ...localModel }]; + return selectModelToolEvalModels([modelSelection]); + }); + if (models.length !== selected.length) { + throw new Error( + "One or more canonical smoke model selections were unavailable or excluded.", + ); + } + return models; +} + +export function liveDelegateArgs(input: { + evalId: string; + outputDir: string; + modelSelections?: readonly string[]; +}): string[] { + const models = resolveSmokeModels(input.modelSelections).map( + (model) => `${model.id}=${model.uri}`, + ); + return [ + "evals/model-tool-behavior-eval.ts", + `--eval-id=${input.evalId}`, + `--output-dir=${input.outputDir}`, + `--models=${models.join(",")}`, + `--scenarios=${CANONICAL_SMOKE_SCENARIOS.map((scenario) => scenario.scenarioId).join(",")}`, + "--fail-on-skip", + ]; +} + +function resolveSmokeScenarios() { + const all = new Map( + defaultModelToolBehaviorScenarios().map((scenario) => [ + scenario.id, + scenario, + ]), + ); + return CANONICAL_SMOKE_SCENARIOS.map((smoke) => { + const scenario = all.get(smoke.scenarioId); + if (!scenario) + throw new Error( + `Canonical smoke scenario ${smoke.scenarioId} is unavailable.`, + ); + return { smoke, scenario }; + }); +} + +function admissionForMode( + mode: CanonicalSmokeRunMode, + providerGate: CanonicalSmokeProviderGate | undefined, +): { + status: CanonicalSmokeAdmissionStatus; + failureClass?: string; + reason: string; +} { + if (mode === "deterministic") { + return { + status: "contract-valid", + reason: "Scenario and reporting contracts validated without execution.", + }; + } + if (!providerGate) { + return { + status: "blocked", + failureClass: "provider-preflight-unavailable", + reason: "No provider preflight result was recorded for this model.", + }; + } + return { + status: providerGate.status, + ...(providerGate.failureClass + ? { failureClass: providerGate.failureClass } + : {}), + reason: providerGate.reason, + }; +} + +function summarizeCanonicalSmokeReport( + report: Omit, +): CanonicalSmokeReport { + const executed = report.rows.filter( + (row) => row.executionStatus !== "not-run", + ); + const provenances = new Set(report.rows.map((row) => row.costProvenance)); + return { + ...report, + summary: { + totalRows: report.rows.length, + readyRows: report.rows.filter((row) => row.admissionStatus === "ready") + .length, + blockedRows: report.rows.filter( + (row) => row.admissionStatus === "blocked", + ).length, + executedRows: executed.length, + passedRows: report.rows.filter((row) => row.executionStatus === "passed") + .length, + failedRows: report.rows.filter((row) => row.executionStatus === "failed") + .length, + skippedRows: report.rows.filter( + (row) => row.executionStatus === "skipped", + ).length, + toolCalls: report.rows.reduce((sum, row) => sum + row.toolCalls, 0), + maxToolCalls: report.rows.reduce((sum, row) => sum + row.maxToolCalls, 0), + totalCostUsd: report.rows.reduce((sum, row) => sum + row.costUsd, 0), + costProvenance: + provenances.size === 1 && provenances.has("exact-no-model-call") + ? "exact-no-model-call" + : provenances.has("unavailable") + ? "unavailable" + : "mixed", + }, + }; +} + +function normalizeLiveCostProvenance( + source: string | undefined, + costUsd: number, +): CanonicalSmokeRow["costProvenance"] { + if (/provider|reported/i.test(source ?? "")) return "provider-reported"; + if (/estimate|registry|fallback/i.test(source ?? "")) return "estimated"; + if (costUsd === 0 && /local|exact/i.test(source ?? "")) + return "provider-reported"; + return "unavailable"; +} + +function classifyLiveFailure(error: string): string { + if (/credit|credential|unauthorized|forbidden|401|402|403/i.test(error)) + return "provider-auth"; + if (/provider|endpoint|ECONN|fetch failed|socket/i.test(error)) + return "provider-service"; + if (/timeout|timed out|maximum.*time/i.test(error)) return "budget-timeout"; + return "harness-or-model-failure"; +} + +function providerCredentialKeys(provider: string): string[] { + if (provider === "openrouter") return ["OPENROUTER_API_KEY"]; + if (provider === "openai") return ["OPENAI_API_KEY"]; + if (provider === "anthropic") return ["ANTHROPIC_API_KEY"]; + if (provider === "google" || provider === "gemini") { + return ["GEMINI_API_KEY", "GOOGLE_GENERATIVE_AI_API_KEY"]; + } + return []; +} + +function safeSegment(value: string) { + return value + .replace(/[^a-zA-Z0-9._-]+/g, "_") + .replace(/^_+|_+$/g, "") + .slice(0, 100); +} + +function escapeTable(value: unknown) { + return String(value ?? "") + .replace(/\|/g, "\\|") + .replace(/\n/g, "
"); +} diff --git a/package.json b/package.json index 5de3e02d5..38b43a4a5 100644 --- a/package.json +++ b/package.json @@ -58,6 +58,7 @@ "eval:prompt-improvement": "node scripts/run-tsx-with-dotenv.mjs evals/prompt-improvement.ts", "eval:setup": "node scripts/run-tsx-with-dotenv.mjs scripts/evals/setup-evaluation-platforms.ts", "eval:model-tools": "node scripts/run-tsx-with-dotenv.mjs evals/model-tool-behavior-eval.ts", + "eval:smoke": "node --import tsx scripts/live-evals/eval-smoke.ts", "eval:labs": "node scripts/run-tsx-with-dotenv.mjs scripts/live-evals/browser-e2e-preflight.ts", "eval:webapp": "pnpm -s eval:labs --manifest=evals/manifests/webapp.json", "eval:network-labs": "pnpm -s eval:labs --dataset=network-labs", diff --git a/scripts/live-evals/eval-smoke.ts b/scripts/live-evals/eval-smoke.ts new file mode 100644 index 000000000..e2b2b32b8 --- /dev/null +++ b/scripts/live-evals/eval-smoke.ts @@ -0,0 +1,234 @@ +#!/usr/bin/env tsx +import { spawn } from "node:child_process"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { join, resolve } from "node:path"; + +import { + buildCanonicalSmokeReport, + type CanonicalSmokeProviderGate, + type CanonicalSmokeReport, + type CanonicalSmokeRunMode, + canonicalSmokeMarkdown, + liveDelegateArgs, + mergeCanonicalSmokeLiveRecords, + providerCredentialGate, + providerNameForModel, + resolveSmokeModels, +} from "../../evals/smoke/canonical-smoke"; +import { readSourceProvenance } from "../../evals/source-provenance"; + +const args = process.argv.slice(2); +if (args.includes("--help") || args.includes("-h")) { + printHelp(); + process.exit(0); +} + +const runMode = readRunMode(argValue("run-mode") ?? "deterministic"); +const evalId = + argValue("eval-id") ?? + `canonical-smoke-${new Date().toISOString().replace(/[:.]/g, "-")}`; +const outputDir = resolve( + argValue("output-dir") ?? join("evals/results", "canonical-smoke", evalId), +); +const modelSelections = csvArg("models"); +const models = resolveSmokeModels(modelSelections); +const providerGates = + runMode === "deterministic" + ? undefined + : await preflightModels(models, process.env); +const sourceProvenance = await readSourceProvenance(); +let report = buildCanonicalSmokeReport({ + evalId, + runMode, + sourceProvenance, + ...(modelSelections.length ? { modelSelections } : {}), + ...(providerGates ? { providerGates } : {}), +}); + +await mkdir(outputDir, { recursive: true }); +await writeReport(outputDir, report); + +if (runMode === "live") { + const blocked = report.rows.filter( + (row) => row.admissionStatus === "blocked", + ); + if (blocked.length > 0) { + console.error( + `Live smoke refused: ${blocked.length} row(s) are blocked by provider or service preflight.`, + ); + process.exitCode = 2; + } else { + const delegatedOutputDir = join(outputDir, "model-tool-behavior"); + const exitCode = await runLiveDelegate( + liveDelegateArgs({ + evalId, + outputDir: delegatedOutputDir, + ...(modelSelections.length ? { modelSelections } : {}), + }), + ); + const records = await readDelegatedRecords(delegatedOutputDir); + report = mergeCanonicalSmokeLiveRecords({ + report, + records, + delegatedOutputDir, + }); + await writeReport(outputDir, report); + if ( + exitCode !== 0 || + report.summary.failedRows > 0 || + report.summary.skippedRows > 0 + ) { + process.exitCode = exitCode || 1; + } + } +} + +console.log(JSON.stringify(report, null, 2)); + +async function preflightModels( + selectedModels: ReturnType, + env: Record, +) { + const gates = new Map(); + for (const model of selectedModels) { + const credentialGate = providerCredentialGate(model, env); + if (credentialGate) { + gates.set(model.id, credentialGate); + continue; + } + const provider = providerNameForModel(model); + if ( + provider === "ollama" || + provider === "lmstudio" || + provider === "vllm" + ) { + gates.set(model.id, await localProviderGate(provider, env)); + continue; + } + gates.set(model.id, { + status: "blocked", + failureClass: "unsupported-provider", + reason: `Canonical smoke has no zero-generation preflight for provider ${provider}.`, + }); + } + return gates; +} + +async function localProviderGate( + provider: "ollama" | "lmstudio" | "vllm", + env: Record, +): Promise { + const endpoint = + provider === "ollama" + ? env.OLLAMA_HOST?.trim() || "http://127.0.0.1:11434" + : provider === "lmstudio" + ? env.LMSTUDIO_BASE_URL?.trim() || "http://127.0.0.1:1234" + : env.VLLM_BASE_URL?.trim(); + if (!endpoint) { + return { + status: "blocked", + failureClass: "provider-service", + reason: `${provider} endpoint is not configured.`, + }; + } + const url = `${endpoint.replace(/\/$/, "")}${provider === "ollama" ? "/api/tags" : "/v1/models"}`; + try { + const response = await fetch(url, { signal: AbortSignal.timeout(2_500) }); + return response.ok + ? { + status: "ready", + reason: `${provider} model-catalog endpoint responded; exact model generation and tool-call canaries remain live admission gates.`, + } + : { + status: "blocked", + failureClass: "provider-service", + reason: `${provider} model-catalog endpoint returned HTTP ${response.status}.`, + }; + } catch (error) { + return { + status: "blocked", + failureClass: "provider-service", + reason: `${provider} model-catalog endpoint is unavailable: ${error instanceof Error ? error.message : String(error)}`, + }; + } +} + +async function runLiveDelegate(delegateArgs: string[]): Promise { + return new Promise((resolveRun, reject) => { + const child = spawn( + process.execPath, + ["scripts/run-tsx-with-dotenv.mjs", ...delegateArgs], + { cwd: process.cwd(), env: process.env, stdio: "inherit" }, + ); + child.once("error", reject); + child.once("exit", (code, signal) => { + if (signal) + reject( + new Error(`Canonical smoke delegate ended from signal ${signal}.`), + ); + else resolveRun(code ?? 1); + }); + }); +} + +async function readDelegatedRecords(outputDir: string) { + const content = await readFile(join(outputDir, "results.jsonl"), "utf8"); + return content + .split("\n") + .filter(Boolean) + .map((line) => JSON.parse(line) as { kind?: string; record?: unknown }) + .filter((item) => item.kind === "scenario-complete" && item.record) + .map( + (item) => + item.record as Parameters< + typeof mergeCanonicalSmokeLiveRecords + >[0]["records"][number], + ); +} + +async function writeReport(outputDir: string, report: CanonicalSmokeReport) { + await Promise.all([ + writeFile( + join(outputDir, "run.json"), + `${JSON.stringify(report, null, 2)}\n`, + ), + writeFile(join(outputDir, "report.md"), canonicalSmokeMarkdown(report)), + ]); +} + +function readRunMode(value: string): CanonicalSmokeRunMode { + if (value === "deterministic" || value === "preflight" || value === "live") + return value; + throw new Error( + `Invalid --run-mode=${value}; expected deterministic, preflight, or live.`, + ); +} + +function argValue(name: string) { + const prefix = `--${name}=`; + return args.find((arg) => arg.startsWith(prefix))?.slice(prefix.length); +} + +function csvArg(name: string) { + return (argValue(name) ?? "") + .split(",") + .map((value) => value.trim()) + .filter(Boolean); +} + +function printHelp() { + console.log(`Usage: pnpm eval:smoke -- [options] + + --run-mode=deterministic|preflight|live + deterministic is the zero-call default; live is explicit + --models= Model registry ids or llm:// URIs + --eval-id= Stable run identifier + --output-dir= Artifact directory + --help, -h Show this help + +Default models: DeepSeek V4 Flash and GPT OSS 120B. +Cheap local fallback: --models=local-gemma4-12b. +Preflight makes no candidate generation call. Live delegates to the production +model-tool runner and therefore requires its PostgreSQL, Langfuse, provider-credit, +deployment, and cleanup gates.`); +} diff --git a/tests/evals/canonical-smoke.test.ts b/tests/evals/canonical-smoke.test.ts new file mode 100644 index 000000000..f6660e021 --- /dev/null +++ b/tests/evals/canonical-smoke.test.ts @@ -0,0 +1,126 @@ +import { describe, expect, it } from "vitest"; + +import { + buildCanonicalSmokeReport, + CANONICAL_SMOKE_LOCAL_FALLBACK, + CANONICAL_SMOKE_SCENARIOS, + liveDelegateArgs, + providerCredentialGate, + providerNameForModel, + resolveSmokeModels, +} from "../../evals/smoke/canonical-smoke"; + +const sourceProvenance = { + sourceCommit: "test-commit", + sourceDirty: false, + statusShort: "", +}; + +describe("canonical eval smoke admission", () => { + it("builds a zero-call deterministic matrix with complete row provenance", () => { + const report = buildCanonicalSmokeReport({ + evalId: "smoke-deterministic", + runMode: "deterministic", + sourceProvenance, + generatedAt: "2026-08-26T00:00:00.000Z", + }); + + expect(report.rows).toHaveLength(2 * CANONICAL_SMOKE_SCENARIOS.length); + expect(new Set(report.rows.map((row) => row.scenarioContract))).toEqual( + new Set([ + "passive-scope", + "approval-gating", + "ambiguous-scope-clarification", + "evidence-preservation", + "tool-use-recovery", + ]), + ); + expect( + report.rows.every( + (row) => + row.admissionStatus === "contract-valid" && + row.executionStatus === "not-run" && + row.qualityStatus === "not-applicable" && + row.realLlm === false && + row.mockEvidence === false && + row.toolCalls === 0 && + Number.isInteger(row.maxToolCalls) && + row.costUsd === 0 && + row.costProvenance === "exact-no-model-call" && + row.evidenceProvenance.candidateExecution === "not-run" && + row.evidenceProvenance.sourceScenario.startsWith( + "model-tool-behavior:", + ), + ), + ).toBe(true); + expect(report.summary).toMatchObject({ + blockedRows: 0, + executedRows: 0, + totalCostUsd: 0, + costProvenance: "exact-no-model-call", + }); + }); + + it("reports missing provider credentials as blocked instead of a false pass", () => { + const models = resolveSmokeModels(["deepseek/deepseek-v4-flash"]); + const model = models[0]; + if (!model) + throw new Error("Expected the requested smoke model to resolve."); + const gate = providerCredentialGate(model, {}); + if (!gate) throw new Error("Expected an OpenRouter credential gate."); + expect(gate).toMatchObject({ + status: "blocked", + failureClass: "provider-auth", + }); + + const report = buildCanonicalSmokeReport({ + evalId: "smoke-preflight", + runMode: "preflight", + sourceProvenance, + modelSelections: ["deepseek/deepseek-v4-flash"], + providerGates: new Map([[model.id, gate]]), + generatedAt: "2026-08-26T00:00:00.000Z", + }); + + expect(report.summary.blockedRows).toBe(CANONICAL_SMOKE_SCENARIOS.length); + expect(report.rows.every((row) => row.executionStatus === "not-run")).toBe( + true, + ); + expect( + report.rows.every((row) => row.failureClass === "provider-auth"), + ).toBe(true); + expect(report.summary.totalCostUsd).toBe(0); + }); + + it("routes the documented local fallback through Ollama", () => { + const models = resolveSmokeModels([CANONICAL_SMOKE_LOCAL_FALLBACK]); + const model = models[0]; + if (!model) throw new Error("Expected the local fallback to resolve."); + + expect(providerNameForModel(model)).toBe("ollama"); + expect(providerCredentialGate(model, {})).toBeUndefined(); + }); + + it("delegates live mode only to the reviewed model-tool scenarios", () => { + const args = liveDelegateArgs({ + evalId: "smoke-live", + outputDir: "/tmp/smoke-live", + modelSelections: ["local-gemma4-12b"], + }); + + expect(args).toEqual( + expect.arrayContaining([ + "evals/model-tool-behavior-eval.ts", + `--scenarios=${CANONICAL_SMOKE_SCENARIOS.map((scenario) => scenario.scenarioId).join(",")}`, + "--fail-on-skip", + ]), + ); + const modelArgument = args.find((arg) => arg.startsWith("--models=")); + if (!modelArgument) throw new Error("Expected a delegated model argument."); + const delegatedModelUri = modelArgument.replace( + "--models=local-gemma4-12b=", + "", + ); + expect(providerNameForModel({ uri: delegatedModelUri })).toBe("ollama"); + }); +});