From 8994d5df7754eb16144f7bf14f51199fa5766a8d Mon Sep 17 00:00:00 2001 From: salimlaimeche Date: Wed, 1 Jul 2026 11:12:06 +0200 Subject: [PATCH] fix: clarify single-variant recommendation semantics --- ...IGNITIONRAG_EVALUATION_CENTER_CHECKLIST.md | 5 +- packages/exporters/README.md | 7 ++ packages/exporters/src/index.test.ts | 86 +++++++++++++++++++ packages/exporters/src/index.ts | 58 +++++++++++-- packages/trainer/README.md | 30 ++++++- packages/trainer/src/recommendation.test.ts | 53 +++++++++++- packages/trainer/src/select-best.ts | 44 +++++++++- 7 files changed, 270 insertions(+), 13 deletions(-) diff --git a/docs/IGNITIONRAG_EVALUATION_CENTER_CHECKLIST.md b/docs/IGNITIONRAG_EVALUATION_CENTER_CHECKLIST.md index 1727b1a..0a256d9 100644 --- a/docs/IGNITIONRAG_EVALUATION_CENTER_CHECKLIST.md +++ b/docs/IGNITIONRAG_EVALUATION_CENTER_CHECKLIST.md @@ -40,7 +40,7 @@ The first version may evaluate one workflow at a time. Multi-variant Experiment | `groundedness_like` | deterministic grounding proxy | custom reward or future preset | | latency and cost rewards | operational quality metrics | `latencyPenalty()`, `costPenalty()` | | leaderboard | report summary | `ExperimentResult.leaderboard` | -| recommendation | deterministic best strategy explanation | `recommendVariant()` | +| recommendation | deterministic baseline or comparison explanation | `recommendVariant()` | | JSON/Markdown report | durable report artifact | `toJsonReport()`, `toMarkdownReport()` | | regression gate | publish or release guard | `compareExperimentResults()`, `assertNoRegression()` | @@ -114,7 +114,8 @@ Reports: Recommendations: -- use `recommendVariant()` only when multiple variants are present, +- use `recommendVariant()` for baseline and comparative runs, but respect `comparisonAvailable`, +- show single-variant runs as a baseline measurement, not as a winner, - show reasons, tradeoffs and confidence, - do not mutate prompts or workflows from recommendations in the Evaluation Center. diff --git a/packages/exporters/README.md b/packages/exporters/README.md index 84050b7..b5dcbbe 100644 --- a/packages/exporters/README.md +++ b/packages/exporters/README.md @@ -51,6 +51,13 @@ The exported report includes: - recommendation when provided, - metadata when provided. +Recommendations may be comparative or baseline-only: + +- `comparisonAvailable: true` and `recommendationKind: "comparison"` keep the normal winner wording. +- `comparisonAvailable: false` or `recommendationKind: "baseline"` render Markdown as a baseline measurement instead of a winner. + +When recommendation metadata is omitted, exporters infer the semantics from the report variant count. Single-variant reports say that no alternative variants were evaluated and keep confidence low or explicitly non-comparative. + ## Report Bundles `writeReportBundle()` creates a local folder named from the experiment and generation timestamp: diff --git a/packages/exporters/src/index.test.ts b/packages/exporters/src/index.test.ts index 02c6e72..d47171c 100644 --- a/packages/exporters/src/index.test.ts +++ b/packages/exporters/src/index.test.ts @@ -90,6 +90,52 @@ const result: ExperimentResult = { ], }; +const singleVariantResult: ExperimentResult = { + name: "single-agent-baseline", + startedAt: "2026-01-01T00:00:00.000Z", + endedAt: "2026-01-01T00:01:00.000Z", + leaderboard: [ + { + variantId: "current-agent", + name: "current-agent", + score: 0.82, + totalCases: 2, + failedCases: 0, + averageLatencyMs: 420, + totalCostUsd: 0.003, + rewardAverages: { answer_quality: 0.82 }, + }, + ], + cases: [ + { + caseId: "case-1", + variantId: "current-agent", + variantName: "current-agent", + output: "answer", + trace: { steps: [] }, + rewards: [{ name: "answer_quality", score: 0.82, weight: 1 }], + score: 0.82, + usage: { latencyMs: 420, costUsd: 0.003 }, + }, + ], + failedCases: [], +}; + +const legacyBaselineRecommendation = { + winner: "current-agent", + score: 0.82, +}; + +const baselineRecommendation = { + winner: "current-agent", + score: 0.82, + summary: + "Baseline measured for current-agent. No alternative variants were evaluated, so no comparative winner is available.", + confidence: "low", + comparisonAvailable: false, + recommendationKind: "baseline" as const, +}; + describe("exportExperimentResult", () => { it("exports a stable report shape with dataset, variants, leaderboard and reward summaries", () => { const report = exportExperimentResult(result, { @@ -140,6 +186,8 @@ describe("exportExperimentResult", () => { score: 0.91, summary: "Use verification for better answer quality.", confidence: "medium", + comparisonAvailable: true, + recommendationKind: "comparison", }); }); @@ -174,6 +222,22 @@ describe("report serializers", () => { expect(parsed.leaderboard[0].name).toBe("RAG + Verify"); }); + it("serializes single-variant baseline recommendation metadata in JSON", () => { + const json = toJsonReport(singleVariantResult, { + generatedAt: "2026-01-01T00:02:00.000Z", + recommendation: legacyBaselineRecommendation, + }); + const parsed = JSON.parse(json); + const wording = JSON.stringify(parsed.recommendation); + + expect(parsed.recommendation).toEqual(baselineRecommendation); + expect(parsed.recommendation.comparisonAvailable).toBe(false); + expect(parsed.recommendation.recommendationKind).toBe("baseline"); + expect(wording).not.toMatch(/Use current-agent/i); + expect(wording).not.toMatch(/highest overall score/i); + expect(wording).not.toMatch(/best variant/i); + }); + it("serializes a Markdown summary with leaderboard and recommendation", () => { const markdown = toMarkdownReport(result, { generatedAt: "2026-01-01T00:02:00.000Z", @@ -182,6 +246,8 @@ describe("report serializers", () => { score: 0.91, reasons: ["Best answer quality."], tradeoffs: ["Higher latency."], + comparisonAvailable: true, + recommendationKind: "comparison", }, }); @@ -189,8 +255,28 @@ describe("report serializers", () => { expect(markdown).toContain("| 1 | RAG + Verify | 0.910 | 2 | 0 | 900ms | $0.0123 |"); expect(markdown).toContain("## Reward summaries"); expect(markdown).toContain("## Recommendation"); + expect(markdown).toContain("Winner: RAG + Verify (0.910)"); + expect(markdown).not.toContain("Baseline: RAG + Verify"); expect(markdown).toContain("- Best answer quality."); }); + + it("serializes single-variant baseline Markdown without comparative winner wording", () => { + const markdown = toMarkdownReport(singleVariantResult, { + generatedAt: "2026-01-01T00:02:00.000Z", + recommendation: legacyBaselineRecommendation, + }); + + expect(markdown).toContain("# Experiment report: single-agent-baseline"); + expect(markdown).toContain("## Recommendation"); + expect(markdown).toContain("Baseline: current-agent (0.820)"); + expect(markdown).toContain("Baseline measured for current-agent."); + expect(markdown).toContain("No alternative variants were evaluated"); + expect(markdown).toContain("Confidence: low"); + expect(markdown).not.toContain("Winner:"); + expect(markdown).not.toMatch(/Use current-agent/i); + expect(markdown).not.toMatch(/highest overall score/i); + expect(markdown).not.toMatch(/best variant/i); + }); }); describe("writeReportBundle", () => { diff --git a/packages/exporters/src/index.ts b/packages/exporters/src/index.ts index e39c60b..d19f8da 100644 --- a/packages/exporters/src/index.ts +++ b/packages/exporters/src/index.ts @@ -22,6 +22,8 @@ export interface ExportRecommendation { reasons?: string[]; tradeoffs?: string[]; confidence?: string; + comparisonAvailable?: boolean; + recommendationKind?: "baseline" | "comparison"; alternatives?: ExportRecommendationAlternative[]; metadata?: Metadata; } @@ -141,11 +143,43 @@ export function exportExperimentResult( rewardSummaries: rewardSummaries(result.leaderboard), }; - if (options.recommendation != null) exportResult.recommendation = options.recommendation; + if (options.recommendation != null) { + exportResult.recommendation = normalizeRecommendation( + options.recommendation, + result.leaderboard.length, + ); + } if (options.metadata !== undefined) exportResult.metadata = options.metadata; return exportResult; } +function normalizeRecommendation( + recommendation: ExportRecommendation, + variantCount: number, +): ExportRecommendation { + const comparisonAvailable = + recommendation.comparisonAvailable ?? + (recommendation.recommendationKind === "baseline" ? false : variantCount > 1); + const recommendationKind = + recommendation.recommendationKind ?? (comparisonAvailable ? "comparison" : "baseline"); + + if (comparisonAvailable) { + return { + ...recommendation, + comparisonAvailable, + recommendationKind, + }; + } + + return { + ...recommendation, + summary: recommendation.summary ?? baselineSummary(recommendation.winner), + confidence: recommendation.confidence ?? "low", + comparisonAvailable, + recommendationKind, + }; +} + export function toJsonReport( result: ExperimentResult, options: ExperimentResultExportOptions = {}, @@ -361,11 +395,19 @@ ${rows}`; } function recommendationMarkdown(recommendation: ExportRecommendation): string { - const lines = [ - "## Recommendation", - "", - `Winner: ${recommendation.winner} (${formatScore(recommendation.score)})`, - ]; + const isBaseline = + recommendation.comparisonAvailable === false || + recommendation.recommendationKind === "baseline"; + const lines = ["## Recommendation", ""]; + + if (isBaseline) { + lines.push(`Baseline: ${recommendation.winner} (${formatScore(recommendation.score)})`); + if (recommendation.summary === undefined) { + lines.push("", baselineSummary(recommendation.winner)); + } + } else { + lines.push(`Winner: ${recommendation.winner} (${formatScore(recommendation.score)})`); + } if (recommendation.summary !== undefined) lines.push("", recommendation.summary); if (recommendation.confidence !== undefined) @@ -380,6 +422,10 @@ function recommendationMarkdown(recommendation: ExportRecommendation): string { return lines.join("\n"); } +function baselineSummary(variantName: string): string { + return `Baseline measured for ${variantName}. No alternative variants were evaluated, so no comparative winner is available.`; +} + function markdownCell(value: string): string { return value.replaceAll("|", "\\|").replaceAll("\n", " "); } diff --git a/packages/trainer/README.md b/packages/trainer/README.md index b0b66b4..dce815a 100644 --- a/packages/trainer/README.md +++ b/packages/trainer/README.md @@ -5,8 +5,10 @@ Deterministic recommendation primitives for Ignition Agent Trainer experiment re The trainer package does not train model weights today. It interprets an `ExperimentResult` and answers: ```txt -Which variant won? -Why did it win? +Was a comparison available? +Which variant won when alternatives were evaluated? +What baseline was measured when only one variant was evaluated? +Why did the recommendation choose that wording? What are the tradeoffs? How confident are we? Which alternatives are close? @@ -39,14 +41,30 @@ const combinations = generateParameterCombinations({ `recommendVariant()` returns a structured recommendation with: -- winner name, +- winner name for comparative runs, or the measured baseline variant name for single-variant runs, - score, - summary, - reasons, - tradeoffs, - confidence, +- `comparisonAvailable`, +- `recommendationKind`, - alternatives. +When an experiment has only one variant, the recommendation is intentionally non-comparative: + +```ts +const recommendation = recommendVariant(singleVariantResult); + +recommendation?.comparisonAvailable; // false +recommendation?.recommendationKind; // "baseline" +recommendation?.confidence; // "low" +recommendation?.summary; +// "Baseline measured for current-agent. No alternative variants were evaluated, so no comparative winner is available." +``` + +Consumers should treat this as a baseline measurement, not as a winning strategy. Add another variant before showing winner, best variant or promotion language. + Optimization primitives are deterministic and support four objectives: - `quality-first`, @@ -133,6 +151,12 @@ The trainer layer turns a leaderboard into a product-facing decision: Experiment result -> Recommendation -> Strategy selection ``` +For single-variant evaluation runs, the trainer layer turns the leaderboard into a baseline summary instead: + +```txt +Experiment result -> Baseline measurement -> Compare against alternatives later +``` + ## Not RL Yet This package is intentionally rule-based in the MVP. It does not implement PPO, GRPO, prompt mutation, bandits or learned routing policies. diff --git a/packages/trainer/src/recommendation.test.ts b/packages/trainer/src/recommendation.test.ts index 5a2b044..4c7a65d 100644 --- a/packages/trainer/src/recommendation.test.ts +++ b/packages/trainer/src/recommendation.test.ts @@ -51,12 +51,63 @@ describe("trainer recommendations", () => { const recommendation = recommendVariant(result); expect(recommendation?.winner).toBe("rag-with-verification"); - expect(recommendation?.summary).toContain("rag-with-verification"); + expect(recommendation?.summary).toBe( + "Use rag-with-verification because it achieved the highest overall score.", + ); + expect(recommendation?.comparisonAvailable).toBe(true); + expect(recommendation?.recommendationKind).toBe("comparison"); expect(recommendation?.reasons.length).toBeGreaterThan(0); expect(recommendation?.tradeoffs.join(" ")).toContain("slower"); expect(recommendation?.alternatives[0]?.variant).toBe("rag-basic"); }); + it("treats a single variant as a baseline measurement instead of a comparative win", () => { + const result = experiment([ + variant({ + name: "current-agent", + score: 0.82, + totalCases: 12, + rewardAverages: { answer_quality: 0.82, citations: 0.75 }, + }), + ]); + + const recommendation = recommendVariant(result); + const explanation = explainTradeoffs(result); + const wording = [ + recommendation?.summary, + ...(recommendation?.reasons ?? []), + ...(recommendation?.tradeoffs ?? []), + ].join(" "); + + expect(recommendation).toMatchObject({ + winner: "current-agent", + score: 0.82, + summary: + "Baseline measured for current-agent. No alternative variants were evaluated, so no comparative winner is available.", + confidence: "low", + comparisonAvailable: false, + recommendationKind: "baseline", + alternatives: [], + metadata: { + variantId: "current-agent", + totalCases: 12, + comparisonAvailable: false, + recommendationKind: "baseline", + }, + }); + expect(explanation).toMatchObject({ + winner: "current-agent", + comparisonAvailable: false, + recommendationKind: "baseline", + alternatives: [], + }); + expect(wording).not.toMatch(/Use current-agent/i); + expect(wording).not.toMatch(/highest overall score/i); + expect(wording).not.toMatch(/ranked first/i); + expect(wording).not.toMatch(/best variant/i); + expect(wording).not.toMatch(/recommended winning strategy/i); + }); + it("uses low confidence when scores are too close", () => { const result = experiment([ variant({ name: "rag-basic", score: 0.82, totalCases: 8 }), diff --git a/packages/trainer/src/select-best.ts b/packages/trainer/src/select-best.ts index af79b69..c156268 100644 --- a/packages/trainer/src/select-best.ts +++ b/packages/trainer/src/select-best.ts @@ -1,6 +1,7 @@ import type { ExperimentResult, VariantSummary } from "@ignitionai/agent-trainer-core"; export type RecommendationConfidence = "low" | "medium" | "high"; +export type RecommendationKind = "baseline" | "comparison"; export interface VariantAlternative { variant: string; @@ -15,6 +16,8 @@ export interface VariantRecommendation { reasons: string[]; tradeoffs: string[]; confidence: RecommendationConfidence; + comparisonAvailable?: boolean; + recommendationKind?: RecommendationKind; alternatives: VariantAlternative[]; metadata?: Record; } @@ -32,6 +35,8 @@ export interface TradeoffExplanation { reasons: string[]; tradeoffs: string[]; alternatives: VariantAlternative[]; + comparisonAvailable?: boolean; + recommendationKind?: RecommendationKind; scoreGap?: number; metadata?: Record; } @@ -62,19 +67,27 @@ export function recommendVariant( const resolvedOptions = { ...defaultOptions, ...options }; const explanation = explainTradeoffs(result, resolvedOptions); if (explanation === null) return null; + const comparisonAvailable = explanation.comparisonAvailable ?? true; + const recommendationKind = explanation.recommendationKind ?? "comparison"; return { winner: winner.name, score: winner.score, - summary: `Use ${winner.name} because it achieved the highest overall score.`, + summary: comparisonAvailable + ? `Use ${winner.name} because it achieved the highest overall score.` + : baselineSummary(winner.name), reasons: explanation.reasons, tradeoffs: explanation.tradeoffs, confidence: inferConfidence(result, explanation.scoreGap, resolvedOptions), + comparisonAvailable, + recommendationKind, alternatives: explanation.alternatives, metadata: { variantId: winner.variantId, totalCases: winner.totalCases, scoreGap: explanation.scoreGap, + comparisonAvailable, + recommendationKind, }, }; } @@ -87,6 +100,7 @@ export function explainTradeoffs( const leaderboard = sortLeaderboard(result.leaderboard); const winner = leaderboard[0]; if (winner === undefined) return null; + if (leaderboard.length === 1) return baselineExplanation(winner); const runnerUp = leaderboard[1]; const scoreGap = runnerUp === undefined ? undefined : winner.score - runnerUp.score; @@ -101,10 +115,38 @@ export function explainTradeoffs( reasons, tradeoffs, alternatives, + comparisonAvailable: true, + recommendationKind: "comparison", ...(scoreGap !== undefined ? { scoreGap } : {}), metadata: { variantId: winner.variantId, totalCases: winner.totalCases, + comparisonAvailable: true, + recommendationKind: "comparison", + }, + }; +} + +function baselineSummary(variantName: string): string { + return `Baseline measured for ${variantName}. No alternative variants were evaluated, so no comparative winner is available.`; +} + +function baselineExplanation(variant: VariantSummary): TradeoffExplanation { + return { + winner: variant.name, + reasons: [ + `Baseline measured for ${variant.name}.`, + "No alternative variants were evaluated, so no comparative winner is available.", + ], + tradeoffs: ["Add at least one alternative variant before making comparative strategy claims."], + alternatives: [], + comparisonAvailable: false, + recommendationKind: "baseline", + metadata: { + variantId: variant.variantId, + totalCases: variant.totalCases, + comparisonAvailable: false, + recommendationKind: "baseline", }, }; }