diff --git a/docs/reference/benchmark-campaign-report.md b/docs/reference/benchmark-campaign-report.md index fd8807704e..94ec77feb4 100644 --- a/docs/reference/benchmark-campaign-report.md +++ b/docs/reference/benchmark-campaign-report.md @@ -182,3 +182,10 @@ node --experimental-strip-types --test \ The suite exercises the real CLI and missing versus zero scores, replacements, unique-case incidents, incompatible normalization, known-invalid exclusions and qualification contradictions, configuration heterogeneity and unknown fields. + +Append `--score-curves` to also write a self-contained local HTML report. +Choose a case and normalization by its section. The default chart shows the +running best from the first observed score, with a new step on improvement. +Expand **All score observations** for the full range, including unsuccessful +iterations. Each chart uses its own labeled score axis. Lines end at observed +samples. The figure is descriptive and includes provisional runs. diff --git a/scripts/benchmark-campaign-report.ts b/scripts/benchmark-campaign-report.ts index 1935a35939..972d03624a 100644 --- a/scripts/benchmark-campaign-report.ts +++ b/scripts/benchmark-campaign-report.ts @@ -2,6 +2,7 @@ * Benchmark toolkit owns run/countability authority. This developer helper neither * selects attempts nor normalizes native scores, adjudicates faults or launches work. */ +import { armColors, escapeChartText, scoreChart, scoreChartWidth } from './benchmark-score-chart.ts'; import { createHash } from 'node:crypto'; import { readFileSync, writeFileSync, mkdirSync, renameSync } from 'node:fs'; import { resolve, dirname } from 'node:path'; @@ -298,13 +299,38 @@ export function renderCsv(r: Report): string { ...r.selection.map(s => [s.arm_id,s.case_id,s.run_id,s.status,s.score?.value,s.score?.origin,s.score?.normalization_id,s.score?.countable,s.rerun_reason,s.exclude_reason,s.score?.evidence_ref,s.effort?.duration_ms,s.effort?.token_count,s.effort?.model_request_count,s.effort?.evidence_ref,s.score_samples?.length ?? 0,s.configuration?.source_revision_id,s.configuration?.scorer_revision_id,s.configuration?.model_id,s.configuration?.budget_ms,s.configuration?.active_submission_cooldown_ms,s.configuration?.background_evaluation_interval_ms,s.configuration?.resource_profile,s.configuration?.evidence_ref])].map(row => row.map(q).join(',')).join('\n')+'\n'; } +export function renderScoreCurves(report: Report): string { + const sections: string[] = []; + for (const caseId of new Set(report.score_time_series.map(s => s.case_id))) { + const samples = report.score_time_series.filter(s => s.case_id === caseId); + for (const mapping of new Set(samples.map(s => s.normalization_id))) { + const series = report.summary.map(arm => ({ + label: arm.label, + points: samples.filter(s => s.arm_id === arm.arm_id && s.normalization_id === mapping) + .map(s => ({hour: s.elapsed_ms / 3_600_000, value: s.value})), + })); + const hours = series.reduce((max, run) => Math.max(max, run.points.at(-1)?.hour ?? 0), 0) || 1; + const options = {higherIsBetter: true, hours, unit: 'Normalized score'}; + const legend = series.map((run, i) => run.points.length + ? `
  • ${escapeChartText(run.label)}
    • ` : '').join(''); + sections.push(`

    ${escapeChartText(caseId)}

    ${escapeChartText(mapping)}

    +

    Best observed score

    ${scoreChart(series, {...options, best: true})}
    +
    All score observations
    ${scoreChart(series, {...options, best: false})}
    `); + } + } + return ` + ${escapeChartText(report.title)} +

    ${escapeChartText(report.title)}

    Observed scores · includes provisional runs · ${escapeChartText(report.observed_at)}

    ${sections.join('') || '

    No observed scores.

    '}`; +} + if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) { try { const [input, prefix, ...extra] = process.argv.slice(2); - requireThat(input && prefix && !extra.length, 'Usage: node --experimental-strip-types scripts/benchmark-campaign-report.ts INPUT.json OUTPUT_PREFIX'); + requireThat(input && prefix && (!extra.length || (extra.length === 1 && extra[0] === '--score-curves')), 'Usage: node --experimental-strip-types scripts/benchmark-campaign-report.ts INPUT.json OUTPUT_PREFIX [--score-curves]'); const report = buildReport(JSON.parse(readFileSync(input, 'utf8'))); mkdirSync(dirname(resolve(prefix)), { recursive: true }); - const files = {json: JSON.stringify(report,null,2)+'\n', md:renderMarkdown(report), csv:renderCsv(report)}; + const files: Record = {json: JSON.stringify(report,null,2)+'\n', md:renderMarkdown(report), csv:renderCsv(report)}; + if (extra.length) files.html = renderScoreCurves(report); for (const [ext, body] of Object.entries(files)) { const path = `${prefix}.${ext}`; const temporary = `${path}.${process.pid}.tmp`; writeFileSync(temporary,body,{mode:0o600}); renameSync(temporary,path); diff --git a/scripts/benchmark-score-chart.ts b/scripts/benchmark-score-chart.ts new file mode 100644 index 0000000000..f9420e467b --- /dev/null +++ b/scripts/benchmark-score-chart.ts @@ -0,0 +1,89 @@ +// Render released observations only. The caller owns units, clock alignment and run selection. +export type ChartSeries = { + label: string; + points: { hour: number; value: number }[]; + observedUntil?: number; +}; + +export const scoreChartWidth = 960; +export const armColors = ['#3f5e80', '#7b4398', '#007b70', '#b55219']; +const dashPatterns = ['', '7 3', '', '3 3']; +export function escapeChartText(value: string) { + return value.replaceAll('&', '&').replaceAll('<', '<').replaceAll('>', '>') + .replaceAll('"', '"').replaceAll("'", '''); +} + +export function scoreChart(series: ChartSeries[], options: { + best: boolean; higherIsBetter: boolean; hours: number; unit: string; axisDivisor?: number; + marker?: { from: number; to: number; label: string }; +}) { + const {best, higherIsBetter, hours, marker} = options; + if (!Number.isFinite(hours) || hours <= 0) throw new Error('Expected a positive chart duration'); + const runs = series.map(run => { + let incumbent: number | undefined; + let previousHour = 0; + const points = run.points.filter(({hour, value}) => { + if (!Number.isFinite(value) || !Number.isFinite(hour) || hour < previousHour || hour > hours) + throw new Error('Score samples must be finite, ordered and within the time axis'); + previousHour = hour; + if (!best) return true; + if (incumbent !== undefined && (higherIsBetter ? value <= incumbent : value >= incumbent)) return false; + incumbent = value; + return true; + }); + return {...run, points}; + }); + let low = Infinity, high = -Infinity; + for (const run of runs) for (const point of run.points) { + low = Math.min(low, point.value); + high = Math.max(high, point.value); + } + if (!Number.isFinite(low)) return '

    No observed scores.

    '; + const pad = (high - low) * .07 || Math.max(Math.abs(high) * .001, .01); + low -= pad; + high += pad; + const width = scoreChartWidth, left = 105, right = 932, top = 37, bottom = 281; + const x = (hour: number) => (left + (right-left) * hour/hours).toFixed(2); + const y = (score: number) => (bottom - (bottom-top) * (score-low)/(high-low)).toFixed(2); + const divisor = options.axisDivisor ?? 1; + if (!Number.isFinite(divisor) || divisor <= 0) throw new Error('Expected a positive score axis divisor'); + const targetStep = (high-low)/4; + const magnitude = 10 ** Math.floor(Math.log10(targetStep)); + const step = ([1, 2, 2.5, 5, 10].find(n => n*magnitude >= targetStep) ?? 10) * magnitude; + const precision = Math.max(0, Math.min(12, 1 - Math.floor(Math.log10(step/divisor)))); + const fmt = new Intl.NumberFormat('en-US', {maximumFractionDigits: precision}); + const title = best ? 'Best observed score' : 'Individual score observations'; + const svg = [`${title}`]; + svg.push(`${escapeChartText(options.unit)}`); + if (marker) { + if (!(0 <= marker.from && marker.from <= marker.to && marker.to <= hours)) throw new Error('Marker outside chart'); + svg.push(``, + `${escapeChartText(marker.label)}`); + } + for (let tick = Math.ceil(low / step); tick <= Math.floor(high / step); tick++) { + const score = tick * step; + svg.push(`${fmt.format(score / divisor)}`); + } + for (let i = 0; i <= 6; i++) { + const hour = hours*i/6; + svg.push(`${Number(hour.toPrecision(3))} h`); + } + runs.forEach(({points, label, observedUntil}, index) => { + const last = points.at(-1); + if (!last) return; + const color = armColors[index % armColors.length]; + if (best) { + const path = [`M${x(points[0].hour)} ${y(points[0].value)}`]; + for (const p of points.slice(1)) path.push(`H${x(p.hour)}V${y(p.value)}`); + const horizon = observedUntil ?? series[index].points.at(-1)!.hour; + if (!Number.isFinite(horizon) || horizon < series[index].points.at(-1)!.hour || horizon > hours) + throw new Error('Observed horizon outside series'); + path.push(`H${x(horizon)}`); + svg.push(``); + } + for (const point of points) { + svg.push(`${escapeChartText(label)} · ${point.hour.toFixed(2)} h · ${point.value.toLocaleString('en-US',{maximumSignificantDigits:15})}`); + } + }); + return svg.join('') + ''; +} diff --git a/tests/control_plane_ts/benchmark_campaign_report.test.ts b/tests/control_plane_ts/benchmark_campaign_report.test.ts index 57751feb0b..1d00d913c0 100644 --- a/tests/control_plane_ts/benchmark_campaign_report.test.ts +++ b/tests/control_plane_ts/benchmark_campaign_report.test.ts @@ -170,3 +170,29 @@ test('configuration requires typed public-safe fields and evidence', () => { d.selection[0].configuration.budget_ms=-1; assert.throws(()=>buildReport(d),/configuration numeric/); }); + +test('opt-in HTML separates the record trajectory from all samples and splits different scales', () => { + const dir = mkdtempSync(join(tmpdir(), 'campaign-curve-')); + try { + const input = fixture(); + input.arms[0].label = 'A