feat(benchmark): show best-score trajectories alongside all observations

docs/reference/benchmark-campaign-report.md

@@ -182,3 +182,10 @@ node --experimental-strip-types --test \
 The suite exercises the real CLI and missing versus zero scores, replacements,
 unique-case incidents, incompatible normalization, known-invalid exclusions and
 qualification contradictions, configuration heterogeneity and unknown fields.
+
+Append `--score-curves` to also write a self-contained local HTML report.
+Choose a case and normalization by its section. The default chart shows the
+running best from the first observed score, with a new step on improvement.
+Expand **All score observations** for the full range, including unsuccessful
+iterations. Each chart uses its own labeled score axis. Lines end at observed
+samples. The figure is descriptive and includes provisional runs.

scripts/benchmark-campaign-report.ts

@@ -2,6 +2,7 @@
  * Benchmark toolkit owns run/countability authority. This developer helper neither
  * selects attempts nor normalizes native scores, adjudicates faults or launches work.
  */
+import { armColors, escapeChartText, scoreChart, scoreChartWidth } from './benchmark-score-chart.ts';
 import { createHash } from 'node:crypto';
 import { readFileSync, writeFileSync, mkdirSync, renameSync } from 'node:fs';
 import { resolve, dirname } from 'node:path';
@@ -298,13 +299,38 @@ export function renderCsv(r: Report): string {
     ...r.selection.map(s => [s.arm_id,s.case_id,s.run_id,s.status,s.score?.value,s.score?.origin,s.score?.normalization_id,s.score?.countable,s.rerun_reason,s.exclude_reason,s.score?.evidence_ref,s.effort?.duration_ms,s.effort?.token_count,s.effort?.model_request_count,s.effort?.evidence_ref,s.score_samples?.length ?? 0,s.configuration?.source_revision_id,s.configuration?.scorer_revision_id,s.configuration?.model_id,s.configuration?.budget_ms,s.configuration?.active_submission_cooldown_ms,s.configuration?.background_evaluation_interval_ms,s.configuration?.resource_profile,s.configuration?.evidence_ref])].map(row => row.map(q).join(',')).join('\n')+'\n';
 }
 
+export function renderScoreCurves(report: Report): string {
+  const sections: string[] = [];
+  for (const caseId of new Set(report.score_time_series.map(s => s.case_id))) {
+    const samples = report.score_time_series.filter(s => s.case_id === caseId);
+    for (const mapping of new Set(samples.map(s => s.normalization_id))) {
+      const series = report.summary.map(arm => ({
+        label: arm.label,
+        points: samples.filter(s => s.arm_id === arm.arm_id && s.normalization_id === mapping)
+          .map(s => ({hour: s.elapsed_ms / 3_600_000, value: s.value})),
+      }));
+      const hours = series.reduce((max, run) => Math.max(max, run.points.at(-1)?.hour ?? 0), 0) || 1;
+      const options = {higherIsBetter: true, hours, unit: 'Normalized score'};
+      const legend = series.map((run, i) => run.points.length
+        ? `<li style="color:${armColors[i % armColors.length]}">${escapeChartText(run.label)}</li>` : '').join('');
+      sections.push(`<section><h2>${escapeChartText(caseId)}</h2><p>${escapeChartText(mapping)}</p><ul>${legend}</ul>
+        <h3>Best observed score</h3><div class="plot" tabindex="0" role="region" aria-label="Best observed scores for ${escapeChartText(caseId)}, ${escapeChartText(mapping)}">${scoreChart(series, {...options, best: true})}</div>
+        <details><summary>All score observations</summary><div class="plot" tabindex="0" role="region" aria-label="All score observations for ${escapeChartText(caseId)}, ${escapeChartText(mapping)}">${scoreChart(series, {...options, best: false})}</div></details></section>`);
+    }
+  }
+  return `<!doctype html><html lang="en"><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
+    <title>${escapeChartText(report.title)}</title><style>body{max-width:${scoreChartWidth}px;margin:2rem auto;padding:0 1rem;color:#20302c;font:16px system-ui}.plot{overflow-x:auto}.plot:focus-visible{outline:2px solid #3f5e80;outline-offset:3px}svg{display:block;width:${scoreChartWidth}px;max-width:none;height:auto}ul{display:flex;flex-wrap:wrap;gap:1.3rem;padding:0;list-style:none;font-weight:600}section{border-top:1px solid #dce4e1}summary{cursor:pointer}</style>
+    <h1>${escapeChartText(report.title)}</h1><p>Observed scores · includes provisional runs · ${escapeChartText(report.observed_at)}</p>${sections.join('') || '<p>No observed scores.</p>'}</html>`;
+}
+
 if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) {
   try {
     const [input, prefix, ...extra] = process.argv.slice(2);
-    requireThat(input && prefix && !extra.length, 'Usage: node --experimental-strip-types scripts/benchmark-campaign-report.ts INPUT.json OUTPUT_PREFIX');
+    requireThat(input && prefix && (!extra.length || (extra.length === 1 && extra[0] === '--score-curves')), 'Usage: node --experimental-strip-types scripts/benchmark-campaign-report.ts INPUT.json OUTPUT_PREFIX [--score-curves]');
     const report = buildReport(JSON.parse(readFileSync(input, 'utf8')));
     mkdirSync(dirname(resolve(prefix)), { recursive: true });
-    const files = {json: JSON.stringify(report,null,2)+'\n', md:renderMarkdown(report), csv:renderCsv(report)};
+    const files: Record<string, string> = {json: JSON.stringify(report,null,2)+'\n', md:renderMarkdown(report), csv:renderCsv(report)};
+    if (extra.length) files.html = renderScoreCurves(report);
     for (const [ext, body] of Object.entries(files)) {
       const path = `${prefix}.${ext}`; const temporary = `${path}.${process.pid}.tmp`;
       writeFileSync(temporary,body,{mode:0o600}); renameSync(temporary,path);

scripts/benchmark-score-chart.ts

@@ -0,0 +1,89 @@
+// Render released observations only. The caller owns units, clock alignment and run selection.
+export type ChartSeries = {
+  label: string;
+  points: { hour: number; value: number }[];
+  observedUntil?: number;
+};
+
+export const scoreChartWidth = 960;
+export const armColors = ['#3f5e80', '#7b4398', '#007b70', '#b55219'];
+const dashPatterns = ['', '7 3', '', '3 3'];
+export function escapeChartText(value: string) {
+  return value.replaceAll('&', '&amp;').replaceAll('<', '&lt;').replaceAll('>', '&gt;')
+    .replaceAll('"', '&quot;').replaceAll("'", '&#39;');
+}
+
+export function scoreChart(series: ChartSeries[], options: {
+  best: boolean; higherIsBetter: boolean; hours: number; unit: string; axisDivisor?: number;
+  marker?: { from: number; to: number; label: string };
+}) {
+  const {best, higherIsBetter, hours, marker} = options;
+  if (!Number.isFinite(hours) || hours <= 0) throw new Error('Expected a positive chart duration');
+  const runs = series.map(run => {
+    let incumbent: number | undefined;
+    let previousHour = 0;
+    const points = run.points.filter(({hour, value}) => {
+      if (!Number.isFinite(value) || !Number.isFinite(hour) || hour < previousHour || hour > hours)
+        throw new Error('Score samples must be finite, ordered and within the time axis');
+      previousHour = hour;
+      if (!best) return true;
+      if (incumbent !== undefined && (higherIsBetter ? value <= incumbent : value >= incumbent)) return false;
+      incumbent = value;
+      return true;
+    });
+    return {...run, points};
+  });
+  let low = Infinity, high = -Infinity;
+  for (const run of runs) for (const point of run.points) {
+    low = Math.min(low, point.value);
+    high = Math.max(high, point.value);
+  }
+  if (!Number.isFinite(low)) return '<p>No observed scores.</p>';
+  const pad = (high - low) * .07 || Math.max(Math.abs(high) * .001, .01);
+  low -= pad;
+  high += pad;
+  const width = scoreChartWidth, left = 105, right = 932, top = 37, bottom = 281;
+  const x = (hour: number) => (left + (right-left) * hour/hours).toFixed(2);
+  const y = (score: number) => (bottom - (bottom-top) * (score-low)/(high-low)).toFixed(2);
+  const divisor = options.axisDivisor ?? 1;
+  if (!Number.isFinite(divisor) || divisor <= 0) throw new Error('Expected a positive score axis divisor');
+  const targetStep = (high-low)/4;
+  const magnitude = 10 ** Math.floor(Math.log10(targetStep));
+  const step = ([1, 2, 2.5, 5, 10].find(n => n*magnitude >= targetStep) ?? 10) * magnitude;
+  const precision = Math.max(0, Math.min(12, 1 - Math.floor(Math.log10(step/divisor))));
+  const fmt = new Intl.NumberFormat('en-US', {maximumFractionDigits: precision});
+  const title = best ? 'Best observed score' : 'Individual score observations';
+  const svg = [`<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 ${width} 322" role="img" aria-label="${title}" font-family="system-ui,sans-serif" font-size="13" fill="#4c5c59"><title>${title}</title>`];
+  svg.push(`<text x="${left}" y="22">${escapeChartText(options.unit)}</text>`);
+  if (marker) {
+    if (!(0 <= marker.from && marker.from <= marker.to && marker.to <= hours)) throw new Error('Marker outside chart');
+    svg.push(`<rect x="${x(marker.from)}" y="${top}" width="${Number(x(marker.to))-Number(x(marker.from))}" height="${bottom-top}" fill="#edf0ef"/>`,
+      `<path d="M${x(marker.to)} ${top}V${bottom}" stroke="#778780" stroke-dasharray="3 5"/><text x="${x(marker.to)}" y="22" text-anchor="middle">${escapeChartText(marker.label)}</text>`);
+  }
+  for (let tick = Math.ceil(low / step); tick <= Math.floor(high / step); tick++) {
+    const score = tick * step;
+    svg.push(`<path d="M${left} ${y(score)}H${right}" stroke="#e4e9e7"/><text x="${left-10}" y="${Number(y(score))+4}" text-anchor="end">${fmt.format(score / divisor)}</text>`);
+  }
+  for (let i = 0; i <= 6; i++) {
+    const hour = hours*i/6;
+    svg.push(`<text x="${x(hour)}" y="${bottom+26}" text-anchor="middle">${Number(hour.toPrecision(3))} h</text>`);
+  }
+  runs.forEach(({points, label, observedUntil}, index) => {
+    const last = points.at(-1);
+    if (!last) return;
+    const color = armColors[index % armColors.length];
+    if (best) {
+      const path = [`M${x(points[0].hour)} ${y(points[0].value)}`];
+      for (const p of points.slice(1)) path.push(`H${x(p.hour)}V${y(p.value)}`);
+      const horizon = observedUntil ?? series[index].points.at(-1)!.hour;
+      if (!Number.isFinite(horizon) || horizon < series[index].points.at(-1)!.hour || horizon > hours)
+        throw new Error('Observed horizon outside series');
+      path.push(`H${x(horizon)}`);
+      svg.push(`<path d="${path.join(' ')}" fill="none" stroke="${color}" stroke-width="2.8" stroke-dasharray="${dashPatterns[index % dashPatterns.length]}"/>`);
+    }
+    for (const point of points) {
+      svg.push(`<circle cx="${x(point.hour)}" cy="${y(point.value)}" r="4" fill="${color}" stroke="white" stroke-width="1"><title>${escapeChartText(label)} · ${point.hour.toFixed(2)} h · ${point.value.toLocaleString('en-US',{maximumSignificantDigits:15})}</title></circle>`);
+    }
+  });
+  return svg.join('') + '</svg>';
+}

tests/control_plane_ts/benchmark_campaign_report.test.ts

@@ -170,3 +170,29 @@ test('configuration requires typed public-safe fields and evidence', () => {
   d.selection[0].configuration.budget_ms=-1;
   assert.throws(()=>buildReport(d),/configuration numeric/);
 });
+
+test('opt-in HTML separates the record trajectory from all samples and splits different scales', () => {
+  const dir = mkdtempSync(join(tmpdir(), 'campaign-curve-'));
+  try {
+    const input = fixture();
+    input.arms[0].label = 'A <script>';
+    input.selection[0].score_samples = [95, 2, 97, 96].map((value, i) => ({
+      elapsed_ms: i * 3600000, value, normalization_id: 'scale-a', evidence_ref: 'receipt:sample',
+    }));
+    input.selection[3].score_samples = [{elapsed_ms: 0, value: 92, normalization_id: 'scale-b', evidence_ref: 'receipt:b'}];
+    const path = join(dir, 'selection.json'), prefix = join(dir, 'report');
+    writeFileSync(path, JSON.stringify(input));
+    const run = spawnSync(process.execPath, ['--experimental-strip-types', resolve('scripts/benchmark-campaign-report.ts'),
+      path, prefix, '--score-curves'], {encoding:'utf8'});
+    assert.equal(run.status, 0, run.stderr);
+    const html = readFileSync(prefix + '.html', 'utf8');
+    assert.equal(html.includes('<script>'), false);
+    const charts = [...html.matchAll(/<svg\b.*?<\/svg>/gs)].map(match => match[0]);
+    assert.match(html, /\.plot\{overflow-x:auto\}.*svg\{display:block;width:960px;max-width:none;height:auto\}/);
+    assert.equal([...html.matchAll(/class="plot" tabindex="0" role="region"/g)].length, 4);
+    assert.deepEqual(charts.map(chart => [...chart.matchAll(/<circle /g)].length), [2, 4, 1, 1]);
+    assert.doesNotMatch(charts[0], /1\.00 h · 2</);
+    assert.match(charts[1], /1\.00 h · 2</);
+    assert.match(charts[0], /H932\.00"/); // retain the best through the last observed regression
+  } finally { rmSync(dir, {recursive: true, force: true}); }
+});