feat(evals): grade & report metric-tool adoption (#2694)

## Summary



The metrics runs needs to know whether the agent actually *reaches for the metric tools*, separately from whether it solves the incident. Grading only ever looked at the final answer text, so tool usage was invisible. This adds an **adoption** signal end-to-end — graded, reported, and shown in the viewer — while keeping it strictly out of the outcome score.

- New transcript-aware check kind: rubric regexes run against a serialized tool-call transcript (names + args) via `serializeTranscript()` / `runTranscriptChecks()`; optional `rubric.transcript` block parsed in `rubric.ts`, results stored in `GradeRecord.adoption`.
- `adoption` is **excluded from `combinedScore`**.
- Sample rubric on `metric-saturation` (`used_metric_tool`, `described_jvm_memory_metric`, `described_gc_pause_metric`); `used_metric_tool` = `clickstack_(list|describe)_metric|metricType`, so metric reads via `clickstack_timeseries` count without false-positiving trace/log charts.
- Reports: `aggregate.ts` emits `CellSummary.adoption {mean, perCheck}` + `DeltaSummary.adoptionScore` to `_summary.json`; `markdown.ts` adds an "Adoption (tool use)" row + "Adoption per-check (usage rate)" table to `_summary.md`.
- Viewer: adoption section in the run grading tab, `adopt%` badges in nav/summary, and adoption row + per-check heatmap in the summary dashboard.

### Results (`metric-saturation`, opus-4-6, n=3/arm)
| Signal | clickhouse | hyperdx |
|---|---|---|
| **Adoption (mean)** | **44%** | **100%** |
| ↳ `used_metric_tool` | 0% | 100% |
| ↳ `described_jvm_memory_metric` | 33% | 100% |
| ↳ `described_gc_pause_metric` | 100% | 100% |
| Combined (outcome) | ~97% | ~93% |

The metric-tool arm used them in every run; the raw-SQL arm never invoked a metric tool and only incidentally hit metric names via SQL. Outcome stays comparable across arms — adoption diverges sharply while `combinedScore` does not.

### Screenshots
<img width="1002" height="176" alt="Screenshot 2026-07-21 at 15 23 00" src="https://github.com/user-attachments/assets/9ccc4bd8-511e-409f-87f8-5cd0ca21ecbd" />
<img width="1185" height="163" alt="Screenshot 2026-07-21 at 15 22 45" src="https://github.com/user-attachments/assets/3d2ee2ea-a54e-429f-90b9-c05128c5a1f8" />



### References



- Linear Issue: Closes HDX-4786, Closes HDX-4785

Author: karl-power

.changeset/hungry-chefs-wink.md

@@ -0,0 +1,5 @@
+---
+"@hyperdx/hdx-eval": patch
+---
+
+feat(evals): grade & report metric-tool adoption

packages/hdx-eval/scripts/viewer/public/app.js

@@ -293,6 +293,17 @@
             r.toolErrors > 0
               ? el('span', { class: 'tag error' }, `${r.toolErrors}!`)
               : null;
+          const adoptBadge =
+            r.adoptionScore != null
+              ? el(
+                  'span',
+                  {
+                    class: 'tag adopt',
+                    title: 'metric-tool adoption (not part of combined score)',
+                  },
+                  `adopt ${pct(r.adoptionScore)}`,
+                )
+              : null;
           const termBadge =
             r.termination && r.termination !== 'final_answer'
               ? el('span', { class: 'tag' }, r.termination)
@@ -310,7 +321,7 @@
               ' ',
               el('span', { class: 'muted small' }, `${r.toolCalls ?? '?'} calls`),
             ),
-            el('span', {}, termBadge, ' ', errBadge, ' ', chip),
+            el('span', {}, termBadge, ' ', adoptBadge, ' ', errBadge, ' ', chip),
           );
           mNode.appendChild(node);
         }
@@ -350,6 +361,11 @@
         'combined',
         grade?.combinedScore != null ? grade.combinedScore.toFixed(3) : '—',
       ],
+      // Adoption is only present when the scenario rubric defines transcript
+      // checks; omit the field entirely otherwise to avoid a dangling '—'.
+      ...(grade?.adoption
+        ? [['adoption', pct(grade.adoption.score)]]
+        : []),
     ];
     for (const [k, v] of fields) {
       hdr.appendChild(
@@ -638,6 +654,48 @@
       pane.appendChild(checks);
     }
 
+    // Adoption checks (transcript-aware tool usage). Reported alongside the
+    // outcome score but intentionally excluded from the combined score.
+    if (g.adoption?.hits) {
+      const checks = el('div', { class: 'grade-section' });
+      checks.appendChild(
+        el(
+          'h3',
+          {},
+          'Adoption (tool usage)',
+          el('span', { class: 'score' }, fmtScore(g.adoption.score)),
+        ),
+      );
+      for (const h of g.adoption.hits) {
+        const good = !!h.satisfied;
+        checks.appendChild(
+          el(
+            'div',
+            { class: 'check' + (h.negative ? ' negative' : '') },
+            el(
+              'div',
+              { class: 'marker ' + (good ? 'good' : 'bad') },
+              good ? '✓' : '✗',
+            ),
+            el('div', { class: 'id' }, h.id),
+            el(
+              'div',
+              { class: 'weight' },
+              `w${h.weight}${h.matched != null ? ` · matched=${h.matched}` : ''}`,
+            ),
+          ),
+        );
+      }
+      checks.appendChild(
+        el(
+          'div',
+          { class: 'muted small', style: 'margin-top:0.6rem; font-size:11px' },
+          'Regexes run against the serialized tool-call transcript (tool names + args). Adoption is reported but NOT part of the combined score.',
+        ),
+      );
+      pane.appendChild(checks);
+    }
+
     // Judge scores
     if (g.judge?.scores) {
       const judge = el('div', { class: 'grade-section' });
@@ -857,6 +915,14 @@
     const section = el('div', { class: 'summary-section scenario-breakdown' });
     section.appendChild(el('h2', {}, scenario.scenario));
 
+    // Forward-compat: the aggregate/summary layer (reports/aggregate.ts) does
+    // not yet emit adoption stats (that's HDX-4785). Only surface the adoption
+    // metric row + per-check table when the summary actually carries the data,
+    // so the viewer lights up automatically once aggregation lands.
+    const hasAdoption = Object.values(scenario.cells || {}).some(
+      (c) => c?.adoption != null,
+    );
+
     // Metrics table
     const metrics = [
       {
@@ -873,6 +939,17 @@
         dKey: 'programmaticScore',
         dFmt: signedPct,
       },
+      ...(hasAdoption
+        ? [
+            {
+              label: 'Adoption (tool use)',
+              get: (c) => c?.adoption?.mean,
+              fmt: pct,
+              dKey: 'adoptionScore',
+              dFmt: signedPct,
+            },
+          ]
+        : []),
       {
         label: 'Judge (weighted)',
         get: (c) => c?.judge?.weightedMean,
@@ -1049,6 +1126,51 @@
       );
     }
 
+    // Adoption per-check (forward-compatible: only when the summary carries
+    // adoption.perCheck, populated once HDX-4785 extends aggregate.ts).
+    const allAdoptionChecks = new Set();
+    for (const cell of Object.values(scenario.cells || {})) {
+      if (cell?.adoption?.perCheck) {
+        for (const id of Object.keys(cell.adoption.perCheck))
+          allAdoptionChecks.add(id);
+      }
+    }
+    if (allAdoptionChecks.size > 0) {
+      const aHeader = [el('th', {}, 'Adoption check')];
+      for (const m of mcpOrder) aHeader.push(el('th', {}, m));
+      const aRows = [...allAdoptionChecks].sort().map((id) => {
+        const cells = [el('td', { class: 'metric-label' }, id)];
+        for (const m of mcpOrder) {
+          const v = scenario.cells?.[m]?.adoption?.perCheck?.[id];
+          const cls =
+            v != null
+              ? v >= 0.99
+                ? 'check-pass'
+                : v >= 0.5
+                  ? 'check-partial'
+                  : 'check-fail'
+              : '';
+          cells.push(
+            el('td', { class: `heatmap-cell ${cls}` }, v != null ? pct(v) : '—'),
+          );
+        }
+        return el('tr', {}, ...cells);
+      });
+      section.appendChild(
+        el(
+          'div',
+          { class: 'sub-table' },
+          el('h3', {}, 'Adoption per-check (usage rate)'),
+          el(
+            'table',
+            { class: 'summary-table heatmap' },
+            el('thead', {}, el('tr', {}, ...aHeader)),
+            el('tbody', {}, ...aRows),
+          ),
+        ),
+      );
+    }
+
     // Side-by-side run comparison
     const scenarioCells = state.cells.filter((c) => c.scenario === scenario.scenario);
     if (scenarioCells.length > 0) {
@@ -1075,6 +1197,17 @@
             r.toolErrors > 0
               ? el('span', { class: 'tag error small' }, `${r.toolErrors}err`)
               : null;
+          const adoptBit =
+            r.adoptionScore != null
+              ? el(
+                  'span',
+                  {
+                    class: 'tag adopt small',
+                    title: 'metric-tool adoption',
+                  },
+                  `adopt ${pct(r.adoptionScore)}`,
+                )
+              : null;
           const termBit =
             r.termination !== 'final_answer'
               ? el('span', { class: 'tag small' }, r.termination)
@@ -1090,6 +1223,7 @@
               ' ',
               el('span', { class: 'muted small' }, `${r.toolCalls ?? '?'}c ${fmtMs(r.durationMs)}`),
               termBit ? el('span', {}, ' ', termBit) : null,
+              adoptBit ? el('span', {}, ' ', adoptBit) : null,
               errBit ? el('span', {}, ' ', errBit) : null,
             ),
           );

packages/hdx-eval/scripts/viewer/public/style.css

@@ -473,6 +473,10 @@ pre.code.error {
   font-size: 10px;
   padding: 0 0.3rem;
 }
+.tag.adopt {
+  color: var(--accent);
+  border-color: var(--accent);
+}
 .small {
   font-size: 11px;
 }

packages/hdx-eval/scripts/viewer/server.js

@@ -83,6 +83,9 @@ function collectRuns(cellDir) {
       idx: i,
       combinedScore: grade?.combinedScore ?? null,
       programmaticScore: grade?.programmatic?.score ?? null,
+      // Transcript-aware tool-adoption score (null when the scenario rubric
+      // has no `transcript` block, so the grade has no `adoption`).
+      adoptionScore: grade?.adoption?.score ?? null,
       judgeScore: grade?.judge?.weightedScore ?? null,
       termination: traj?.termination ?? null,
       durationMs: traj?.durationMs ?? null,

packages/hdx-eval/src/__tests__/aggregate.test.ts

@@ -17,6 +17,8 @@ function pair(args: {
   durationMs: number;
   model?: string;
   plugin?: string;
+  /** When set, attaches a transcript-aware adoption block to the grade. */
+  adoption?: { score: number; hits: Array<{ id: string; satisfied: boolean }> };
 }): GradedRunPair {
   const run: RunRecord = {
     schemaVersion: 1,
@@ -98,6 +100,17 @@ function pair(args: {
     gradedAt: '',
     judgeModel: 'claude-opus-4-7',
   };
+  if (args.adoption) {
+    grade.adoption = {
+      score: args.adoption.score,
+      hits: args.adoption.hits.map(h => ({
+        id: h.id,
+        weight: 1,
+        matched: h.satisfied,
+        satisfied: h.satisfied,
+      })),
+    };
+  }
   return { run, grade };
 }
 
@@ -448,4 +461,100 @@ describe('buildAggregate', () => {
     expect(summary.multiPlugin).toBe(false);
     expect(summary.columnOrder).toEqual(['clickhouse', 'hyperdx']);
   });
+
+  it('aggregates adoption mean + per-check usage rate with a baseline delta', () => {
+    const base = {
+      scenario: 'metric-saturation',
+      programmaticScore: 1,
+      judgeScore: 1,
+      toolCalls: 10,
+      outputTokens: 3000,
+      durationMs: 40_000,
+    };
+    const pairs: GradedRunPair[] = [
+      // baseline (clickhouse): no metric tools used.
+      pair({
+        ...base,
+        mcp: 'clickhouse',
+        i: 0,
+        adoption: {
+          score: 0,
+          hits: [
+            { id: 'used_metric_tool', satisfied: false },
+            { id: 'named_jvm_memory', satisfied: false },
+          ],
+        },
+      }),
+      // challenger (hyperdx): run 0 fully adopts, run 1 partially adopts.
+      pair({
+        ...base,
+        mcp: 'hyperdx',
+        i: 0,
+        adoption: {
+          score: 1,
+          hits: [
+            { id: 'used_metric_tool', satisfied: true },
+            { id: 'named_jvm_memory', satisfied: true },
+          ],
+        },
+      }),
+      pair({
+        ...base,
+        mcp: 'hyperdx',
+        i: 1,
+        adoption: {
+          score: 0.5,
+          hits: [
+            { id: 'used_metric_tool', satisfied: true },
+            { id: 'named_jvm_memory', satisfied: false },
+          ],
+        },
+      }),
+    ];
+    const summary = buildAggregate({ batchDir: '/tmp/x', pairs });
+    const sc = summary.scenarios[0];
+    const h = sc.cells['hyperdx']!;
+    const c = sc.cells['clickhouse']!;
+    // Mean adoption score across the two hyperdx runs: (1 + 0.5) / 2.
+    expect(h.adoption?.mean).toBeCloseTo(0.75, 5);
+    expect(c.adoption?.mean).toBeCloseTo(0, 5);
+    // Per-check usage rate: used_metric_tool hit in both hyperdx runs (1.0),
+    // named_jvm_memory only in run 0 (0.5).
+    expect(h.adoption?.perCheck['used_metric_tool']).toBeCloseTo(1, 5);
+    expect(h.adoption?.perCheck['named_jvm_memory']).toBeCloseTo(0.5, 5);
+    expect(c.adoption?.perCheck['used_metric_tool']).toBeCloseTo(0, 5);
+    // Delta = challenger - baseline.
+    expect(sc.deltas['hyperdx'].adoptionScore).toBeCloseTo(0.75, 5);
+  });
+
+  it('omits adoption from cells and deltas when no grade carries it', () => {
+    const pairs: GradedRunPair[] = [
+      pair({
+        scenario: 'error-root-cause',
+        mcp: 'clickhouse',
+        i: 0,
+        programmaticScore: 1,
+        judgeScore: 1,
+        toolCalls: 7,
+        outputTokens: 1500,
+        durationMs: 30_000,
+      }),
+      pair({
+        scenario: 'error-root-cause',
+        mcp: 'hyperdx',
+        i: 0,
+        programmaticScore: 1,
+        judgeScore: 0.9,
+        toolCalls: 13,
+        outputTokens: 4000,
+        durationMs: 60_000,
+      }),
+    ];
+    const summary = buildAggregate({ batchDir: '/tmp/x', pairs });
+    const sc = summary.scenarios[0];
+    expect(sc.cells['hyperdx'].adoption).toBeUndefined();
+    expect(sc.cells['clickhouse'].adoption).toBeUndefined();
+    // The delta object should not carry an adoptionScore key at all.
+    expect('adoptionScore' in sc.deltas['hyperdx']).toBe(false);
+  });
 });