Repository navigation
✨ Spend and contributor run stats by model: add the canonical model name #18781
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
5387a33
4a4383d
161fc76
98bd408
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1 @@ | ||
| - Spend by model and contributor run stats by model now carry `canonical_model`, the same folded model id the Governor's PRs-by-model card uses, so `claude-opus-5-5`, `anthropic/claude-opus-5-5` and `claude-opus-5.5` can be lined up across all three views. `/api/cost` `estimated.by_model[]` and every `/api/contribute/run-stats` and `/api/contribute/run-stats/models` row gain the field; spend stays one row per raw model, since prices are per raw model. Both run-stats endpoints accept an optional `group=canonical` that merges spellings into one row (runs summed, rates recomputed from the summed counts) with `raw_models[]` keeping each spelling's run count; without it the rows are unchanged. The Cost card's by-model list and the contributor wall's model evidence show the canonical id, with the reported spelling in a tooltip (#18718). |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,70 @@ | ||
| 'use strict'; | ||
|
|
||
| // #18718: the Cost card's by-model list and the contributor wall's model | ||
| // evidence name the canonical model id (modelfamily.Canonical, the id the | ||
| // Governor's PRs-by-model card groups by), with the raw spelling in a tooltip. | ||
|
|
||
| const { test } = require('node:test'); | ||
| const assert = require('node:assert/strict'); | ||
| const fs = require('node:fs'); | ||
| const path = require('node:path'); | ||
| const vm = require('node:vm'); | ||
|
|
||
| const dashboardDir = path.join(__dirname, '..', '..', 'src', 'pkg', 'dashboard'); | ||
| const indexHtml = fs.readFileSync(path.join(dashboardDir, 'static', 'index.html'), 'utf8'); | ||
| const costJs = fs.readFileSync(path.join(dashboardDir, 'static-src', 'js', 'cost.js'), 'utf8'); | ||
| const landingGo = fs.readFileSync(path.join(dashboardDir, 'contribute_landing.go'), 'utf8'); | ||
|
|
||
| function extractCostHelpers(src) { | ||
| const start = src.indexOf('const costModelCanonical = m =>'); | ||
| const end = src.indexOf('const costModelTitle = m =>', start); | ||
| assert.ok(start >= 0 && end > start, 'cost.js canonical helpers not found'); | ||
| const lineEnd = src.indexOf('\n', end); | ||
| return src.slice(start, lineEnd); | ||
| } | ||
|
|
||
| function costHelpers() { | ||
| const ctx = { | ||
| esc: s => String(s).replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>').replace(/"/g, '"'), | ||
| }; | ||
| vm.createContext(ctx); | ||
| vm.runInContext(`${extractCostHelpers(costJs)}\nthis.label = costModelLabel; this.title = costModelTitle;`, ctx); | ||
| return ctx; | ||
| } | ||
|
|
||
| test('Cost card by-model row shows the canonical id with the raw name in the tooltip', () => { | ||
| const h = costHelpers(); | ||
| const row = { name: 'anthropic/claude-opus-5-5', canonical_model: 'claude-opus-5.5' }; | ||
| assert.equal(h.label(row), 'claude-opus-5.5'); | ||
| assert.equal(h.title(row), ' title="Reported as anthropic/claude-opus-5-5"'); | ||
| }); | ||
|
|
||
| test('Cost card keeps the raw name when there is no distinct canonical id', () => { | ||
| const h = costHelpers(); | ||
| assert.equal(h.label({ name: 'claude-opus-5.5', canonical_model: 'claude-opus-5.5' }), 'claude-opus-5.5'); | ||
| assert.equal(h.title({ name: 'claude-opus-5.5', canonical_model: 'claude-opus-5.5' }), ''); | ||
| assert.equal(h.label({ name: 'old-model' }), 'old-model'); | ||
| assert.equal(h.title({ name: 'old-model' }), ''); | ||
| assert.equal(h.label({ name: 'auto', canonical_model: 'unknown' }), 'auto'); | ||
| assert.equal(h.label({}), 'unknown'); | ||
| }); | ||
|
|
||
| test('Cost card wiring is in both static-src and the generated index.html', () => { | ||
| for (const expected of [ | ||
| 'const costModelCanonical = m =>', | ||
| '<td class="cmodel"${costModelTitle(m)}>${esc(costModelLabel(m))}</td>', | ||
| ]) { | ||
| assert.ok(costJs.includes(expected), `cost.js missing ${expected}`); | ||
| assert.ok(indexHtml.includes(expected), `index.html missing ${expected}`); | ||
| } | ||
| }); | ||
|
|
||
| test('Contributor wall model evidence names the canonical model with the raw spelling in a tooltip', () => { | ||
| for (const expected of [ | ||
| 'p.model_evidence.canonical_model', | ||
| 'class="wall-canonical-model" title="Reported as \'+esc(evRaw)+\'">\'+esc(evCanon)+\'</span>', | ||
| "evidence='<div class=\"ops-note\">model evidence'+evName+': '", | ||
| ]) { | ||
| assert.ok(landingGo.includes(expected), `contribute_landing.go missing ${expected}`); | ||
| } | ||
| }); |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -13,6 +13,7 @@ import ( | |
| "time" | ||
|
|
||
| "github.com/hivecommons/hive/pkg/config" | ||
| "github.com/hivecommons/hive/pkg/modelfamily" | ||
| ) | ||
|
|
||
| const ( | ||
|
|
@@ -84,6 +85,16 @@ type WallModelStats struct { | |
| Abandoned int `json:"abandoned"` | ||
| Scenarios map[string]int `json:"scenarios"` | ||
| FailureKinds WallModelFailureKinds `json:"failure_kinds"` | ||
| // CanonicalModel is modelfamily.Canonical(Model), the id the Governor's | ||
| // PRs-by-model card groups by, so run stats line up with it (#18718). | ||
| CanonicalModel string `json:"canonical_model"` | ||
| // RawModels lists the raw spellings folded into this row, each with its | ||
| // own run count. Set only when the caller asks for group=canonical. | ||
| RawModels []WallRawModelRuns `json:"raw_models,omitempty"` | ||
|
|
||
| // verifiedRuns and failedRuns are the counts behind the two rates, kept | ||
| // so group=canonical can recompute the rates exactly. | ||
| verifiedRuns, failedRuns int | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
|
||
| } | ||
|
|
||
| // WallModelFailureKinds counts a model's failed runs by the failure kind the | ||
|
|
@@ -679,6 +690,8 @@ func readTaskRunModelStats(path string, window time.Duration) (map[string]WallMo | |
| a.VerifiedPRShare = float64(a.verified) / float64(a.Runs) | ||
| a.FailureRate = float64(a.failed) / float64(a.Runs) | ||
| } | ||
| a.CanonicalModel = modelfamily.Canonical(a.Model) | ||
| a.verifiedRuns, a.failedRuns = a.verified, a.failed | ||
| out[k] = a.WallModelStats | ||
| } | ||
| return out, nil | ||
|
|
@@ -705,5 +718,9 @@ func (s *Server) handleContributeRunStatsByModel(w http.ResponseWriter, r *http. | |
| http.Error(w, "task-run log unreadable", http.StatusInternalServerError) | ||
| return | ||
| } | ||
| jsonResponse(w, map[string]any{"window_days": days, "models": wallStatsList(stats)}) | ||
| models := wallStatsList(stats) | ||
| if runStatsGroupCanonical(r) { | ||
| models = groupWallModelStatsByCanonical(models) | ||
| } | ||
| jsonResponse(w, map[string]any{"window_days": days, "models": models}) | ||
| } | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Still present: this names the canonical model, but
model_evidencecomes fromcontribute_wall.go:299-303, which looks up one raw spelling's stats. The evidence line therefore undercounts runs relative to the canonical id it displays. Build the evidence lookup fromgroupWallModelStatsByCanonicalkeyed by canonical id (or drop the canonical label).