diff --git a/changelog.d/added-18718-canonical-model-spend-runstats.md b/changelog.d/added-18718-canonical-model-spend-runstats.md new file mode 100644 index 0000000000..1edab6f024 --- /dev/null +++ b/changelog.d/added-18718-canonical-model-spend-runstats.md @@ -0,0 +1 @@ +- Spend by model and contributor run stats by model now carry `canonical_model`, the same folded model id the Governor's PRs-by-model card uses, so `claude-opus-5-5`, `anthropic/claude-opus-5-5` and `claude-opus-5.5` can be lined up across all three views. `/api/cost` `estimated.by_model[]` and every `/api/contribute/run-stats` and `/api/contribute/run-stats/models` row gain the field; spend stays one row per raw model, since prices are per raw model. Both run-stats endpoints accept an optional `group=canonical` that merges spellings into one row (runs summed, rates recomputed from the summed counts) with `raw_models[]` keeping each spelling's run count; without it the rows are unchanged. The Cost card's by-model list and the contributor wall's model evidence show the canonical id, with the reported spelling in a tooltip (#18718). diff --git a/dashboard/openapi.json b/dashboard/openapi.json index 20b0fd2483..b9a9112c93 100644 --- a/dashboard/openapi.json +++ b/dashboard/openapi.json @@ -22826,6 +22826,18 @@ "default": 7 }, "description": "Trailing window in days; 0 aggregates everything in the live log." + }, + { + "name": "group", + "in": "query", + "required": false, + "schema": { + "type": "string", + "enum": [ + "canonical" + ] + }, + "description": "canonical merges model rows by canonical_model (modelfamily.Canonical): runs summed, rates recomputed from summed counts, model set to the canonical id, raw_models[] listing each raw spelling with its runs. Omitted: one row per raw model, each with canonical_model (#18718)." } ], "responses": { @@ -38083,7 +38095,21 @@ "Contribute" ], "summary": "Contributor run stats by model", - "description": "Returns per-model run counts, verified-PR share, and failure rate for contributor-wall evidence, plus why runs ended (#18721): `abandoned` (count), `scenarios` (scenario -> count, the same keys as /api/contribute/run-stats backends[].scenarios, summing to runs) and `failure_kinds` ({environment, task, unspecified}, summing to the failed runs behind failure_rate).", + "description": "Returns per-model run counts, verified-PR share, failure rate and canonical_model for contributor-wall evidence, plus why runs ended (#18721): `abandoned` (count), `scenarios` (scenario -> count, the same keys as /api/contribute/run-stats backends[].scenarios, summing to runs) and `failure_kinds` ({environment, task, unspecified}, summing to the failed runs behind failure_rate). group=canonical merges rows by canonical model id.", + "parameters": [ + { + "name": "group", + "in": "query", + "required": false, + "schema": { + "type": "string", + "enum": [ + "canonical" + ] + }, + "description": "canonical merges model rows by canonical_model (modelfamily.Canonical): runs, abandoned, scenarios and failure_kinds summed, rates recomputed from summed counts, model set to the canonical id, raw_models[] listing each raw spelling with its runs. Omitted: one row per raw model, each with canonical_model (#18718)." + } + ], "responses": { "200": { "description": "OK" diff --git a/dashboard/test/canonical-model-ui.test.js b/dashboard/test/canonical-model-ui.test.js new file mode 100644 index 0000000000..fbe82c4dc9 --- /dev/null +++ b/dashboard/test/canonical-model-ui.test.js @@ -0,0 +1,70 @@ +'use strict'; + +// #18718: the Cost card's by-model list and the contributor wall's model +// evidence name the canonical model id (modelfamily.Canonical, the id the +// Governor's PRs-by-model card groups by), with the raw spelling in a tooltip. + +const { test } = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const vm = require('node:vm'); + +const dashboardDir = path.join(__dirname, '..', '..', 'src', 'pkg', 'dashboard'); +const indexHtml = fs.readFileSync(path.join(dashboardDir, 'static', 'index.html'), 'utf8'); +const costJs = fs.readFileSync(path.join(dashboardDir, 'static-src', 'js', 'cost.js'), 'utf8'); +const landingGo = fs.readFileSync(path.join(dashboardDir, 'contribute_landing.go'), 'utf8'); + +function extractCostHelpers(src) { + const start = src.indexOf('const costModelCanonical = m =>'); + const end = src.indexOf('const costModelTitle = m =>', start); + assert.ok(start >= 0 && end > start, 'cost.js canonical helpers not found'); + const lineEnd = src.indexOf('\n', end); + return src.slice(start, lineEnd); +} + +function costHelpers() { + const ctx = { + esc: s => String(s).replace(/&/g, '&').replace(//g, '>').replace(/"/g, '"'), + }; + vm.createContext(ctx); + vm.runInContext(`${extractCostHelpers(costJs)}\nthis.label = costModelLabel; this.title = costModelTitle;`, ctx); + return ctx; +} + +test('Cost card by-model row shows the canonical id with the raw name in the tooltip', () => { + const h = costHelpers(); + const row = { name: 'anthropic/claude-opus-5-5', canonical_model: 'claude-opus-5.5' }; + assert.equal(h.label(row), 'claude-opus-5.5'); + assert.equal(h.title(row), ' title="Reported as anthropic/claude-opus-5-5"'); +}); + +test('Cost card keeps the raw name when there is no distinct canonical id', () => { + const h = costHelpers(); + assert.equal(h.label({ name: 'claude-opus-5.5', canonical_model: 'claude-opus-5.5' }), 'claude-opus-5.5'); + assert.equal(h.title({ name: 'claude-opus-5.5', canonical_model: 'claude-opus-5.5' }), ''); + assert.equal(h.label({ name: 'old-model' }), 'old-model'); + assert.equal(h.title({ name: 'old-model' }), ''); + assert.equal(h.label({ name: 'auto', canonical_model: 'unknown' }), 'auto'); + assert.equal(h.label({}), 'unknown'); +}); + +test('Cost card wiring is in both static-src and the generated index.html', () => { + for (const expected of [ + 'const costModelCanonical = m =>', + '${esc(costModelLabel(m))}', + ]) { + assert.ok(costJs.includes(expected), `cost.js missing ${expected}`); + assert.ok(indexHtml.includes(expected), `index.html missing ${expected}`); + } +}); + +test('Contributor wall model evidence names the canonical model with the raw spelling in a tooltip', () => { + for (const expected of [ + 'p.model_evidence.canonical_model', + 'class="wall-canonical-model" title="Reported as \'+esc(evRaw)+\'">\'+esc(evCanon)+\'', + "evidence='
model evidence'+evName+': '", + ]) { + assert.ok(landingGo.includes(expected), `contribute_landing.go missing ${expected}`); + } +}); diff --git a/src/docs/api-reference/contribute.md b/src/docs/api-reference/contribute.md index 2de912ceeb..044a063f77 100644 --- a/src/docs/api-reference/contribute.md +++ b/src/docs/api-reference/contribute.md @@ -21,7 +21,7 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/contribute/limits` | Public | Contribute Limits | `pkg/dashboard/api_contribute.go:275` | | `GET` | `/api/contribute/metrics` | Public | Contribute Metrics | `pkg/dashboard/api_contribute.go:281` | | `GET` | `/api/contribute/me` | Public path; caller identity resolved server-side (401 anonymous; a signed-in caller with no profile on this hive gets 200 with `has_profile:false` and no counters, [#12111](https://github.com/hivecommons/hive/issues/12111)) | The caller's OWN contribution stats — issues worked (24h and all-time), completions that produced a verified PR (24h and all-time, [#7894](https://github.com/hivecommons/hive/issues/7894)), and failures. No username parameter, so it only ever answers for its caller ([#6543](https://github.com/hivecommons/hive/issues/6543)) | `pkg/dashboard/api_contribute.go:291` | -| `GET` | `/api/contribute/run-stats/models` | Public | Per-model run evidence for the contributor wall: runs, verified-PR share, and failure rate over the selected window, plus why runs ended ([#18721](https://github.com/hivecommons/hive/issues/18721)): `abandoned`, `scenarios` (same keys as `/api/contribute/run-stats` `backends[].scenarios`, summing to `runs`) and `failure_kinds` (`environment`/`task`/`unspecified`, summing to the failed runs). `/api/contribute/run-stats` returns the same rows as `models`. | `pkg/dashboard/api_contribute.go` | +| `GET` | `/api/contribute/run-stats/models` | Public | Per-model run evidence for the contributor wall: runs, verified-PR share, and failure rate over the selected window, plus why runs ended ([#18721](https://github.com/hivecommons/hive/issues/18721)): `abandoned`, `scenarios` (same keys as `/api/contribute/run-stats` `backends[].scenarios`, summing to `runs`) and `failure_kinds` (`environment`/`task`/`unspecified`, summing to the failed runs). `/api/contribute/run-stats` returns the same rows as `models`. Each row also carries `canonical_model`, the id the Governor's PRs-by-model card groups by (`modelfamily.Canonical`). `group=canonical` merges rows by that id: `runs`, `abandoned`, `scenarios` and `failure_kinds` summed, rates recomputed from the summed counts, `model` set to the canonical id, and `raw_models[]` listing each raw spelling with its own `runs` ([#18718](https://github.com/hivecommons/hive/issues/18718)). Without `group` the rows are unchanged apart from the added `canonical_model`. | `pkg/dashboard/api_contribute.go` | | `GET` | `/api/contribute/wall` | Public | Contributor wall posts when the hive-wide wall switch is enabled; anonymous readers see visible posts, authors can still see their own hidden posts. | `pkg/dashboard/api_contribute.go` | | `POST` | `/api/contribute/wall` | Signed-in contributor | Create a 500-character plain-text wall post or one-level reply, with optional `model`, `backend`, and `repo` tags. Returns 401 anonymous, 403 when disabled/revoked/muted, and 429 when rate-limited. | `pkg/dashboard/api_contribute.go` | | `DELETE` | `/api/contribute/wall/{id}` | Post author | Author deletes their own wall post; action is audit logged. | `pkg/dashboard/api_contribute.go` | @@ -43,7 +43,7 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `POST` | `/api/contributors/{id}/requeue` | Dashboard auth/session | Contributor Requeue | `pkg/dashboard/api_contribute.go:347` | | `DELETE` | `/api/contributors/{id}` | Owner only | Contributor Delete | `pkg/dashboard/api_contribute.go:348` | | `GET` | `/contribute/dossier/{username}` | Public | Contributor Dossier Page (HTML) | `pkg/dashboard/api_contribute.go:240` | -| `GET` | `/api/contribute/run-stats` | Public | Contribute Run Stats | `pkg/dashboard/api_contribute.go:299` | +| `GET` | `/api/contribute/run-stats` | Public | Contribute Run Stats; its `models` array takes the same optional `group=canonical` as `/api/contribute/run-stats/models` and carries `canonical_model` on every row ([#18718](https://github.com/hivecommons/hive/issues/18718)) | `pkg/dashboard/api_contribute.go:299` | | `GET` | `/api/contribute/runs` | Public | Contribute Per-Run History | `pkg/dashboard/api_contribute.go:389` | | `GET` | `/api/contribute/decisions` | Owner/read-write | Contribute Hub Decisions | `pkg/dashboard/api_contribute.go:399` | | `GET` | `/api/contribute/dossier` | Public | Contribute Dossier Get | `pkg/dashboard/api_contribute.go:336` | diff --git a/src/docs/api-reference/cost-tokens-telemetry.md b/src/docs/api-reference/cost-tokens-telemetry.md index ee842ada0b..b538efcb7a 100644 --- a/src/docs/api-reference/cost-tokens-telemetry.md +++ b/src/docs/api-reference/cost-tokens-telemetry.md @@ -8,7 +8,7 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/trends` | Dashboard auth/session | Trends | `pkg/dashboard/api.go:87` | | `GET` | `/api/token-access` | Dashboard auth/session | Token Access | `pkg/dashboard/api.go:142` | | `GET` | `/api/tokens` | Dashboard auth/session | Tokens | `pkg/dashboard/api.go:143` | -| `GET` | `/api/cost` | Dashboard auth/session | Cost; `merged_prs` / `closed_issues` come from the webhook-incremented ForgeState counters reconciled hourly against search, with `count_stale`, `count_stale_since` and `count_source` (`forgestate` / `search`) when last-good counts are being served (#13376), `count_partial` when only some configured repos have an attributed baseline yet, and `count_error` naming why the latest attributed refresh failed or was partial (the reconcile asks GitHub about a budgeted slice of repos per pass so a many-repo hive stays inside search's rate limit); `daily_rollup` lists per-UTC-day estimated spend (`day`, `cost_usd`, `input_tokens`, `output_tokens`, `samples`) kept for ~2 years so the Cost chart can reach past the 30-day snapshot ring; `recording` reports `agent_models`, `unrecorded_models`, `recent_models`, `last_priced_growth_at`, `last_agent_model_growth_at`, `stale_after_ms` and `stale` so the card can flag stale live usage recording (#13485) | `pkg/dashboard/api.go:144` | +| `GET` | `/api/cost` | Dashboard auth/session | Cost; `merged_prs` / `closed_issues` come from the webhook-incremented ForgeState counters reconciled hourly against search, with `count_stale`, `count_stale_since` and `count_source` (`forgestate` / `search`) when last-good counts are being served (#13376), `count_partial` when only some configured repos have an attributed baseline yet, and `count_error` naming why the latest attributed refresh failed or was partial (the reconcile asks GitHub about a budgeted slice of repos per pass so a many-repo hive stays inside search's rate limit); `daily_rollup` lists per-UTC-day estimated spend (`day`, `cost_usd`, `input_tokens`, `output_tokens`, `samples`) kept for ~2 years so the Cost chart can reach past the 30-day snapshot ring; `recording` reports `agent_models`, `unrecorded_models`, `recent_models`, `last_priced_growth_at`, `last_agent_model_growth_at`, `stale_after_ms` and `stale` so the card can flag stale live usage recording (#13485); every `estimated.by_model[]` row carries `canonical_model` (`modelfamily.Canonical` of its `name`, the id PRs-by-model and run stats group by), while spend stays one row per raw model because prices are per raw model (#18718) | `pkg/dashboard/api.go:144` | | `GET` | `/api/repo-activity` | Dashboard auth/session | Per-repo audited activity | `pkg/dashboard/api.go` | | `GET` | `/api/repo-cost` | Dashboard auth/session | Per-repo estimated token cost (interval join, cached) | `pkg/dashboard/api.go` | | `GET` | `/api/launch-cost` | Dashboard auth/session | Per-issue / PR / head-SHA estimated token cost from kick launch attribution (cached; `?ref=owner/repo#N` narrows) | `pkg/dashboard/api.go` | diff --git a/src/docs/backend-smoke.md b/src/docs/backend-smoke.md index 29c421de55..ad4e902cf2 100644 --- a/src/docs/backend-smoke.md +++ b/src/docs/backend-smoke.md @@ -147,6 +147,12 @@ the failed runs). That is the split that tells "this model's contributors have broken environments" from "this model gets the work wrong". The contributor wall shows it as the tooltip on a post's "failed" model-evidence figure. +Each `models` row also carries `canonical_model`, the id the Governor's +PRs-by-model card uses. Add `group=canonical` to fold spellings such as +`claude-opus-5-5` and `anthropic/claude-opus-5-5` into one row (counts and the +#18721 splits summed, rates recomputed); `raw_models[]` keeps each spelling's +run count (#18718). + This is DECLARE-only telemetry, per the boundary `src/pkg/dashboard/contribute_protocol.go` documents: nothing routes, cooldowns, or gates on any of it, and a telemetry write failure never fails a diff --git a/src/pkg/dashboard/contribute_landing.go b/src/pkg/dashboard/contribute_landing.go index e1dfa99933..907c2c3338 100644 --- a/src/pkg/dashboard/contribute_landing.go +++ b/src/pkg/dashboard/contribute_landing.go @@ -6078,8 +6078,12 @@ function ccWallPostHTML(p){ var tags=p.tags||{}; var evidence=''; if(p.model_evidence){ + // Name the canonical model id (the one the Governor's PRs-by-model card + // groups by, #18718) when it differs from the tag, raw spelling in the tooltip. + var evCanon=String(p.model_evidence.canonical_model||''),evRaw=String(p.model_evidence.model||tags.model||''); + var evName=(evCanon&&evCanon!=='unknown'&&evCanon.toLowerCase()!==evRaw.toLowerCase())?' for '+esc(evCanon)+'':''; var failTip=ccModelEvidenceFailTip(p.model_evidence); - evidence='
model evidence: '+(p.model_evidence.runs||0)+' runs · '+Math.round((p.model_evidence.verified_pr_share||0)*100)+'%% verified PR · '+Math.round((p.model_evidence.failure_rate||0)*100)+'%% failed
'; + evidence='
model evidence'+evName+': '+(p.model_evidence.runs||0)+' runs · '+Math.round((p.model_evidence.verified_pr_share||0)*100)+'%% verified PR · '+Math.round((p.model_evidence.failure_rate||0)*100)+'%% failed
'; } var controls=''; if(ccMeUsername&&p.author&&p.author.toLowerCase()===ccMeUsername.toLowerCase())controls+=''; diff --git a/src/pkg/dashboard/contribute_wall.go b/src/pkg/dashboard/contribute_wall.go index 1cd0faf810..e628c64c7e 100644 --- a/src/pkg/dashboard/contribute_wall.go +++ b/src/pkg/dashboard/contribute_wall.go @@ -13,6 +13,7 @@ import ( "time" "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/modelfamily" ) const ( @@ -84,6 +85,16 @@ type WallModelStats struct { Abandoned int `json:"abandoned"` Scenarios map[string]int `json:"scenarios"` FailureKinds WallModelFailureKinds `json:"failure_kinds"` + // CanonicalModel is modelfamily.Canonical(Model), the id the Governor's + // PRs-by-model card groups by, so run stats line up with it (#18718). + CanonicalModel string `json:"canonical_model"` + // RawModels lists the raw spellings folded into this row, each with its + // own run count. Set only when the caller asks for group=canonical. + RawModels []WallRawModelRuns `json:"raw_models,omitempty"` + + // verifiedRuns and failedRuns are the counts behind the two rates, kept + // so group=canonical can recompute the rates exactly. + verifiedRuns, failedRuns int } // WallModelFailureKinds counts a model's failed runs by the failure kind the @@ -679,6 +690,8 @@ func readTaskRunModelStats(path string, window time.Duration) (map[string]WallMo a.VerifiedPRShare = float64(a.verified) / float64(a.Runs) a.FailureRate = float64(a.failed) / float64(a.Runs) } + a.CanonicalModel = modelfamily.Canonical(a.Model) + a.verifiedRuns, a.failedRuns = a.verified, a.failed out[k] = a.WallModelStats } return out, nil @@ -705,5 +718,9 @@ func (s *Server) handleContributeRunStatsByModel(w http.ResponseWriter, r *http. http.Error(w, "task-run log unreadable", http.StatusInternalServerError) return } - jsonResponse(w, map[string]any{"window_days": days, "models": wallStatsList(stats)}) + models := wallStatsList(stats) + if runStatsGroupCanonical(r) { + models = groupWallModelStatsByCanonical(models) + } + jsonResponse(w, map[string]any{"window_days": days, "models": models}) } diff --git a/src/pkg/dashboard/contribute_wall_canonical.go b/src/pkg/dashboard/contribute_wall_canonical.go new file mode 100644 index 0000000000..f4a2c0d5a0 --- /dev/null +++ b/src/pkg/dashboard/contribute_wall_canonical.go @@ -0,0 +1,76 @@ +package dashboard + +import ( + "net/http" + "sort" + "strings" + + "github.com/hivecommons/hive/pkg/modelfamily" +) + +// WallRawModelRuns is one raw model spelling folded into a canonical +// run-stats row, with its own run count (#18718). +type WallRawModelRuns struct { + Model string `json:"model"` + Runs int `json:"runs"` +} + +// runStatsGroupCanonical reports whether a run-stats request asked for +// group=canonical. Any other value, or none, keeps today's per-raw-model rows. +func runStatsGroupCanonical(r *http.Request) bool { + return r != nil && strings.EqualFold(strings.TrimSpace(r.URL.Query().Get("group")), "canonical") +} + +// groupWallModelStatsByCanonical merges run-stats rows that share a canonical +// model id (modelfamily.Canonical, the rule PRs-by-model uses). Runs and the +// verified and failed counts are summed and the two rates recomputed from the +// sums; abandoned, scenarios and failure_kinds (#18721) are summed too, so a merged row says exactly what one row over all its spellings +// would. Model becomes the canonical id; RawModels lists each spelling with +// its own runs, most runs first. Output is sorted by model, like wallStatsList. +func groupWallModelStatsByCanonical(rows []WallModelStats) []WallModelStats { + by := map[string]*WallModelStats{} + order := make([]string, 0, len(rows)) + for _, row := range rows { + canonical := row.CanonicalModel + if canonical == "" { + canonical = modelfamily.Canonical(row.Model) + } + g := by[canonical] + if g == nil { + g = &WallModelStats{Model: canonical, CanonicalModel: canonical, RawModels: []WallRawModelRuns{}} + by[canonical] = g + order = append(order, canonical) + } + g.Runs += row.Runs + g.verifiedRuns += row.verifiedRuns + g.failedRuns += row.failedRuns + g.Abandoned += row.Abandoned + for sc, n := range row.Scenarios { + if g.Scenarios == nil { + g.Scenarios = map[string]int{} + } + g.Scenarios[sc] += n + } + g.FailureKinds.Environment += row.FailureKinds.Environment + g.FailureKinds.Task += row.FailureKinds.Task + g.FailureKinds.Unspecified += row.FailureKinds.Unspecified + g.RawModels = append(g.RawModels, WallRawModelRuns{Model: row.Model, Runs: row.Runs}) + } + out := make([]WallModelStats, 0, len(order)) + for _, canonical := range order { + g := by[canonical] + if g.Runs > 0 { + g.VerifiedPRShare = float64(g.verifiedRuns) / float64(g.Runs) + g.FailureRate = float64(g.failedRuns) / float64(g.Runs) + } + sort.SliceStable(g.RawModels, func(i, j int) bool { + if g.RawModels[i].Runs != g.RawModels[j].Runs { + return g.RawModels[i].Runs > g.RawModels[j].Runs + } + return g.RawModels[i].Model < g.RawModels[j].Model + }) + out = append(out, *g) + } + sort.Slice(out, func(i, j int) bool { return strings.ToLower(out[i].Model) < strings.ToLower(out[j].Model) }) + return out +} diff --git a/src/pkg/dashboard/contribute_wall_canonical_test.go b/src/pkg/dashboard/contribute_wall_canonical_test.go new file mode 100644 index 0000000000..da21a0037b --- /dev/null +++ b/src/pkg/dashboard/contribute_wall_canonical_test.go @@ -0,0 +1,223 @@ +package dashboard + +import ( + "bytes" + "encoding/json" + "net/http/httptest" + "os" + "path/filepath" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/modelfamily" +) + +// writeCanonicalRunLog writes run records under several spellings of the +// same models: claude-opus-5-5 with and without a provider prefix, and the +// Ornith model in two cases (#18718). +func writeCanonicalRunLog(t *testing.T, dir string) { + t.Helper() + now := time.Now().UTC().Format(time.RFC3339) + recs := []TaskRunRecord{ + {TS: now, Model: "claude-opus-5-5", Outcome: outcomeCompleted, PRVerified: true}, + {TS: now, Model: "claude-opus-5-5", Outcome: outcomeFailed}, + {TS: now, Model: "claude-opus-5-5", Outcome: outcomeCompleted}, + {TS: now, Model: "anthropic/claude-opus-5-5", Outcome: outcomeCompleted, PRVerified: true}, + {TS: now, Model: "lemonade/Ornith-1.5-35B-A3B-GGUF-Q6_K", Outcome: outcomeFailed}, + {TS: now, Model: "lemonade/ornith-1.5-35b-a3b-gguf-q6_k", Outcome: outcomeCompleted}, + {TS: now, Model: "m2", Outcome: outcomeCompleted}, + } + var buf bytes.Buffer + for _, r := range recs { + b, _ := json.Marshal(r) + buf.Write(b) + buf.WriteByte('\n') + } + if err := os.WriteFile(filepath.Join(dir, taskRunLogFileName), buf.Bytes(), 0o600); err != nil { + t.Fatal(err) + } +} + +func runStatsModelRows(t *testing.T, s *Server, url string) []map[string]any { + t.Helper() + rr := httptest.NewRecorder() + s.handleContributeRunStatsByModel(rr, httptest.NewRequest("GET", url, nil)) + body := decodeWallResp(t, rr) + raw := body["models"].([]any) + out := make([]map[string]any, 0, len(raw)) + for _, m := range raw { + out = append(out, m.(map[string]any)) + } + return out +} + +func TestContributeRunStatsByModelCanonicalField(t *testing.T) { + s, dir := newWallTestServer(t, true) + writeCanonicalRunLog(t, dir) + rows := runStatsModelRows(t, s, "/api/contribute/run-stats/models?days=1") + if len(rows) != 4 { + t.Fatalf("ungrouped rows = %d, want 4 (raw spellings stay separate): %#v", len(rows), rows) + } + byModel := map[string]map[string]any{} + for _, row := range rows { + model := row["model"].(string) + byModel[model] = row + if got, want := row["canonical_model"], modelfamily.Canonical(model); got != want { + t.Errorf("%s canonical_model = %v, want %q", model, got, want) + } + if _, ok := row["raw_models"]; ok { + t.Errorf("%s: raw_models must be absent without group=canonical", model) + } + for _, k := range []string{"runs", "verified_pr_share", "failure_rate"} { + if _, ok := row[k].(float64); !ok { + t.Errorf("%s: existing field %s missing or retyped: %#v", model, k, row) + } + } + } + opus := byModel["claude-opus-5-5"] + if opus == nil || opus["runs"].(float64) != 3 || opus["verified_pr_share"].(float64) != 1.0/3 || opus["failure_rate"].(float64) != 1.0/3 { + t.Fatalf("claude-opus-5-5 row changed: %#v", opus) + } + if opus["canonical_model"] != "claude-opus-5.5" { + t.Fatalf("claude-opus-5-5 canonical = %v", opus["canonical_model"]) + } + if pfx := byModel["anthropic/claude-opus-5-5"]; pfx == nil || pfx["canonical_model"] != "claude-opus-5.5" || pfx["runs"].(float64) != 1 { + t.Fatalf("anthropic/claude-opus-5-5 row: %#v", pfx) + } + // The two Ornith cases were already one row (case-insensitive key); the + // first spelling seen is displayed, as before. + orn := byModel["lemonade/Ornith-1.5-35B-A3B-GGUF-Q6_K"] + if orn == nil || orn["runs"].(float64) != 2 || orn["canonical_model"] != "ornith-1.5-35b-a3b-gguf-q6-k" { + t.Fatalf("ornith row: %#v", orn) + } +} + +func TestContributeRunStatsByModelGroupCanonical(t *testing.T) { + s, dir := newWallTestServer(t, true) + writeCanonicalRunLog(t, dir) + plain := runStatsModelRows(t, s, "/api/contribute/run-stats/models?days=1") + grouped := runStatsModelRows(t, s, "/api/contribute/run-stats/models?days=1&group=canonical") + if len(grouped) != 3 { + t.Fatalf("grouped rows = %d, want 3: %#v", len(grouped), grouped) + } + sum := func(rows []map[string]any) float64 { + n := 0.0 + for _, r := range rows { + n += r["runs"].(float64) + } + return n + } + if sum(plain) != sum(grouped) || sum(grouped) != 7 { + t.Fatalf("runs: plain=%v grouped=%v, want both 7", sum(plain), sum(grouped)) + } + byModel := map[string]map[string]any{} + for _, row := range grouped { + if row["model"] != row["canonical_model"] { + t.Errorf("grouped row model %v != canonical_model %v", row["model"], row["canonical_model"]) + } + byModel[row["model"].(string)] = row + } + opus := byModel["claude-opus-5.5"] + if opus == nil { + t.Fatalf("no claude-opus-5.5 row: %#v", grouped) + } + // 4 runs: 2 verified (one per spelling), 1 failed. Rates are the + // summed-count ratios, not an average of the per-spelling rates. + if opus["runs"].(float64) != 4 || opus["verified_pr_share"].(float64) != 0.5 || opus["failure_rate"].(float64) != 0.25 { + t.Fatalf("claude-opus-5.5 grouped stats: %#v", opus) + } + raws := opus["raw_models"].([]any) + if len(raws) != 2 { + t.Fatalf("raw_models = %#v", raws) + } + first, second := raws[0].(map[string]any), raws[1].(map[string]any) + if first["model"] != "claude-opus-5-5" || first["runs"].(float64) != 3 || second["model"] != "anthropic/claude-opus-5-5" || second["runs"].(float64) != 1 { + t.Fatalf("raw_models order/counts = %#v", raws) + } + orn := byModel["ornith-1.5-35b-a3b-gguf-q6-k"] + if orn == nil || orn["runs"].(float64) != 2 || orn["failure_rate"].(float64) != 0.5 { + t.Fatalf("ornith grouped: %#v", orn) + } + if m2 := byModel["m2"]; m2 == nil || m2["runs"].(float64) != 1 || len(m2["raw_models"].([]any)) != 1 { + t.Fatalf("m2 grouped: %#v", m2) + } + + // Any other group value keeps the ungrouped rows. + other := runStatsModelRows(t, s, "/api/contribute/run-stats/models?days=1&group=raw") + if len(other) != len(plain) { + t.Fatalf("group=raw rows = %d, want %d", len(other), len(plain)) + } +} + +func TestContributeRunStatsModelsGroupCanonical(t *testing.T) { + s, dir := newWallTestServer(t, true) + writeCanonicalRunLog(t, dir) + get := func(url string) []any { + rr := httptest.NewRecorder() + s.handleContributeRunStats(rr, httptest.NewRequest("GET", url, nil)) + return decodeWallResp(t, rr)["models"].([]any) + } + if n := len(get("/api/contribute/run-stats?days=1")); n != 4 { + t.Fatalf("run-stats models ungrouped = %d, want 4", n) + } + grouped := get("/api/contribute/run-stats?days=1&group=canonical") + if len(grouped) != 3 { + t.Fatalf("run-stats models grouped = %d, want 3", len(grouped)) + } + for _, m := range grouped { + row := m.(map[string]any) + if _, ok := row["raw_models"]; !ok { + t.Errorf("grouped run-stats row lacks raw_models: %#v", row) + } + } +} + +func TestGroupWallModelStatsByCanonicalUnit(t *testing.T) { + rows := []WallModelStats{ + {Model: "openai-codex/gpt-6.1-sol", Runs: 2, CanonicalModel: "gpt-6.1-sol", verifiedRuns: 2}, + {Model: "gpt-6.1-sol", Runs: 2, CanonicalModel: "gpt-6.1-sol", failedRuns: 1}, + {Model: "openai/gpt-6.1-sol", Runs: 0}, + } + out := groupWallModelStatsByCanonical(rows) + if len(out) != 1 { + t.Fatalf("out = %#v", out) + } + g := out[0] + if g.Model != "gpt-6.1-sol" || g.Runs != 4 || g.VerifiedPRShare != 0.5 || g.FailureRate != 0.25 || len(g.RawModels) != 3 { + t.Fatalf("grouped = %#v", g) + } + if g.RawModels[0].Model != "gpt-6.1-sol" || g.RawModels[1].Model != "openai-codex/gpt-6.1-sol" || g.RawModels[2].Runs != 0 { + t.Fatalf("raw order = %#v", g.RawModels) + } + if got := groupWallModelStatsByCanonical(nil); len(got) != 0 { + t.Fatalf("nil input = %#v", got) + } +} + +// group=canonical sums the #18721 run-cause split along with the counts, so a +// merged row's scenarios still sum to runs and failure_kinds to the failures. +func TestGroupWallModelStatsByCanonicalSumsRunCauses(t *testing.T) { + rows := []WallModelStats{ + {Model: "claude-opus-5-5", Runs: 3, CanonicalModel: "claude-opus-5-5", failedRuns: 1, Abandoned: 1, + Scenarios: map[string]int{scenarioEnvFailure: 1, "abandoned_handback": 1, "clean_complete": 1}, + FailureKinds: WallModelFailureKinds{Environment: 1}}, + {Model: "anthropic/claude-opus-5-5", Runs: 2, CanonicalModel: "claude-opus-5-5", failedRuns: 1, + Scenarios: map[string]int{scenarioTaskFailure: 1, "clean_complete": 1}, + FailureKinds: WallModelFailureKinds{Task: 1}}, + } + out := groupWallModelStatsByCanonical(rows) + if len(out) != 1 { + t.Fatalf("out = %#v", out) + } + g := out[0] + if g.Abandoned != 1 || g.FailureKinds != (WallModelFailureKinds{Environment: 1, Task: 1}) { + t.Fatalf("grouped causes = %#v", g) + } + sum := 0 + for _, n := range g.Scenarios { + sum += n + } + if sum != g.Runs || g.Scenarios["clean_complete"] != 2 { + t.Fatalf("scenarios = %#v, runs %d", g.Scenarios, g.Runs) + } +} diff --git a/src/pkg/dashboard/cost.go b/src/pkg/dashboard/cost.go index 0d02308ff3..9a5487f035 100644 --- a/src/pkg/dashboard/cost.go +++ b/src/pkg/dashboard/cost.go @@ -16,6 +16,7 @@ import ( "github.com/hivecommons/hive/pkg/config" "github.com/hivecommons/hive/pkg/dashboard/collect" ghpkg "github.com/hivecommons/hive/pkg/github" + "github.com/hivecommons/hive/pkg/modelfamily" "github.com/hivecommons/hive/pkg/tokens" ) @@ -125,6 +126,10 @@ type costModelEntry struct { Output int64 `json:"output"` CacheRead int64 `json:"cache_read"` CacheCreate int64 `json:"cache_create"` + // CanonicalModel is modelfamily.Canonical(Name) on by_model rows, the id + // PRs-by-model and run stats group by (#18718). Empty on by_agent rows. + // Spend is still one row per raw model: prices are per raw model. + CanonicalModel string `json:"canonical_model,omitempty"` } type costEstimated struct { @@ -537,13 +542,14 @@ func flattenEstimated(est tokens.EstimatedCost) costEstimated { } for name, mc := range est.ByModel { out.ByModel = append(out.ByModel, costModelEntry{ - Name: name, - USD: mc.USD, - Source: sourceForCost(mc), - Input: mc.Input, - Output: mc.Output, - CacheRead: mc.CacheRead, - CacheCreate: mc.CacheCreate, + Name: name, + USD: mc.USD, + Source: sourceForCost(mc), + Input: mc.Input, + Output: mc.Output, + CacheRead: mc.CacheRead, + CacheCreate: mc.CacheCreate, + CanonicalModel: modelfamily.Canonical(name), }) } for name, ac := range est.ByAgent { diff --git a/src/pkg/dashboard/cost_pure_functions_test.go b/src/pkg/dashboard/cost_pure_functions_test.go index 2b78da8069..8e6a6ed498 100644 --- a/src/pkg/dashboard/cost_pure_functions_test.go +++ b/src/pkg/dashboard/cost_pure_functions_test.go @@ -3,6 +3,7 @@ package dashboard import ( "testing" + "github.com/hivecommons/hive/pkg/modelfamily" "github.com/hivecommons/hive/pkg/tokens" ) @@ -233,3 +234,36 @@ func TestEstimatedSessions_CapsAtMaxCostSessions(t *testing.T) { t.Errorf("len = %d, exceeds maxCostSessions = %d", len(out), maxCostSessions) } } + +// TestFlattenEstimated_CanonicalModel: every by_model row carries the +// canonical id PRs-by-model groups by, the raw name stays as the row's name, +// spellings are not regrouped, and by_agent rows carry none (#18718). +func TestFlattenEstimated_CanonicalModel(t *testing.T) { + est := tokens.EstimatedCost{ + ByModel: map[string]tokens.ModelCost{ + "claude-opus-5-5": {USD: 1, Priced: true, Input: 10}, + "anthropic/claude-opus-5-5": {USD: 2, Priced: true, Input: 20}, + "gpt-5.6-sol:high": {USD: 3, Priced: true, Input: 30}, + }, + ByAgent: map[string]tokens.ModelCost{"quality": {USD: 1, Priced: true}}, + } + out := flattenEstimated(est) + if len(out.ByModel) != 3 { + t.Fatalf("ByModel len = %d, want 3 (spend is not regrouped)", len(out.ByModel)) + } + want := map[string]string{ + "claude-opus-5-5": "claude-opus-5.5", + "anthropic/claude-opus-5-5": "claude-opus-5.5", + "gpt-5.6-sol:high": "gpt-5.6-sol", + } + for _, m := range out.ByModel { + if m.CanonicalModel != want[m.Name] || m.CanonicalModel != modelfamily.Canonical(m.Name) { + t.Errorf("%s canonical_model = %q, want %q", m.Name, m.CanonicalModel, want[m.Name]) + } + } + for _, a := range out.ByAgent { + if a.CanonicalModel != "" { + t.Errorf("by_agent %s canonical_model = %q, want empty", a.Name, a.CanonicalModel) + } + } +} diff --git a/src/pkg/dashboard/static-src/js/cost.js b/src/pkg/dashboard/static-src/js/cost.js index 72e523bfc1..7216456a60 100644 --- a/src/pkg/dashboard/static-src/js/cost.js +++ b/src/pkg/dashboard/static-src/js/cost.js @@ -406,6 +406,17 @@ return counts; })(); + // A by_model row shows the canonical model id (the one the Governor's + // PRs-by-model card and run stats group by, #18718) with the raw name the + // token collector recorded in its tooltip. Rows are still one per raw + // name: spend is priced per raw model and is not regrouped. + const costModelCanonical = m => { + const c = String(m.canonical_model || ''); + return c && c !== 'unknown' && c !== m.name ? c : ''; + }; + const costModelLabel = m => costModelCanonical(m) || m.name || 'unknown'; + const costModelTitle = m => costModelCanonical(m) ? ` title="Reported as ${esc(m.name)}"` : ''; + // Per-model estimated cost table. Every model shows a $ estimate now — // exact list price where known, a coarse tier estimate (~) otherwise. const models = sortCostRows((est.by_model || []).map(m => { @@ -422,7 +433,7 @@ ? `${users.length}` : `—`; return ` - ${esc(m.name || 'unknown')} + ${esc(costModelLabel(m))} ${agentsCell} ${fmtTokens(m.input)}${miniSparkSvg(modelSeries(m.name, 'i'), 'var(--blue)', `${m.name || 'model'} input tokens`)} ${fmtTokens(m.output)}${miniSparkSvg(modelSeries(m.name, 'o'), 'var(--purple)', `${m.name || 'model'} output tokens`)} diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index 13e9591a86..0777b6aece 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -41041,6 +41041,17 @@

All labels

return counts; })(); + // A by_model row shows the canonical model id (the one the Governor's + // PRs-by-model card and run stats group by, #18718) with the raw name the + // token collector recorded in its tooltip. Rows are still one per raw + // name: spend is priced per raw model and is not regrouped. + const costModelCanonical = m => { + const c = String(m.canonical_model || ''); + return c && c !== 'unknown' && c !== m.name ? c : ''; + }; + const costModelLabel = m => costModelCanonical(m) || m.name || 'unknown'; + const costModelTitle = m => costModelCanonical(m) ? ` title="Reported as ${esc(m.name)}"` : ''; + // Per-model estimated cost table. Every model shows a $ estimate now — // exact list price where known, a coarse tier estimate (~) otherwise. const models = sortCostRows((est.by_model || []).map(m => { @@ -41057,7 +41068,7 @@

All labels

? `${users.length}` : `—`; return ` - ${esc(m.name || 'unknown')} + ${esc(costModelLabel(m))} ${agentsCell} ${fmtTokens(m.input)}${miniSparkSvg(modelSeries(m.name, 'i'), 'var(--blue)', `${m.name || 'model'} input tokens`)} ${fmtTokens(m.output)}${miniSparkSvg(modelSeries(m.name, 'o'), 'var(--purple)', `${m.name || 'model'} output tokens`)} diff --git a/src/pkg/dashboard/task_run_log.go b/src/pkg/dashboard/task_run_log.go index edeed946f2..bfc64bc45a 100644 --- a/src/pkg/dashboard/task_run_log.go +++ b/src/pkg/dashboard/task_run_log.go @@ -414,11 +414,15 @@ func (s *Server) handleContributeRunStats(w http.ResponseWriter, r *http.Request http.Error(w, "task-run log unreadable", http.StatusInternalServerError) return } + models := wallStatsList(modelStats) + if runStatsGroupCanonical(r) { + models = groupWallModelStatsByCanonical(models) + } jsonResponse(w, map[string]any{ "window_days": days, "total": total, "backends": stats, - "models": wallStatsList(modelStats), + "models": models, }) }