diff --git a/changelog.d/fixed-dashboard-pr-models-rank-score.md b/changelog.d/fixed-dashboard-pr-models-rank-score.md new file mode 100644 index 0000000000..e358cad444 --- /dev/null +++ b/changelog.d/fixed-dashboard-pr-models-rank-score.md @@ -0,0 +1 @@ +- dashboard: "PRs by model" rank now uses a confidence-weighted score (Wilson 95% lower bound of first-pass merge rate × (1 − fail) × (1 − no ship)) instead of raw first-pass %, folds model-id spellings (provider prefix, `:variant`, `5-5`→`5.5`) into one row with the raw ids in the tooltip, lists unattributed rows unranked at the bottom, renders the metric column labels once in a sticky header with `?` tooltips, and explains the formula and each column in the help text (#15618). diff --git a/src/docs/dashboard-sections.md b/src/docs/dashboard-sections.md index f0cc5fe321..b88587e4a2 100644 --- a/src/docs/dashboard-sections.md +++ b/src/docs/dashboard-sections.md @@ -127,21 +127,27 @@ How busy Hive thinks your projects are right now, and how often it wakes its wor ## PRs by model -Which AI models wrote the pull requests Hive opened, and how often each model's work was merged without extra fixes. +Which AI models wrote the pull requests Hive opened, how well each model's work landed, and how the models rank against each other. -**What it tells you.** This part of the Governor section compares the AI models your workers use. It shows how many pull requests each model wrote and how they ended. +**What it tells you.** This part of the Governor section compares the AI models your workers use. It shows how many pull requests each model wrote, how they ended, and a rank that favours models with a long, clean record over models with a short, lucky one. **How the numbers are worked out.** -- Each row is one model. The bar splits its pull requests into merged, still open, and closed without merging. -- **First-pass rate** is the share of merged pull requests that needed no follow-up fixes. -- **Failure rate** and **nothing to ship** count runs that failed or finished without a pull request. -- A model gets a **rank** only after it has at least 5 merged pull requests in the window. With fewer, the sample is too small to compare. +- Each row is one model. Model ids are normalised so one model gets one row: a provider prefix (`openai-codex/gpt-6.1-sol`), a `:variant` suffix (`gpt-5.6-sol:high`) and a dash between version digits (`claude-opus-5-5`) are folded into `gpt-6.1-sol`, `gpt-5.6-sol` and `claude-opus-5.5`. The backends that drove the model appear as chips under its name; hover the row to see the raw ids and how many pull requests each contributed. +- The bar splits a model's pull requests into merged, still open, and closed without merging. +- **Merged** is the number of merged pull requests in the window. +- **1st pass** is the share of merged pull requests that merged without a review-requested fix round. +- **PR runs** is the share of recorded task runs (from the contributor run log) that ended with a verified pull request. +- **Fail** is the share of recorded task runs that failed: crashed, timed out, or exited non-zero. +- **No ship** is the share of recorded task runs that completed but opened no pull request. +- **Rank** orders models by a score that combines those columns and penalises small samples: `score = Wilson 95% lower bound(1st-pass merged ÷ merged) × (1 − Fail) × (1 − No ship)`. The Wilson lower bound is the pessimistic end of the confidence interval for the first-pass rate, so 5 of 5 (lower bound 0.57) scores below 21 of 22 (0.78) and well below 203 of 313 (0.60 after the bound, but with no failures). A model with no recorded runs keeps its bound as the score. +- A model gets a rank only after it has at least 5 merged pull requests in the window. With fewer, the sample is too small to compare and the row is listed without a rank. +- Rows whose pull requests carry no model id are shown last, muted, as **unattributed**. They are counted but never ranked. - You can show the last 7 days, the last 30 days, or all time. You can sort by rank or by count. -**What it is good for.** Use it when choosing which model a worker should use. A model with a low first-pass rate costs more review time. +**What it is good for.** Use it when choosing which model a worker should use. A model with a low first-pass rate or a high failure rate costs more review and retry time; the rank already weighs both. -**Example.** Model A has 40 pull requests and a 70% first-pass rate. Model B has 6 pull requests and a 30% rate. Model A is the safer default. +**Example.** Model A has 40 merged pull requests, 28 of them first pass, and 2% failed runs. Model B has 5 merged pull requests, all first pass, and 51% failed runs. A scores about 0.54 and B about 0.28, so A ranks higher even though B shows 100% first pass. **When it appears.** Inside the Governor section, so at autonomy level L2 and above. diff --git a/src/internal/testutil/dashboard_import_allowlist.txt b/src/internal/testutil/dashboard_import_allowlist.txt index 740cdd5ef3..7598aa273d 100644 --- a/src/internal/testutil/dashboard_import_allowlist.txt +++ b/src/internal/testutil/dashboard_import_allowlist.txt @@ -47,6 +47,7 @@ github.com/hivecommons/hive/pkg/hubbackup github.com/hivecommons/hive/pkg/inferencehealth github.com/hivecommons/hive/pkg/ioscan github.com/hivecommons/hive/pkg/knowledge +github.com/hivecommons/hive/pkg/modelfamily github.com/hivecommons/hive/pkg/notify github.com/hivecommons/hive/pkg/nous github.com/hivecommons/hive/pkg/planning diff --git a/src/pkg/dashboard/api_governor_pr_models.go b/src/pkg/dashboard/api_governor_pr_models.go index f6c693cabc..9b18959c08 100644 --- a/src/pkg/dashboard/api_governor_pr_models.go +++ b/src/pkg/dashboard/api_governor_pr_models.go @@ -8,6 +8,7 @@ import ( "time" ghpkg "github.com/hivecommons/hive/pkg/github" + "github.com/hivecommons/hive/pkg/modelfamily" ) const ( @@ -28,9 +29,16 @@ type governorPRModelsResponse struct { MostReworked []governorPRMostReworkedItem `json:"most_reworked,omitempty"` } +// governorPRModelBucket is one canonical model (modelfamily.Canonical folds +// "openai-codex/gpt-6.1-sol", "gpt-6.1-sol" and "gpt-6.1-sol:high" into one +// row). Backend lists the distinct backends that drove it, comma-joined; +// Sources keeps the raw model/backend spellings and their counts so the +// dashboard can show them in a tooltip. type governorPRModelBucket struct { Model string `json:"model"` Backend string `json:"backend"` + Backends []string `json:"backends,omitempty"` + Sources []governorPRModelSource `json:"sources,omitempty"` PRs int `json:"prs"` Merged int `json:"merged"` ClosedUnmerged int `json:"closed_unmerged"` @@ -40,6 +48,14 @@ type governorPRModelBucket struct { Agents []governorPRModelAgentBucket `json:"agents,omitempty"` } +// governorPRModelSource is one raw attribution spelling folded into a bucket. +type governorPRModelSource struct { + Model string `json:"model"` + Backend string `json:"backend"` + PRs int `json:"prs"` + Merged int `json:"merged"` +} + type governorPRModelRework struct { SamplePRs int `json:"sample_prs"` FirstPassMerged int `json:"first_pass_merged"` @@ -56,7 +72,10 @@ type governorPRModelRework struct { type governorPRModelEffectiveness struct { Rank int `json:"rank,omitempty"` Eligible bool `json:"eligible"` + Unattributed bool `json:"unattributed,omitempty"` + Score float64 `json:"score"` MergedPRs int `json:"merged_prs"` + FirstPassMerged int `json:"first_pass_merged"` FirstPassMergeRate float64 `json:"first_pass_merge_rate"` RunCount int `json:"runs"` VerifiedPRRunRate float64 `json:"verified_pr_run_rate"` @@ -132,8 +151,10 @@ func aggregateGovernorPRModels(prs []ghpkg.PullRequest, window string, now time. type bucketState struct { governorPRModelBucket - agents map[string]*governorPRModelAgentBucket - rework reworkAccumulator + agents map[string]*governorPRModelAgentBucket + sources map[string]*governorPRModelSource + backends map[string]bool + rework reworkAccumulator } buckets := map[string]*bucketState{} resp := governorPRModelsResponse{Window: window, MinMergedPRs: effectiveModelsMinMergedPRs()} @@ -144,23 +165,36 @@ func aggregateGovernorPRModels(prs []ghpkg.PullRequest, window string, now time. if !cutoff.IsZero() && pr.CreatedAt.Before(cutoff) { continue } - model := ghpkg.NormalizeAttributionModel(pr.HiveModel) + rawModel := ghpkg.NormalizeAttributionModel(pr.HiveModel) + model := modelfamily.Canonical(pr.HiveModel) backend := ghpkg.NormalizeAttributionValue(pr.HiveBackend) agent := ghpkg.NormalizeAttributionValue(pr.HiveAgent) if agent == "unknown" { agent = "" } - key := model + "\x00" + backend - b := buckets[key] + b := buckets[model] if b == nil { b = &bucketState{ - governorPRModelBucket: governorPRModelBucket{Model: model, Backend: backend}, + governorPRModelBucket: governorPRModelBucket{Model: model}, agents: map[string]*governorPRModelAgentBucket{}, + sources: map[string]*governorPRModelSource{}, + backends: map[string]bool{}, } - buckets[key] = b + buckets[model] = b } + b.backends[backend] = true outcome := governorPRModelOutcome(pr) addOutcome(&b.PRs, &b.Merged, &b.ClosedUnmerged, &b.Open, outcome) + srcKey := rawModel + "\x00" + backend + src := b.sources[srcKey] + if src == nil { + src = &governorPRModelSource{Model: rawModel, Backend: backend} + b.sources[srcKey] = src + } + src.PRs++ + if outcome == "merged" { + src.Merged++ + } if outcome == "merged" { b.rework.add(pr.Rework) } @@ -187,7 +221,7 @@ func aggregateGovernorPRModels(prs []ghpkg.PullRequest, window string, now time. } addOutcome(&ab.PRs, &ab.Merged, &ab.ClosedUnmerged, &ab.Open, outcome) resp.Total++ - if model == "unknown" { + if model == modelfamily.Unknown { resp.Unknown++ } } @@ -195,6 +229,19 @@ func aggregateGovernorPRModels(prs []ghpkg.PullRequest, window string, now time. resp.Buckets = make([]governorPRModelBucket, 0, len(buckets)) for _, b := range buckets { b.Rework = b.rework.finish() + b.Backends, b.Backend = joinedBackends(b.backends) + for _, src := range b.sources { + b.Sources = append(b.Sources, *src) + } + sort.Slice(b.Sources, func(i, j int) bool { + if b.Sources[i].PRs != b.Sources[j].PRs { + return b.Sources[i].PRs > b.Sources[j].PRs + } + if b.Sources[i].Model != b.Sources[j].Model { + return b.Sources[i].Model < b.Sources[j].Model + } + return b.Sources[i].Backend < b.Sources[j].Backend + }) for _, ab := range b.agents { b.Agents = append(b.Agents, *ab) } @@ -232,6 +279,17 @@ func aggregateGovernorPRModels(prs []ghpkg.PullRequest, window string, now time. return resp } +// joinedBackends returns the sorted distinct backends and their comma-joined +// display form ("codex, pi"). +func joinedBackends(set map[string]bool) ([]string, string) { + out := make([]string, 0, len(set)) + for b := range set { + out = append(out, b) + } + sort.Strings(out) + return out, strings.Join(out, ", ") +} + func applyGovernorPRModelEffectiveness(resp *governorPRModelsResponse, effective contributeEffectiveModelsResponse, minMergedPRs int) { if resp == nil { return @@ -239,20 +297,23 @@ func applyGovernorPRModelEffectiveness(resp *governorPRModelsResponse, effective resp.MinMergedPRs = minMergedPRs rows := map[string]contributeEffectiveModelRow{} for _, row := range effective.Ranked { - rows[effectiveModelKey(row.Model, row.Backend)] = row + rows[row.Model] = row } for _, row := range effective.Insufficient { - rows[effectiveModelKey(row.Model, row.Backend)] = row + rows[row.Model] = row } for i := range resp.Buckets { - row, ok := rows[effectiveModelKey(resp.Buckets[i].Model, resp.Buckets[i].Backend)] + row, ok := rows[resp.Buckets[i].Model] if !ok { continue } resp.Buckets[i].Effectiveness = governorPRModelEffectiveness{ Rank: row.EffectivenessRank, - Eligible: row.MergedPRs >= minMergedPRs, + Eligible: row.EffectivenessRank > 0, + Unattributed: row.Unattributed, + Score: row.Score, MergedPRs: row.MergedPRs, + FirstPassMerged: row.FirstPassMerged, FirstPassMergeRate: row.FirstPassMergeRate, RunCount: row.RunCount, VerifiedPRRunRate: row.VerifiedPRRunRate, diff --git a/src/pkg/dashboard/contribute_effective_models.go b/src/pkg/dashboard/contribute_effective_models.go index 65bcfb2f60..2cbeb5e3f6 100644 --- a/src/pkg/dashboard/contribute_effective_models.go +++ b/src/pkg/dashboard/contribute_effective_models.go @@ -2,6 +2,7 @@ package dashboard import ( "encoding/json" + "math" "net/http" "net/url" "os" @@ -12,11 +13,15 @@ import ( "time" ghpkg "github.com/hivecommons/hive/pkg/github" + "github.com/hivecommons/hive/pkg/modelfamily" ) const ( effectiveModelsDefaultMinMergedPRs = 5 - effectiveModelsMinPRsEnv = "HIVE_CONTRIBUTE_EFFECTIVE_MODELS_MIN_PRS" + // effectiveModelsWilsonZ is the 95% two-sided normal quantile used for + // the Wilson lower bound on first-pass merge rate. + effectiveModelsWilsonZ = 1.96 + effectiveModelsMinPRsEnv = "HIVE_CONTRIBUTE_EFFECTIVE_MODELS_MIN_PRS" effectiveModelsFilterAll = "all" effectiveModelsFilterContributor = "contributor" @@ -43,13 +48,23 @@ type contributeEffectiveModelsCoverage struct { RunsPartial bool `json:"runs_partial"` } +// contributeEffectiveModelRow is one canonical model. Score is the rank key: +// the Wilson 95% lower bound of first-pass merges over merged PRs, multiplied +// by (1 − failure rate) and (1 − nothing-to-ship rate) from the run log (a +// row with no runs keeps the bound as-is). Unattributed rows (model unknown) +// are never ranked. type contributeEffectiveModelRow struct { Model string `json:"model"` Backend string `json:"backend"` + Backends []string `json:"backends,omitempty"` + Sources []governorPRModelSource `json:"sources,omitempty"` Runtime string `json:"runtime"` EffectivenessRank int `json:"effectiveness_rank,omitempty"` + Unattributed bool `json:"unattributed,omitempty"` + Score float64 `json:"score"` PRs int `json:"prs"` MergedPRs int `json:"merged_prs"` + FirstPassMerged int `json:"first_pass_merged"` FirstPassMergeRate float64 `json:"first_pass_merge_rate"` AvgReviewRounds float64 `json:"avg_review_rounds"` AvgFixAttempts float64 `json:"avg_fix_attempts"` @@ -68,6 +83,8 @@ type effectiveRunStats struct { VerifiedPRRuns int FailedRuns int NothingToShipRuns int + backends map[string]bool + sources map[string]*governorPRModelSource } func (s *Server) handleContributeEffectiveModels(w http.ResponseWriter, r *http.Request) { @@ -219,13 +236,18 @@ func aggregateEffectiveRunStats(runs []TaskRunRecord, filter string) map[string] return runStats } for _, rec := range runs { - model := ghpkg.NormalizeAttributionModel(rec.Model) + rawModel := ghpkg.NormalizeAttributionModel(rec.Model) + model := modelfamily.Canonical(rec.Model) backend := ghpkg.NormalizeAttributionValue(rec.Backend) - key := effectiveModelKey(model, backend) - st := runStats[key] + st := runStats[model] if st == nil { - st = &effectiveRunStats{} - runStats[key] = st + st = &effectiveRunStats{backends: map[string]bool{}, sources: map[string]*governorPRModelSource{}} + runStats[model] = st + } + st.backends[backend] = true + srcKey := rawModel + "\x00" + backend + if st.sources[srcKey] == nil { + st.sources[srcKey] = &governorPRModelSource{Model: rawModel, Backend: backend} } st.Runs++ if rec.PRVerified { @@ -248,19 +270,31 @@ func aggregateEffectiveModelsFromPRAggregation(prAgg governorPRModelsResponse, r } rows := map[string]*contributeEffectiveModelRow{} for _, b := range prAgg.Buckets { - row := effectiveRowForKey(rows, b.Model, b.Backend) + row := effectiveRowForModel(rows, b.Model) + row.Backends = append([]string(nil), b.Backends...) + row.Backend = b.Backend + row.Sources = append([]governorPRModelSource(nil), b.Sources...) row.PRs = b.PRs row.MergedPRs = b.Merged + row.FirstPassMerged = b.Rework.FirstPassMerged row.FirstPassMergeRate = b.Rework.FirstPassMergeRate row.AvgReviewRounds = b.Rework.AvgReviewRounds row.AvgFixAttempts = b.Rework.AvgFixAttempts } - for key, st := range runStats { - parts := strings.Split(key, "\x00") - if len(parts) != 2 { - continue + for model, st := range runStats { + row := effectiveRowForModel(rows, model) + if row.Backend == "" { + row.Backends, row.Backend = joinedBackends(st.backends) + for _, src := range st.sources { + row.Sources = append(row.Sources, *src) + } + sort.Slice(row.Sources, func(i, j int) bool { + if row.Sources[i].Model != row.Sources[j].Model { + return row.Sources[i].Model < row.Sources[j].Model + } + return row.Sources[i].Backend < row.Sources[j].Backend + }) } - row := effectiveRowForKey(rows, parts[0], parts[1]) row.RunCount = st.Runs row.VerifiedPRRuns = st.VerifiedPRRuns row.FailedRuns = st.FailedRuns @@ -272,14 +306,16 @@ func aggregateEffectiveModelsFromPRAggregation(prAgg governorPRModelsResponse, r } } for _, item := range prAgg.MostReworked { - row := effectiveRowForKey(rows, item.Model, item.Backend) + row := effectiveRowForModel(rows, item.Model) row.MostReworked = append(row.MostReworked, item) } resp := contributeEffectiveModelsResponse{Window: prAgg.Window, Filter: filter, MinMergedPRs: minMergedPRs} for _, row := range rows { - row.Runtime = effectiveModelRuntime(row.Model, row.Backend) - if row.MergedPRs >= minMergedPRs { + row.Runtime = effectiveModelRuntimeForSources(row.Model, row.Backend, row.Sources) + row.Unattributed = row.Model == modelfamily.Unknown + row.Score = effectiveModelScore(*row) + if row.MergedPRs >= minMergedPRs && !row.Unattributed { resp.Ranked = append(resp.Ranked, *row) } else { resp.Insufficient = append(resp.Insufficient, *row) @@ -293,20 +329,69 @@ func aggregateEffectiveModelsFromPRAggregation(prAgg governorPRModelsResponse, r return resp } -func effectiveModelKey(model, backend string) string { - return model + "\x00" + backend -} - -func effectiveRowForKey(rows map[string]*contributeEffectiveModelRow, model, backend string) *contributeEffectiveModelRow { - key := model + "\x00" + backend - row := rows[key] +func effectiveRowForModel(rows map[string]*contributeEffectiveModelRow, model string) *contributeEffectiveModelRow { + row := rows[model] if row == nil { - row = &contributeEffectiveModelRow{Model: model, Backend: backend} - rows[key] = row + row = &contributeEffectiveModelRow{Model: model} + rows[model] = row } return row } +// wilsonLowerBound is the lower end of the Wilson score interval for +// successes/n at effectiveModelsWilsonZ: 5/5 reads ~0.57, 21/22 ~0.78, +// 203/313 ~0.60, so a perfect score on a handful of PRs cannot out-rank a +// near-perfect score on many. +func wilsonLowerBound(successes, n int) float64 { + if n <= 0 || successes <= 0 { + return 0 + } + if successes > n { + successes = n + } + z := effectiveModelsWilsonZ + nf := float64(n) + p := float64(successes) / nf + denom := 1 + z*z/nf + centre := p + z*z/(2*nf) + margin := z * math.Sqrt(p*(1-p)/nf+z*z/(4*nf*nf)) + return (centre - margin) / denom +} + +// effectiveModelScore is the composite rank key: Wilson lower bound of +// first-pass merges over merged PRs × (1 − failure rate) × (1 − nothing-to-ship +// rate). Run-log rates are 0 when no runs were recorded, so the factor is 1. +func effectiveModelScore(row contributeEffectiveModelRow) float64 { + score := wilsonLowerBound(row.FirstPassMerged, row.MergedPRs) + score *= math.Max(0, 1-row.FailureRate) + score *= math.Max(0, 1-row.NothingToShipRate) + return score +} + +// effectiveModelRuntimeForSources classifies a canonical row from its raw +// spellings (the provider prefix that marks a Pi row hosted or local is +// stripped from the canonical id): one verdict when every source agrees, +// "mixed" when hosted and local both appear. +func effectiveModelRuntimeForSources(model, backend string, sources []governorPRModelSource) string { + if len(sources) == 0 { + return effectiveModelRuntime(model, backend) + } + seen := map[string]bool{} + for _, src := range sources { + seen[effectiveModelRuntime(src.Model, src.Backend)] = true + } + switch { + case seen["hosted"] && seen["local"]: + return "mixed" + case seen["hosted"]: + return "hosted" + case seen["local"]: + return "local" + default: + return "unknown" + } +} + func contributorPRKeysFromRuns(runs []TaskRunRecord) map[string]bool { out := map[string]bool{} for _, rec := range runs { @@ -393,8 +478,11 @@ func effectiveModelRuntime(model, backend string) string { func sortEffectiveModelRows(rows []contributeEffectiveModelRow, ranked bool) { sort.Slice(rows, func(i, j int) bool { a, b := rows[i], rows[j] - if ranked && a.FirstPassMergeRate != b.FirstPassMergeRate { - return a.FirstPassMergeRate > b.FirstPassMergeRate + if a.Unattributed != b.Unattributed { + return !a.Unattributed + } + if ranked && a.Score != b.Score { + return a.Score > b.Score } if a.MergedPRs != b.MergedPRs { return a.MergedPRs > b.MergedPRs diff --git a/src/pkg/dashboard/effective_models_rank_score_test.go b/src/pkg/dashboard/effective_models_rank_score_test.go new file mode 100644 index 0000000000..e21244da8d --- /dev/null +++ b/src/pkg/dashboard/effective_models_rank_score_test.go @@ -0,0 +1,173 @@ +package dashboard + +import ( + "math" + "strconv" + "testing" + "time" + + ghpkg "github.com/hivecommons/hive/pkg/github" +) + +// liveModelRow mirrors one row of the "PRs by model" panel as seen on the +// dashboard (rank mode, 7d): merged PRs, how many merged first pass, and the +// run-log rates that feed the composite score. +type liveModelRow struct { + model, backend string + merged int + firstPass int + runs int + failed int + noShip int +} + +func liveModelFixture(now time.Time) ([]ghpkg.PullRequest, []TaskRunRecord) { + rows := []liveModelRow{ + {"gpt-6-astra", "codex", 5, 5, 100, 51, 21}, + {"claude-opus-5.5", "copilot", 22, 21, 0, 0, 0}, + {"gpt-6.1-sol", "codex", 9, 8, 100, 2, 35}, + {"unknown", "unknown", 5, 4, 0, 0, 0}, + {"openai-codex/gpt-6.1-sol", "pi", 10, 7, 100, 0, 28}, + {"claude-fable-5.1", "copilot", 313, 203, 0, 0, 0}, + {"unknown", "codex", 37, 23, 100, 0, 36}, + {"claude-sonnet-5.5", "copilot", 17, 10, 0, 0, 0}, + {"claude-opus-5-5", "claude", 15, 6, 100, 0, 45}, + {"claude-sonnet-5-5", "claude", 1, 1, 0, 0, 0}, + {"openai-codex/gpt-5.6-sol:high", "omp", 1, 1, 0, 0, 0}, + } + var prs []ghpkg.PullRequest + var runs []TaskRunRecord + n := 0 + for _, r := range rows { + for i := 0; i < r.merged; i++ { + n++ + prs = append(prs, ghpkg.PullRequest{ + Repo: "org/repo", Number: n, HiveAttributed: true, HiveModel: r.model, HiveBackend: r.backend, + CreatedAt: now, MergedAt: now, Rework: ghpkg.PRReworkStats{FirstPass: i < r.firstPass}, + }) + } + for i := 0; i < r.runs; i++ { + rec := TaskRunRecord{TS: now.Format(time.RFC3339), Backend: r.backend, Model: r.model, Outcome: outcomeCompleted, PRVerified: true} + switch { + case i < r.failed: + rec.Outcome = outcomeFailed + rec.PRVerified = false + case i < r.failed+r.noShip: + rec.PRVerified = false + } + runs = append(runs, rec) + } + } + return prs, runs +} + +func TestWilsonLowerBoundPenalisesSmallSamples(t *testing.T) { + if got := wilsonLowerBound(5, 5); math.Abs(got-0.566) > 0.01 { + t.Fatalf("wilson(5/5) = %.3f, want ≈0.566", got) + } + if got := wilsonLowerBound(21, 22); math.Abs(got-0.781) > 0.01 { + t.Fatalf("wilson(21/22) = %.3f, want ≈0.781", got) + } + if wilsonLowerBound(5, 5) >= wilsonLowerBound(21, 22) { + t.Fatal("5/5 must not beat 21/22 on the lower bound") + } + if wilsonLowerBound(0, 0) != 0 || wilsonLowerBound(0, 7) != 0 { + t.Fatal("empty samples must score 0") + } +} + +func TestEffectiveModelsRankUsesConfidenceWeightedScoreOnLiveRows(t *testing.T) { + now := time.Date(2026, 10, 20, 12, 0, 0, 0, time.UTC) + prs, runs := liveModelFixture(now) + got := aggregateContributeEffectiveModels(prs, runs, "7d", "all", 5, now) + + index := func(model string) int { + for i, r := range got.Ranked { + if r.Model == model { + return i + } + } + return -1 + } + // claude-fable-5.1 (203/313, no run failures) has the tightest bound. + if index("claude-fable-5.1") != 0 { + t.Fatalf("claude-fable-5.1 should rank #1; ranked = %s", rankedSummary(got.Ranked)) + } + astra, opus := index("gpt-6-astra"), index("claude-opus-5.5") + if astra < 0 || astra <= opus { + t.Fatalf("gpt-6-astra (5/5 but 51%% fail) must not out-rank claude-opus-5.5; ranked = %s", rankedSummary(got.Ranked)) + } + if astra != len(got.Ranked)-1 { + t.Fatalf("gpt-6-astra (5/5 on 6 PRs, 51%% fail, 21%% no-ship) should be last of the ranked rows; ranked = %s", rankedSummary(got.Ranked)) + } + for i, r := range got.Ranked { + if r.Model == "unknown" { + t.Fatalf("unknown model must never be ranked, found at #%d", i+1) + } + if r.EffectivenessRank != i+1 { + t.Fatalf("rank[%d] = %d", i, r.EffectivenessRank) + } + if i > 0 && got.Ranked[i-1].Score < r.Score { + t.Fatalf("ranked rows not sorted by score: %s", rankedSummary(got.Ranked)) + } + } + + // Normalisation: the spellings of one model fold into one row that + // carries both backends and the raw ids. + sol := got.Ranked[index("gpt-6.1-sol")] + if sol.MergedPRs != 19 || sol.Backend != "codex, pi" || len(sol.Sources) != 2 { + t.Fatalf("gpt-6.1-sol should fold gpt-6.1-sol/codex + openai-codex/gpt-6.1-sol/pi: %+v", sol) + } + opusRow := got.Ranked[opus] + if opusRow.MergedPRs != 37 || opusRow.Backend != "claude, copilot" { + t.Fatalf("claude-opus-5.5 should fold claude-opus-5-5/claude: %+v", opusRow) + } + if index("claude-opus-5-5") >= 0 || index("openai-codex/gpt-6.1-sol") >= 0 { + t.Fatalf("raw spellings leaked as separate ranked rows: %s", rankedSummary(got.Ranked)) + } + + var unattributed []contributeEffectiveModelRow + for _, r := range got.Insufficient { + if r.Unattributed { + unattributed = append(unattributed, r) + } + } + if len(unattributed) != 1 || unattributed[0].MergedPRs != 42 || unattributed[0].EffectivenessRank != 0 { + t.Fatalf("unknown rows should fold into one unranked unattributed row: %+v", got.Insufficient) + } + if last := got.Insufficient[len(got.Insufficient)-1]; !last.Unattributed { + t.Fatalf("unattributed row should sort last among insufficient: %+v", got.Insufficient) + } + for _, r := range got.Insufficient { + if r.Model == "gpt-5.6-sol" && (len(r.Sources) != 1 || r.Sources[0].Model != "openai-codex/gpt-5.6-sol:high") { + t.Fatalf("gpt-5.6-sol should keep its raw id in Sources: %+v", r) + } + } +} + +func TestGovernorPRModelsEffectivenessCarriesScoreAndUnattributed(t *testing.T) { + now := time.Date(2026, 10, 20, 12, 0, 0, 0, time.UTC) + prs, runs := liveModelFixture(now) + resp := aggregateGovernorPRModels(prs, "7d", now) + applyGovernorPRModelEffectiveness(&resp, aggregateContributeEffectiveModels(prs, runs, "7d", "all", 5, now), 5) + for _, b := range resp.Buckets { + switch b.Model { + case "unknown": + if !b.Effectiveness.Unattributed || b.Effectiveness.Rank != 0 || b.Effectiveness.Eligible { + t.Fatalf("unknown bucket effectiveness = %+v", b.Effectiveness) + } + case "claude-opus-5.5": + if b.Effectiveness.Rank == 0 || !b.Effectiveness.Eligible || b.Effectiveness.Score <= 0 || b.Effectiveness.FirstPassMerged != 27 { + t.Fatalf("opus bucket effectiveness = %+v", b.Effectiveness) + } + } + } +} + +func rankedSummary(rows []contributeEffectiveModelRow) string { + out := "" + for _, r := range rows { + out += r.Model + "(" + strconv.FormatFloat(r.Score, 'f', 3, 64) + ") " + } + return out +} diff --git a/src/pkg/dashboard/pr_models_header_once.test.js b/src/pkg/dashboard/pr_models_header_once.test.js new file mode 100644 index 0000000000..e8c14f0bf5 --- /dev/null +++ b/src/pkg/dashboard/pr_models_header_once.test.js @@ -0,0 +1,84 @@ +// "PRs by model" renders column names (MERGED / 1ST PASS / PR RUNS / FAIL / +// NO SHIP) once, in a sticky header row carrying the ? tooltips, and plain +// numbers per row. Before this test the chip-stat eyebrow pattern repeated +// the five labels above every row's values. The narrow breakpoint restores +// labels via CSS attr(data-label), which is not visible text here. Also +// pins: unattributed ("unknown" model) rows sort last and are never ranked. +const test = require('node:test'); +const assert = require('node:assert'); +const fs = require('node:fs'); +const path = require('node:path'); + +const html = fs.readFileSync(path.join(__dirname, 'static', 'index.html'), 'utf8'); + +function pick(re, what) { + const m = html.match(re); + assert.ok(m, what + ' not found in index.html'); + return m[0]; +} + +const src = [ + 'let govPRModelsSort = "effectiveness"; let govPRModelsWindow = "7d"; let govPRModelsData = null;', + 'const escapeHtml = s => String(s).replace(/[&<>"\']/g, c => ({"&":"&","<":"<",">":">",\'"\':""","\'":"'"}[c]));', + 'const dashboardSectionHelpMark = () => "";', + 'const formatFixTime = () => "";', + pick(/function governorPRModelUnattributed\(b\) \{[\s\S]*?\n \}\n/, 'governorPRModelUnattributed'), + pick(/const GOV_PR_MODEL_COLUMNS = \[[\s\S]*?\n \];\n/, 'GOV_PR_MODEL_COLUMNS'), + pick(/function governorPRModelRows\(data\) \{[\s\S]*?\n \}\n/, 'governorPRModelRows'), + pick(/function renderGovernorPRModelsTile\(\) \{[\s\S]*?\n \}\n\n function renderGovernor\(/, 'renderGovernorPRModelsTile').replace(/\n\n function renderGovernor\($/, ''), +].join('\n'); +const lifted = new Function(src + '; return { render: data => { govPRModelsData = data; return renderGovernorPRModelsTile(); }, rows: governorPRModelRows, columns: GOV_PR_MODEL_COLUMNS };')(); + +const bucket = (model, backend, prs, merged, eff) => ({ + model, backend, backends: backend.split(', '), prs, merged, open: 0, closed_unmerged: prs - merged, + rework: { first_pass_merge_rate: eff.first_pass_merge_rate || 0 }, + effectiveness: { eligible: !!eff.rank, merged_prs: merged, ...eff }, + sources: [{ model, backend, prs, merged }], +}); + +const data = { + window: '7d', total: 60, min_merged_prs: 5, + buckets: [ + bucket('unknown', 'codex', 20, 12, { unattributed: true, first_pass_merge_rate: 0.62 }), + bucket('gpt-6-astra', 'codex', 6, 5, { rank: 3, score: 0.22, first_pass_merged: 5, first_pass_merge_rate: 1, failure_rate: 0.51, nothing_to_ship_rate: 0.21 }), + bucket('claude-opus-5.5', 'claude, copilot', 22, 22, { rank: 1, score: 0.78, first_pass_merged: 21, first_pass_merge_rate: 0.95 }), + bucket('claude-fable-5.1', 'copilot', 12, 12, { rank: 2, score: 0.6, first_pass_merged: 8, first_pass_merge_rate: 0.65 }), + ], +}; + +test('column labels appear exactly once, in a header row with tooltips', () => { + const out = lifted.render(data); + const header = out.match(/