diff --git a/changelog.d/added-18715-pr-models-backend-effort-rework.md b/changelog.d/added-18715-pr-models-backend-effort-rework.md new file mode 100644 index 0000000000..3423ba63da --- /dev/null +++ b/changelog.d/added-18715-pr-models-backend-effort-rework.md @@ -0,0 +1 @@ +- Governor PRs by model now shows rework per backend and a per-effort split in each row's tooltip: every raw `sources[]` entry gains open/closed-unmerged counts, first-pass merged and rate, and average review rounds and fix attempts, each bucket gains an `efforts[]` breakdown read from the attribution trailer's `effort=` value, and the PR snapshot records it as `hive_effort` (#18715). diff --git a/dashboard/test/governor-pr-models-ui.test.js b/dashboard/test/governor-pr-models-ui.test.js index cef6a5e1ea..7c28747268 100644 --- a/dashboard/test/governor-pr-models-ui.test.js +++ b/dashboard/test/governor-pr-models-ui.test.js @@ -37,3 +37,17 @@ test('Governor PRs by model shows unmeasured run columns as a dash and discloses assert.ok(indexHtml.includes(expected), `index.html missing ${expected}`); } }); + +test('Governor PRs by model tooltip lists per-backend rework and the effort split (#18715)', () => { + for (const expected of [ + 'const sliceRework = s =>', + 'first pass ${s.first_pass_merged || 0}/${s.sample_prs || 0}', + '${s.closed_unmerged || 0} closed-unmerged', + 'avg fix attempts ${(s.avg_fix_attempts || 0).toFixed(1)}', + "${src.backend || 'unknown'}: ${sliceRework(src)}", + "(b.efforts || []).map(e => `effort ${e.effort || 'unknown'}: ${sliceRework(e)}`)", + "'\\n\\nBy effort:\\n' + effortBreakdown", + ]) { + assert.ok(indexHtml.includes(expected), `index.html missing ${expected}`); + } +}); diff --git a/src/docs/dashboard-sections/overview-and-metrics.md b/src/docs/dashboard-sections/overview-and-metrics.md index f55cc86cd9..4d73101f7f 100644 --- a/src/docs/dashboard-sections/overview-and-metrics.md +++ b/src/docs/dashboard-sections/overview-and-metrics.md @@ -94,6 +94,7 @@ Which AI models wrote the pull requests Hive opened, how well each model's work **How the numbers are worked out.** - Each row is one model. Model ids are normalised so one model gets one row: a provider prefix (`openai-codex/gpt-6.1-sol`), a `:variant` suffix (`gpt-5.6-sol:high`) and a dash between version digits (`claude-opus-5-5`) are folded into `gpt-6.1-sol`, `gpt-5.6-sol` and `claude-opus-5.5`. The backends that drove the model appear as chips under its name; hover the row to see the raw ids and how many pull requests each contributed. +- The row's tooltip also splits the model by backend and by effort, the reasoning effort the PR's `— hive:` signature line records (`effort=high`; **unknown** when the line has none). Each backend line and each effort line shows its pull requests (merged, open, closed without merging), how many merged first pass, and the average review rounds and fix attempts of its merged pull requests, so you can see whether one harness or effort level needed more fixing. The API returns these as the extra fields on `sources[]` and as `efforts[]` on each model. - The bar splits a model's pull requests into merged, still open, and closed without merging. - **Merged** is the number of merged pull requests in the window. - **1st pass** is the share of merged pull requests that merged without a review-requested fix round. diff --git a/src/docs/scale-envelope.md b/src/docs/scale-envelope.md index e734801b85..918bce3e25 100644 --- a/src/docs/scale-envelope.md +++ b/src/docs/scale-envelope.md @@ -34,7 +34,7 @@ ones to: 2. **Truncation must preserve the prioritization signal, not cut an arbitrary slice.** This holds today and is easy to assume backwards: - Issues are sorted by actionable rank tier, then **oldest first** within each tier before any cap is applied - (`RankActionableIssues`, `src/pkg/github/client.go:5351-5361`, descending `AgeMinutes` within each tier). + (`RankActionableIssues`, `src/pkg/github/client.go:5357-5367`, descending `AgeMinutes` within each tier). - PRs are sorted by **review class then oldest-first within class** (`SortPullRequestsForReview`, `src/pkg/github/review_priority.go:334`; fixes → refactors/docs → tests). diff --git a/src/pkg/dashboard/api_governor_pr_models.go b/src/pkg/dashboard/api_governor_pr_models.go index 0c0225e34c..15c5af130c 100644 --- a/src/pkg/dashboard/api_governor_pr_models.go +++ b/src/pkg/dashboard/api_governor_pr_models.go @@ -54,14 +54,66 @@ type governorPRModelBucket struct { Rework governorPRModelRework `json:"rework"` Effectiveness governorPRModelEffectiveness `json:"effectiveness"` Agents []governorPRModelAgentBucket `json:"agents,omitempty"` + // Efforts splits the bucket by the trailer's effort= value ("unknown" + // when absent), sorted by PRs descending (hivecommons/hive#18715). + Efforts []governorPRModelEffort `json:"efforts,omitempty"` } // governorPRModelSource is one raw attribution spelling folded into a bucket. +// The outcome and rework fields are computed over this source's PRs the same +// way the bucket's Rework is computed over merged PRs (hivecommons/hive#18715). type governorPRModelSource struct { Model string `json:"model"` Backend string `json:"backend"` PRs int `json:"prs"` Merged int `json:"merged"` + governorPRModelSliceStats +} + +// governorPRModelEffort is one effort= value within a bucket. +type governorPRModelEffort struct { + Effort string `json:"effort"` + PRs int `json:"prs"` + Merged int `json:"merged"` + governorPRModelSliceStats +} + +// governorPRModelSliceStats are the outcome and rework figures shared by a +// bucket's per-source and per-effort slices. +type governorPRModelSliceStats struct { + ClosedUnmerged int `json:"closed_unmerged"` + Open int `json:"open"` + SamplePRs int `json:"sample_prs"` + FirstPassMerged int `json:"first_pass_merged"` + FirstPassMergeRate float64 `json:"first_pass_merge_rate"` + AvgReviewRounds float64 `json:"avg_review_rounds"` + AvgFixAttempts float64 `json:"avg_fix_attempts"` +} + +// governorPRModelSliceState accumulates one per-source or per-effort slice. +type governorPRModelSliceState struct { + prs, merged, closedUnmerged, open int + rework reworkAccumulator +} + +func (st *governorPRModelSliceState) add(outcome string, rework ghpkg.PRReworkStats) { + addOutcome(&st.prs, &st.merged, &st.closedUnmerged, &st.open, outcome) + if outcome == "merged" { + st.rework.add(rework) + } +} + +func (st *governorPRModelSliceState) finish() governorPRModelSliceStats { + r := st.rework.finish() + return governorPRModelSliceStats{ + ClosedUnmerged: st.closedUnmerged, + Open: st.open, + SamplePRs: r.SamplePRs, + FirstPassMerged: r.FirstPassMerged, + FirstPassMergeRate: r.FirstPassMergeRate, + AvgReviewRounds: r.AvgReviewRounds, + AvgFixAttempts: r.AvgFixAttempts, + } } type governorPRModelRework struct { @@ -175,6 +227,9 @@ func aggregateGovernorPRModels(prs []ghpkg.PullRequest, window string, now time. sources map[string]*governorPRModelSource backends map[string]bool rework reworkAccumulator + // Per-source and per-effort slice accumulators (#18715). + sourceStats map[string]*governorPRModelSliceState + efforts map[string]*governorPRModelSliceState } buckets := map[string]*bucketState{} resp := governorPRModelsResponse{Window: window, MinMergedPRs: effectiveModelsMinMergedPRs()} @@ -199,6 +254,8 @@ func aggregateGovernorPRModels(prs []ghpkg.PullRequest, window string, now time. agents: map[string]*governorPRModelAgentBucket{}, sources: map[string]*governorPRModelSource{}, backends: map[string]bool{}, + sourceStats: map[string]*governorPRModelSliceState{}, + efforts: map[string]*governorPRModelSliceState{}, } buckets[model] = b } @@ -218,6 +275,8 @@ func aggregateGovernorPRModels(prs []ghpkg.PullRequest, window string, now time. if outcome == "merged" { b.rework.add(pr.Rework) } + governorPRModelSlice(b.sourceStats, srcKey).add(outcome, pr.Rework) + governorPRModelSlice(b.efforts, governorPRModelEffortKey(pr.HiveEffort)).add(outcome, pr.Rework) if score := pr.Rework.ReviewRounds + pr.Rework.FixAttempts; score > 0 { resp.MostReworked = append(resp.MostReworked, governorPRMostReworkedItem{ Repo: pr.Repo, @@ -255,9 +314,13 @@ func aggregateGovernorPRModels(prs []ghpkg.PullRequest, window string, now time. for _, b := range buckets { b.Rework = b.rework.finish() b.Backends, b.Backend = joinedBackends(b.backends) - for _, src := range b.sources { + for key, src := range b.sources { + if st := b.sourceStats[key]; st != nil { + src.governorPRModelSliceStats = st.finish() + } b.Sources = append(b.Sources, *src) } + b.Efforts = finishGovernorPRModelEfforts(b.efforts) sort.Slice(b.Sources, func(i, j int) bool { if b.Sources[i].PRs != b.Sources[j].PRs { return b.Sources[i].PRs > b.Sources[j].PRs @@ -304,6 +367,37 @@ func aggregateGovernorPRModels(prs []ghpkg.PullRequest, window string, now time. return resp } +// governorPRModelSlice returns the accumulator for key, creating it. +func governorPRModelSlice(m map[string]*governorPRModelSliceState, key string) *governorPRModelSliceState { + st := m[key] + if st == nil { + st = &governorPRModelSliceState{} + m[key] = st + } + return st +} + +// governorPRModelEffortKey normalises a PR's effort= value; absent is "unknown". +func governorPRModelEffortKey(effort string) string { + return ghpkg.NormalizeAttributionValue(effort) +} + +// finishGovernorPRModelEfforts renders the per-effort slices sorted by PRs +// descending, then effort name. +func finishGovernorPRModelEfforts(m map[string]*governorPRModelSliceState) []governorPRModelEffort { + out := make([]governorPRModelEffort, 0, len(m)) + for effort, st := range m { + out = append(out, governorPRModelEffort{Effort: effort, PRs: st.prs, Merged: st.merged, governorPRModelSliceStats: st.finish()}) + } + sort.Slice(out, func(i, j int) bool { + if out[i].PRs != out[j].PRs { + return out[i].PRs > out[j].PRs + } + return out[i].Effort < out[j].Effort + }) + return out +} + // joinedBackends returns the sorted distinct backends and their comma-joined // display form ("codex, pi"). func joinedBackends(set map[string]bool) ([]string, string) { diff --git a/src/pkg/dashboard/api_governor_pr_models_test.go b/src/pkg/dashboard/api_governor_pr_models_test.go index 76b02c70db..8f397f98e8 100644 --- a/src/pkg/dashboard/api_governor_pr_models_test.go +++ b/src/pkg/dashboard/api_governor_pr_models_test.go @@ -204,3 +204,102 @@ func TestAggregateGovernorPRModelsReworkStats(t *testing.T) { t.Fatalf("most reworked = %+v, want open PR #3 then merged PR #2", got.MostReworked) } } + +// One canonical model driven by two backends with different rework and two +// efforts plus an effort-less PR: per-source and per-effort figures, and their +// sums match the bucket (hivecommons/hive#18715). +func TestAggregateGovernorPRModelsPerSourceAndEffortRework(t *testing.T) { + now := time.Date(2026, 11, 17, 12, 0, 0, 0, time.UTC) + created := now.AddDate(0, 0, -1) + prs := []ghpkg.PullRequest{ + {Repo: "o/r", Number: 1, HiveAttributed: true, HiveModel: "claude-opus-5-5", HiveBackend: "claude", HiveEffort: "high", CreatedAt: created, MergedAt: now, + Rework: ghpkg.PRReworkStats{FirstPass: true}}, + {Repo: "o/r", Number: 2, HiveAttributed: true, HiveModel: "claude-opus-5-5", HiveBackend: "claude", HiveEffort: "High", CreatedAt: created, MergedAt: now, + Rework: ghpkg.PRReworkStats{ReviewRounds: 1, FixAttempts: 1}}, + {Repo: "o/r", Number: 3, HiveAttributed: true, HiveModel: "anthropic/claude-opus-5-5", HiveBackend: "omp", HiveEffort: "medium", CreatedAt: created, MergedAt: now, + Rework: ghpkg.PRReworkStats{ReviewRounds: 2, FixAttempts: 2}}, + {Repo: "o/r", Number: 4, HiveAttributed: true, HiveModel: "anthropic/claude-opus-5-5", HiveBackend: "omp", CreatedAt: created, State: "closed", ClosedAt: now}, + {Repo: "o/r", Number: 5, HiveAttributed: true, HiveModel: "anthropic/claude-opus-5-5", HiveBackend: "omp", HiveEffort: "medium", CreatedAt: created, State: "open"}, + } + got := aggregateGovernorPRModels(prs, "7d", now) + if len(got.Buckets) != 1 { + t.Fatalf("buckets = %+v, want one canonical row", got.Buckets) + } + b := got.Buckets[0] + // Existing bucket fields keep their values. + if b.PRs != 5 || b.Merged != 3 || b.ClosedUnmerged != 1 || b.Open != 1 { + t.Fatalf("bucket outcomes = %d/%d/%d/%d, want 5/3/1/1", b.PRs, b.Merged, b.ClosedUnmerged, b.Open) + } + if b.Rework.SamplePRs != 3 || b.Rework.FirstPassMerged != 1 { + t.Fatalf("bucket rework = %+v, want sample 3 first pass 1", b.Rework) + } + + if len(b.Sources) != 2 { + t.Fatalf("sources = %+v, want 2", b.Sources) + } + omp, claude := b.Sources[0], b.Sources[1] + if omp.Backend != "omp" || omp.Model != "anthropic/claude-opus-5-5" { + t.Fatalf("sources[0] = %+v, want omp (3 PRs) first", omp) + } + if omp.PRs != 3 || omp.Merged != 1 || omp.ClosedUnmerged != 1 || omp.Open != 1 || + omp.SamplePRs != 1 || omp.FirstPassMerged != 0 || omp.FirstPassMergeRate != 0 || + omp.AvgReviewRounds != 2 || omp.AvgFixAttempts != 2 { + t.Fatalf("omp source = %+v", omp) + } + if claude.Backend != "claude" || claude.PRs != 2 || claude.Merged != 2 || claude.ClosedUnmerged != 0 || claude.Open != 0 || + claude.SamplePRs != 2 || claude.FirstPassMerged != 1 || claude.FirstPassMergeRate != 0.5 || + claude.AvgReviewRounds != 0.5 || claude.AvgFixAttempts != 0.5 { + t.Fatalf("claude source = %+v", claude) + } + + want := []governorPRModelEffort{ + {Effort: "high", PRs: 2, Merged: 2, governorPRModelSliceStats: governorPRModelSliceStats{SamplePRs: 2, FirstPassMerged: 1, FirstPassMergeRate: 0.5, AvgReviewRounds: 0.5, AvgFixAttempts: 0.5}}, + {Effort: "medium", PRs: 2, Merged: 1, governorPRModelSliceStats: governorPRModelSliceStats{Open: 1, SamplePRs: 1, AvgReviewRounds: 2, AvgFixAttempts: 2}}, + {Effort: "unknown", PRs: 1, governorPRModelSliceStats: governorPRModelSliceStats{ClosedUnmerged: 1}}, + } + if len(b.Efforts) != len(want) { + t.Fatalf("efforts = %+v, want %+v", b.Efforts, want) + } + for i := range want { + if b.Efforts[i] != want[i] { + t.Fatalf("efforts[%d] = %+v, want %+v", i, b.Efforts[i], want[i]) + } + } + + var srcPRs, srcMerged, srcFirst, effPRs, effMerged, effFirst, effClosed, effOpen int + for _, s := range b.Sources { + srcPRs, srcMerged, srcFirst = srcPRs+s.PRs, srcMerged+s.Merged, srcFirst+s.FirstPassMerged + } + for _, e := range b.Efforts { + effPRs, effMerged, effFirst = effPRs+e.PRs, effMerged+e.Merged, effFirst+e.FirstPassMerged + effClosed, effOpen = effClosed+e.ClosedUnmerged, effOpen+e.Open + } + if srcPRs != b.PRs || srcMerged != b.Merged || srcFirst != b.Rework.FirstPassMerged { + t.Fatalf("source sums %d/%d/%d != bucket %d/%d/%d", srcPRs, srcMerged, srcFirst, b.PRs, b.Merged, b.Rework.FirstPassMerged) + } + if effPRs != b.PRs || effMerged != b.Merged || effFirst != b.Rework.FirstPassMerged || effClosed != b.ClosedUnmerged || effOpen != b.Open { + t.Fatalf("effort sums %d/%d/%d/%d/%d != bucket", effPRs, effMerged, effFirst, effClosed, effOpen) + } + + raw, err := json.Marshal(b) + if err != nil { + t.Fatal(err) + } + var decoded struct { + Sources []map[string]any `json:"sources"` + Efforts []map[string]any `json:"efforts"` + } + if err := json.Unmarshal(raw, &decoded); err != nil { + t.Fatal(err) + } + for _, key := range []string{"model", "backend", "prs", "merged", "closed_unmerged", "open", "first_pass_merged", "first_pass_merge_rate", "avg_review_rounds", "avg_fix_attempts", "sample_prs"} { + if _, ok := decoded.Sources[0][key]; !ok { + t.Fatalf("sources[0] JSON lacks %q: %s", key, raw) + } + } + for _, key := range []string{"effort", "prs", "merged", "closed_unmerged", "open", "first_pass_merged", "first_pass_merge_rate", "avg_review_rounds", "avg_fix_attempts"} { + if _, ok := decoded.Efforts[0][key]; !ok { + t.Fatalf("efforts[0] JSON lacks %q: %s", key, raw) + } + } +} diff --git a/src/pkg/dashboard/governor_pr_models_ui_test.go b/src/pkg/dashboard/governor_pr_models_ui_test.go index 2f0f5d14b1..a5bb4a3ca3 100644 --- a/src/pkg/dashboard/governor_pr_models_ui_test.go +++ b/src/pkg/dashboard/governor_pr_models_ui_test.go @@ -30,6 +30,10 @@ func TestGovernorPRModelsUIRendersReworkStats(t *testing.T) { "no runs recorded, not a measured 0%", "first_pass_merge_rate_ci", "closed_prs_partial", + "b.efforts", + "By effort:", + "avg_fix_attempts", + "sample_prs", } { if !strings.Contains(s, want) { t.Fatalf("static/index.html missing %q", want) diff --git a/src/pkg/dashboard/static-src/js/governor-view.js b/src/pkg/dashboard/static-src/js/governor-view.js index 60b3cb3e6d..b6be269f95 100644 --- a/src/pkg/dashboard/static-src/js/governor-view.js +++ b/src/pkg/dashboard/static-src/js/governor-view.js @@ -164,11 +164,13 @@ const metrics = `