Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/added-18716-pr-models-repos-weekly.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- Governor PRs by model now shows, in each model's tooltip, a per-repository breakdown and a weekly trend (30d and all-time views), so a model's rework can be traced to one hard repository and its improvement over time is visible without diffing windows. `GET /api/governor/pr-models` adds `repos[]` and `weeks[]` to each bucket; existing fields are unchanged (#18716).
49 changes: 49 additions & 0 deletions dashboard/test/governor-pr-models-ui.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -51,3 +51,52 @@ test('Governor PRs by model tooltip lists per-backend rework and the effort spli
assert.ok(indexHtml.includes(expected), `index.html missing ${expected}`);
}
});

function extractFunction(name) {
const start = indexHtml.indexOf('function ' + name + '(');
assert.notEqual(start, -1, `${name} function exists`);
const brace = indexHtml.indexOf('{', start);
let depth = 0;
for (let i = brace; i < indexHtml.length; i++) {
const ch = indexHtml[i];
if (ch === '{') depth++;
if (ch === '}') depth--;
if (depth === 0) return indexHtml.slice(start, i + 1);
}
throw new Error(`unterminated ${name}`);
}

test('Governor PRs by model tooltip lists repos and weekly trend when present (#18716)', () => {
const vm = require('node:vm');
const ctx = {};
vm.createContext(ctx);
vm.runInContext(extractFunction('governorPRModelSliceDetail') + '\nthis.detail = governorPRModelSliceDetail;', ctx);
const detail = ctx.detail;

const text = detail({
repos: [
{ repo: 'org/alpha', prs: 3, merged: 2, first_pass_merge_rate: 0.5, avg_fix_attempts: 0.5 },
{ repo: 'org/beta', prs: 1, merged: 0 },
],
weeks: [
{ week_start: '2026-09-14', prs: 1, merged: 1, first_pass_merge_rate: 1, avg_fix_attempts: 0 },
{ week_start: '2026-09-21', prs: 2, merged: 2, first_pass_merge_rate: 0.5, avg_fix_attempts: 0.5 },
],
});
assert.match(text, /^By repository:\n/);
assert.ok(text.includes('org/alpha: 3 PRs (2 merged, first-pass 50%, avg fix 0.5)'), text);
assert.ok(text.includes('org/beta: 1 PRs (0 merged)'), text);
assert.ok(text.includes('Weekly trend'), text);
assert.ok(text.indexOf('2026-09-14: 1 PRs (1 merged, first-pass 100%, avg fix 0.0)') < text.indexOf('2026-09-21: 2 PRs'), text);

// 7d buckets carry repos but no weeks.
const sevenDay = detail({ repos: [{ repo: 'org/alpha', prs: 1, merged: 1, first_pass_merge_rate: 1 }] });
assert.ok(sevenDay.includes('By repository:'), sevenDay);
assert.ok(!sevenDay.includes('Weekly trend'), sevenDay);

// Buckets without either (older servers) add nothing to the tooltip.
assert.equal(detail({}), '');
assert.equal(detail({ repos: [], weeks: [] }), '');

assert.ok(indexHtml.includes("${sliceDetail ? '\\n\\n' + sliceDetail : ''}"), 'tooltip appends the slice detail');
});
1 change: 1 addition & 0 deletions src/docs/dashboard-sections/overview-and-metrics.md
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,7 @@ Which AI models wrote the pull requests Hive opened, how well each model's work
- A model gets a rank only after it has at least 5 merged pull requests in the window. With fewer, the sample is too small to compare and the row is listed without a rank.
- Rows whose pull requests carry no model id are shown last, muted, as **unattributed**. They are counted but never ranked.
- You can show the last 7 days, the last 30 days, or all time. You can sort by rank or by count.
- Hover a row to see where its pull requests went and how its work is trending. **By repository** lists each repository (`owner/repo`, so a short configured name and the full name count as one), with its pull requests, merged count, first-pass rate and average fix attempts. **Weekly trend** lists one line per week (weeks start Monday, UTC, by the date the pull request was opened), oldest first, with the same figures; weeks with no pull requests are left out. The weekly lines appear for the 30-day and all-time views only, since 7 days is a single week. First-pass rate and fix attempts count merged pull requests only, as the row does. The API returns these as `repos[]` and `weeks[]` on each bucket of `GET /api/governor/pr-models`.

**What it is good for.** Use it when choosing which model a worker should use. A model with a low first-pass rate or a high failure rate costs more review and retry time; the rank already weighs both.

Expand Down
96 changes: 96 additions & 0 deletions src/pkg/dashboard/api_governor_pr_models.go
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,80 @@ type governorPRModelBucket struct {
// Efforts splits the bucket by the trailer's effort= value ("unknown"
// when absent), sorted by PRs descending (hivecommons/hive#18715).
Efforts []governorPRModelEffort `json:"efforts,omitempty"`
// Repos splits the bucket by canonical owner/repo (effectivePRRepo), PRs
// descending. Weeks splits it by ISO week (Monday, UTC) of created_at,
// oldest first, and is omitted for window=7d. Rework figures in both
// cover merged PRs only, as Rework does.
Repos []governorPRModelRepoBucket `json:"repos,omitempty"`
Weeks []governorPRModelWeekBucket `json:"weeks,omitempty"`
}

// governorPRModelRepoBucket is one repository's share of a model bucket.
type governorPRModelRepoBucket struct {
Repo string `json:"repo"`
PRs int `json:"prs"`
Merged int `json:"merged"`
ClosedUnmerged int `json:"closed_unmerged"`
Open int `json:"open"`
FirstPassMerged int `json:"first_pass_merged"`
FirstPassMergeRate float64 `json:"first_pass_merge_rate"`
AvgReviewRounds float64 `json:"avg_review_rounds"`
AvgFixAttempts float64 `json:"avg_fix_attempts"`
}

// governorPRModelWeekBucket is one ISO week's share of a model bucket.
// WeekStart is the Monday (UTC) as YYYY-MM-DD.
type governorPRModelWeekBucket struct {
WeekStart string `json:"week_start"`
PRs int `json:"prs"`
Merged int `json:"merged"`
FirstPassMerged int `json:"first_pass_merged"`
FirstPassMergeRate float64 `json:"first_pass_merge_rate"`
AvgReviewRounds float64 `json:"avg_review_rounds"`
AvgFixAttempts float64 `json:"avg_fix_attempts"`
}

// governorPRModelWeekStart returns the Monday 00:00 UTC that starts t's ISO
// week, as YYYY-MM-DD.
func governorPRModelWeekStart(t time.Time) string {
t = t.UTC()
offset := (int(t.Weekday()) + 6) % 7 // Monday = 0
return time.Date(t.Year(), t.Month(), t.Day()-offset, 0, 0, 0, 0, time.UTC).Format("2006-01-02")
}

// governorPRModelRepoSlices finishes the per-repo slices, PRs descending.
func governorPRModelRepoSlices(repos map[string]*governorPRModelSliceState, names map[string]string) []governorPRModelRepoBucket {
out := make([]governorPRModelRepoBucket, 0, len(repos))
for key, st := range repos {
rw := st.rework.finish()
out = append(out, governorPRModelRepoBucket{
Repo: names[key], PRs: st.prs, Merged: st.merged, ClosedUnmerged: st.closedUnmerged, Open: st.open,
FirstPassMerged: rw.FirstPassMerged, FirstPassMergeRate: rw.FirstPassMergeRate,
AvgReviewRounds: rw.AvgReviewRounds, AvgFixAttempts: rw.AvgFixAttempts,
})
}
sort.Slice(out, func(i, j int) bool {
if out[i].PRs != out[j].PRs {
return out[i].PRs > out[j].PRs
}
return out[i].Repo < out[j].Repo
})
return out
}

// governorPRModelWeekSlices finishes the per-week slices, oldest first.
func governorPRModelWeekSlices(weeks map[string]*governorPRModelSliceState) []governorPRModelWeekBucket {
out := make([]governorPRModelWeekBucket, 0, len(weeks))
for week, st := range weeks {
rw := st.rework.finish()
out = append(out, governorPRModelWeekBucket{
WeekStart: week, PRs: st.prs, Merged: st.merged,
FirstPassMerged: rw.FirstPassMerged, FirstPassMergeRate: rw.FirstPassMergeRate,
AvgReviewRounds: rw.AvgReviewRounds, AvgFixAttempts: rw.AvgFixAttempts,
})
}
sort.Slice(out, func(i, j int) bool { return out[i].WeekStart < out[j].WeekStart })
return out
}

// governorPRModelSource is one raw attribution spelling folded into a bucket.
Expand Down Expand Up @@ -230,7 +304,13 @@ func aggregateGovernorPRModels(prs []ghpkg.PullRequest, window string, now time.
// Per-source and per-effort slice accumulators (#18715).
sourceStats map[string]*governorPRModelSliceState
efforts map[string]*governorPRModelSliceState
// Per-repo and per-week slices (#18716); repoNames keeps the first
// spelling seen for each case-folded repo key.
repos map[string]*governorPRModelSliceState
repoNames map[string]string
weeks map[string]*governorPRModelSliceState
}
withWeeks := window != governorPRModelsWindow7d
buckets := map[string]*bucketState{}
resp := governorPRModelsResponse{Window: window, MinMergedPRs: effectiveModelsMinMergedPRs()}
for _, pr := range prs {
Expand All @@ -256,6 +336,9 @@ func aggregateGovernorPRModels(prs []ghpkg.PullRequest, window string, now time.
backends: map[string]bool{},
sourceStats: map[string]*governorPRModelSliceState{},
efforts: map[string]*governorPRModelSliceState{},
repos: map[string]*governorPRModelSliceState{},
repoNames: map[string]string{},
weeks: map[string]*governorPRModelSliceState{},
}
buckets[model] = b
}
Expand All @@ -277,6 +360,15 @@ func aggregateGovernorPRModels(prs []ghpkg.PullRequest, window string, now time.
}
governorPRModelSlice(b.sourceStats, srcKey).add(outcome, pr.Rework)
governorPRModelSlice(b.efforts, governorPRModelEffortKey(pr.HiveEffort)).add(outcome, pr.Rework)
repoName := strings.TrimSpace(effectivePRRepo(pr))
repoKey := strings.ToLower(repoName)
if _, ok := b.repoNames[repoKey]; !ok {
b.repoNames[repoKey] = repoName
}
governorPRModelSlice(b.repos, repoKey).add(outcome, pr.Rework)
if withWeeks && !pr.CreatedAt.IsZero() {
governorPRModelSlice(b.weeks, governorPRModelWeekStart(pr.CreatedAt)).add(outcome, pr.Rework)
}
if score := pr.Rework.ReviewRounds + pr.Rework.FixAttempts; score > 0 {
resp.MostReworked = append(resp.MostReworked, governorPRMostReworkedItem{
Repo: pr.Repo,
Expand Down Expand Up @@ -313,6 +405,10 @@ func aggregateGovernorPRModels(prs []ghpkg.PullRequest, window string, now time.
resp.Buckets = make([]governorPRModelBucket, 0, len(buckets))
for _, b := range buckets {
b.Rework = b.rework.finish()
b.Repos = governorPRModelRepoSlices(b.repos, b.repoNames)
if withWeeks && len(b.weeks) > 0 {
b.Weeks = governorPRModelWeekSlices(b.weeks)
}
b.Backends, b.Backend = joinedBackends(b.backends)
for key, src := range b.sources {
if st := b.sourceStats[key]; st != nil {
Expand Down
109 changes: 109 additions & 0 deletions src/pkg/dashboard/api_governor_pr_models_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -303,3 +303,112 @@ func TestAggregateGovernorPRModelsPerSourceAndEffortRework(t *testing.T) {
}
}
}

// TestAggregateGovernorPRModelsReposAndWeeks covers the per-repo and per-week
// slices (#18716): a short repo name with a full URL folds into the full-name
// entry, weeks start Monday 00:00 UTC, totals add back to the bucket, and
// weeks are absent for window=7d.
func TestAggregateGovernorPRModelsReposAndWeeks(t *testing.T) {
// Wednesday 2026-09-23 12:00 UTC.
now := time.Date(2026, 9, 23, 12, 0, 0, 0, time.UTC)
monday := time.Date(2026, 9, 21, 0, 0, 0, 0, time.UTC)
pr := func(repo string, n int, url string, created time.Time, merged bool, rw ghpkg.PRReworkStats) ghpkg.PullRequest {
p := ghpkg.PullRequest{Repo: repo, Number: n, URL: url, HiveAttributed: true, HiveModel: "opus", HiveBackend: "claude", CreatedAt: created, Rework: rw}
if merged {
p.MergedAt = now
} else {
p.State = "open"
}
return p
}
prs := []ghpkg.PullRequest{
// Week of 2026-09-21: exactly Monday 00:00 UTC, plus one more.
pr("org/alpha", 1, "https://github.com/org/alpha/pull/1", monday, true, ghpkg.PRReworkStats{FirstPass: true}),
// Short configured name, full URL: must fold into org/alpha.
pr("alpha", 2, "https://github.com/org/alpha/pull/2", monday.Add(time.Hour), true, ghpkg.PRReworkStats{ReviewRounds: 2, FixAttempts: 1}),
// One second before the Monday boundary: previous week (2026-09-14).
pr("org/beta", 3, "https://github.com/org/beta/pull/3", monday.Add(-time.Second), true, ghpkg.PRReworkStats{FirstPass: true}),
// Week of 2026-09-07; open, so no rework counted.
pr("org/alpha", 4, "https://github.com/org/alpha/pull/4", monday.AddDate(0, 0, -13), false, ghpkg.PRReworkStats{ReviewRounds: 9}),
}

got := aggregateGovernorPRModels(prs, "30d", now)
if len(got.Buckets) != 1 {
t.Fatalf("buckets = %+v, want 1", got.Buckets)
}
b := got.Buckets[0]
if b.PRs != 4 || b.Merged != 3 || b.Rework.FirstPassMerged != 2 {
t.Fatalf("bucket = %+v, want 4 PRs, 3 merged, 2 first pass", b)
}

if len(b.Repos) != 2 {
t.Fatalf("repos = %+v, want org/alpha and org/beta", b.Repos)
}
alpha, beta := b.Repos[0], b.Repos[1]
if alpha.Repo != "org/alpha" || alpha.PRs != 3 || alpha.Merged != 2 || alpha.Open != 1 || alpha.FirstPassMerged != 1 ||
alpha.FirstPassMergeRate != 0.5 || alpha.AvgReviewRounds != 1 || alpha.AvgFixAttempts != 0.5 {
t.Fatalf("org/alpha = %+v", alpha)
}
if beta.Repo != "org/beta" || beta.PRs != 1 || beta.Merged != 1 || beta.FirstPassMerged != 1 || beta.FirstPassMergeRate != 1 {
t.Fatalf("org/beta = %+v", beta)
}

wantWeeks := []governorPRModelWeekBucket{
{WeekStart: "2026-09-07", PRs: 1, Merged: 0},
{WeekStart: "2026-09-14", PRs: 1, Merged: 1, FirstPassMerged: 1, FirstPassMergeRate: 1},
{WeekStart: "2026-09-21", PRs: 2, Merged: 2, FirstPassMerged: 1, FirstPassMergeRate: 0.5, AvgReviewRounds: 1, AvgFixAttempts: 0.5},
}
if len(b.Weeks) != len(wantWeeks) {
t.Fatalf("weeks = %+v, want %+v", b.Weeks, wantWeeks)
}
for i, w := range wantWeeks {
if b.Weeks[i] != w {
t.Fatalf("weeks[%d] = %+v, want %+v", i, b.Weeks[i], w)
}
}

for _, window := range []string{"30d", "all", "7d"} {
resp := aggregateGovernorPRModels(prs, window, now)
for _, bk := range resp.Buckets {
var rp, rm, rf int
for _, r := range bk.Repos {
rp, rm, rf = rp+r.PRs, rm+r.Merged, rf+r.FirstPassMerged
}
if rp != bk.PRs || rm != bk.Merged || rf != bk.Rework.FirstPassMerged {
t.Fatalf("%s %s: repos sum %d/%d/%d, bucket %d/%d/%d", window, bk.Model, rp, rm, rf, bk.PRs, bk.Merged, bk.Rework.FirstPassMerged)
}
if window == "7d" {
if bk.Weeks != nil {
t.Fatalf("7d %s: weeks = %+v, want absent", bk.Model, bk.Weeks)
}
continue
}
var wp, wm, wf int
for _, w := range bk.Weeks {
wp, wm, wf = wp+w.PRs, wm+w.Merged, wf+w.FirstPassMerged
}
if wp != bk.PRs || wm != bk.Merged || wf != bk.Rework.FirstPassMerged {
t.Fatalf("%s %s: weeks sum %d/%d/%d, bucket %d/%d/%d", window, bk.Model, wp, wm, wf, bk.PRs, bk.Merged, bk.Rework.FirstPassMerged)
}
}
}

raw, err := json.Marshal(aggregateGovernorPRModels(prs, "7d", now))
if err != nil {
t.Fatal(err)
}
var decoded struct {
Buckets []map[string]any `json:"buckets"`
}
if err := json.Unmarshal(raw, &decoded); err != nil {
t.Fatal(err)
}
for _, bk := range decoded.Buckets {
if _, ok := bk["weeks"]; ok {
t.Fatalf("window=7d JSON carries weeks: %v", bk)
}
if _, ok := bk["repos"]; !ok {
t.Fatalf("window=7d JSON missing repos: %v", bk)
}
}
}
6 changes: 6 additions & 0 deletions src/pkg/dashboard/governor_pr_models_ui_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,12 @@ func TestGovernorPRModelsUIRendersReworkStats(t *testing.T) {
"By effort:",
"avg_fix_attempts",
"sample_prs",
"governorPRModelSliceDetail",
"b?.repos",
"b?.weeks",
"week_start",
"By repository:",
"Weekly trend",
} {
if !strings.Contains(s, want) {
t.Fatalf("static/index.html missing %q", want)
Expand Down
18 changes: 18 additions & 0 deletions src/pkg/dashboard/pr_models_header_once.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ const src = [
'const dashboardSectionHelpMark = () => "";',
'const formatFixTime = () => "";',
pick(/function governorPRModelUnattributed\(b\) \{[\s\S]*?\n \}\n/, 'governorPRModelUnattributed'),
pick(/function governorPRModelSliceDetail\(b\) \{[\s\S]*?\n \}\n/, 'governorPRModelSliceDetail'),
pick(/const GOV_PR_MODEL_COLUMNS = \[[\s\S]*?\n \];\n/, 'GOV_PR_MODEL_COLUMNS'),
pick(/function governorPRModelRows\(data\) \{[\s\S]*?\n \}\n/, 'governorPRModelRows'),
pick(/function renderGovernorPRModelsTile\(\) \{[\s\S]*?\n \}\n\n function renderGovernor\(/, 'renderGovernorPRModelsTile').replace(/\n\n function renderGovernor\($/, ''),
Expand Down Expand Up @@ -82,3 +83,20 @@ test('raw model ids and backends surface in the row tooltip and chips', () => {
assert.ok(out.includes('<span class="gov-pr-model-backend">copilot</span>'), 'each backend a chip');
assert.ok(out.includes('Wilson lower bound'), 'tooltip states the rank formula');
});

test('per-repository and weekly lines surface in the row tooltip when present (#18716)', () => {
const withSlices = {
...data,
buckets: data.buckets.map(b => b.model === 'claude-opus-5.5' ? {
...b,
repos: [{ repo: 'org/alpha', prs: 22, merged: 22, first_pass_merge_rate: 0.95, avg_fix_attempts: 0.1 }],
weeks: [{ week_start: '2026-11-09', prs: 22, merged: 22, first_pass_merge_rate: 0.95, avg_fix_attempts: 0.1 }],
} : b),
};
const out = lifted.render(withSlices);
assert.ok(out.includes('By repository:'), 'tooltip lists repositories');
assert.ok(out.includes('org/alpha: 22 PRs (22 merged, first-pass 95%, avg fix 0.1)'), 'repo line rendered');
assert.ok(out.includes('2026-11-09: 22 PRs (22 merged, first-pass 95%, avg fix 0.1)'), 'weekly line rendered');
assert.strictEqual((out.match(/By repository:/g) || []).length, 2, 'only the bucket with repos gets the block (title + aria-label)');
assert.ok(!lifted.render(data).includes('By repository:'), 'no slice block for buckets without repos/weeks');
});
Loading
Loading