From 2be85301615cfef668eea78eca64831707b29f77 Mon Sep 17 00:00:00 2001 From: Danathar <6772335+Danathar@users.noreply.github.com> Date: Tue, 17 Nov 2026 11:58:51 -0500 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20PRs=20by=20model:=20split=20closed-?= =?UTF-8?q?unmerged=20PRs=20by=20close=20reason?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each /api/governor/pr-models bucket gains closed_unmerged_reasons {superseded, duplicate, other}, summing to closed_unmerged. Superseded = hive/superseded label, or another PR in the same repo (attributed or open items) declares Supersedes #N for it; duplicate = a duplicate label; other = everything else. Closed attributed PR records now copy Supersedes from the body the scan already has, so no new GitHub calls. The row tooltip shows the split after the closed-unmerged count. Refs #18719 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WcWm7qFmFJp26r9LPdmi3t Signed-off-by: Danathar <6772335+Danathar@users.noreply.github.com> --- .../added-18719-pr-models-close-reasons.md | 1 + .../overview-and-metrics.md | 1 + src/pkg/dashboard/api_governor_pr_models.go | 5 + .../api_governor_pr_models_close_reasons.go | 136 ++++++++++++++++++ ...i_governor_pr_models_close_reasons_test.go | 125 ++++++++++++++++ .../dashboard/pr_models_close_reasons.test.js | 49 +++++++ .../dashboard/static-src/js/governor-view.js | 4 +- src/pkg/dashboard/static/index.html | 4 +- .../attributed_closed_pr_supersedes_test.go | 63 ++++++++ src/pkg/github/client.go | 4 + 10 files changed, 390 insertions(+), 2 deletions(-) create mode 100644 changelog.d/added-18719-pr-models-close-reasons.md create mode 100644 src/pkg/dashboard/api_governor_pr_models_close_reasons.go create mode 100644 src/pkg/dashboard/api_governor_pr_models_close_reasons_test.go create mode 100644 src/pkg/dashboard/pr_models_close_reasons.test.js create mode 100644 src/pkg/github/attributed_closed_pr_supersedes_test.go diff --git a/changelog.d/added-18719-pr-models-close-reasons.md b/changelog.d/added-18719-pr-models-close-reasons.md new file mode 100644 index 0000000000..045c1f61d9 --- /dev/null +++ b/changelog.d/added-18719-pr-models-close-reasons.md @@ -0,0 +1 @@ +- **Governor PRs by model says why unmerged PRs were closed (#18719).** Each `/api/governor/pr-models` bucket gains `closed_unmerged_reasons` (`superseded`, `duplicate`, `other`), which always sum to `closed_unmerged`. A PR counts as superseded when it carries `hive/superseded` or another PR in the same repository says `Supersedes #N` for it; duplicate when it has a `duplicate` label; anything else is other. Hover a row to see the split after the closed-unmerged count. Closed attributed PR records now keep their `Supersedes #N` number too, read from the body the scan already fetches, so no new GitHub calls. Existing fields, the bar, merge rate and rank are unchanged. diff --git a/src/docs/dashboard-sections/overview-and-metrics.md b/src/docs/dashboard-sections/overview-and-metrics.md index 4d73101f7f..a6be9b98c4 100644 --- a/src/docs/dashboard-sections/overview-and-metrics.md +++ b/src/docs/dashboard-sections/overview-and-metrics.md @@ -96,6 +96,7 @@ Which AI models wrote the pull requests Hive opened, how well each model's work - Each row is one model. Model ids are normalised so one model gets one row: a provider prefix (`openai-codex/gpt-6.1-sol`), a `:variant` suffix (`gpt-5.6-sol:high`) and a dash between version digits (`claude-opus-5-5`) are folded into `gpt-6.1-sol`, `gpt-5.6-sol` and `claude-opus-5.5`. The backends that drove the model appear as chips under its name; hover the row to see the raw ids and how many pull requests each contributed. - The row's tooltip also splits the model by backend and by effort, the reasoning effort the PR's `— hive:` signature line records (`effort=high`; **unknown** when the line has none). Each backend line and each effort line shows its pull requests (merged, open, closed without merging), how many merged first pass, and the average review rounds and fix attempts of its merged pull requests, so you can see whether one harness or effort level needed more fixing. The API returns these as the extra fields on `sources[]` and as `efforts[]` on each model. - The bar splits a model's pull requests into merged, still open, and closed without merging. +- Hover a row to see why its closed pull requests were closed: **superseded** (it carries the `hive/superseded` label, or another pull request in the same repository says `Supersedes #N` for it), **duplicate** (it has a `duplicate` label), or **other** (abandoned or rejected). Only **other** says something about the model. The three always add up to the closed count. The API returns them as `closed_unmerged_reasons` on each bucket of `/api/governor/pr-models`. - **Merged** is the number of merged pull requests in the window. - **1st pass** is the share of merged pull requests that merged without a review-requested fix round. - **PR runs** is the share of recorded task runs (from the contributor run log) that ended with a verified pull request. diff --git a/src/pkg/dashboard/api_governor_pr_models.go b/src/pkg/dashboard/api_governor_pr_models.go index 15c5af130c..8c72e75df1 100644 --- a/src/pkg/dashboard/api_governor_pr_models.go +++ b/src/pkg/dashboard/api_governor_pr_models.go @@ -57,6 +57,10 @@ type governorPRModelBucket struct { // Efforts splits the bucket by the trailer's effort= value ("unknown" // when absent), sorted by PRs descending (hivecommons/hive#18715). Efforts []governorPRModelEffort `json:"efforts,omitempty"` + + // ClosedUnmergedReasons splits ClosedUnmerged by why the PR was closed + // (#18719); the three counts always sum to ClosedUnmerged. + ClosedUnmergedReasons governorPRModelCloseReasons `json:"closed_unmerged_reasons"` } // governorPRModelSource is one raw attribution spelling folded into a bucket. @@ -180,6 +184,7 @@ func (s *Server) handleGovernorPRModels(w http.ResponseWriter, r *http.Request) minMergedPRs := effectiveModelsMinMergedPRs() actionable := s.lastActionableForPRModels() resp := aggregateGovernorPRModels(actionable.PRs.Attributed, window, time.Now()) + applyGovernorPRModelCloseReasons(&resp, actionable.PRs.Attributed, governorPRModelsSupersededKeys(actionable.PRs.Attributed, actionable.PRs.Items), time.Now()) resp.FixerQueue = s.governorFixerQueue() now := time.Now() runs, rotated, _ := readEffectiveModelRuns(s.effectiveModelTaskRunLogPath(), governorPRModelsWindowDuration(window)) diff --git a/src/pkg/dashboard/api_governor_pr_models_close_reasons.go b/src/pkg/dashboard/api_governor_pr_models_close_reasons.go new file mode 100644 index 0000000000..b7a25ba784 --- /dev/null +++ b/src/pkg/dashboard/api_governor_pr_models_close_reasons.go @@ -0,0 +1,136 @@ +package dashboard + +import ( + "strings" + "time" + + ghpkg "github.com/hivecommons/hive/pkg/github" + "github.com/hivecommons/hive/pkg/modelfamily" +) + +// Close reasons for the PRs-by-model card (hivecommons/hive#18719). A PR the +// hive closed because another PR landed the same change, or closed as a +// duplicate, says nothing about the model that wrote it; only "other" +// (abandoned or rejected) does. The split is computed from labels and the +// "Supersedes #N" body line the hive already records, with no extra GitHub +// calls. Telling a human's close from an author giving up would need the +// timeline API, so those stay in "other". +const ( + governorPRCloseReasonSuperseded = "superseded" + governorPRCloseReasonDuplicate = "duplicate" + governorPRCloseReasonOther = "other" + + governorPRDuplicateLabel = "duplicate" +) + +// governorPRModelCloseReasons splits a bucket's closed_unmerged count. The +// three fields always sum to closed_unmerged. +type governorPRModelCloseReasons struct { + Superseded int `json:"superseded"` + Duplicate int `json:"duplicate"` + Other int `json:"other"` +} + +// governorPRModelsSupersededKeys returns the repo#number keys of PRs that +// another PR declares it supersedes ("Supersedes #N" in its body, recorded +// as PullRequest.Supersedes). The referenced PR is in the superseder's own +// repository. +func governorPRModelsSupersededKeys(lists ...[]ghpkg.PullRequest) map[string]bool { + keys := map[string]bool{} + for _, prs := range lists { + for _, pr := range prs { + if pr.Supersedes <= 0 || pr.Supersedes == pr.Number { + continue + } + keys[effectivePRKey(effectivePRRepo(pr), pr.Supersedes)] = true + } + } + return keys +} + +// governorPRModelCloseReason classifies a closed-unmerged PR. Superseded +// wins over duplicate when both apply. +func governorPRModelCloseReason(pr ghpkg.PullRequest, superseded map[string]bool) string { + duplicate := false + for _, l := range pr.Labels { + switch { + case strings.EqualFold(strings.TrimSpace(l), ghpkg.SupersededLabel): + return governorPRCloseReasonSuperseded + case strings.EqualFold(strings.TrimSpace(l), governorPRDuplicateLabel): + duplicate = true + } + } + if superseded[effectivePRKey(effectivePRRepo(pr), pr.Number)] { + return governorPRCloseReasonSuperseded + } + if duplicate { + return governorPRCloseReasonDuplicate + } + return governorPRCloseReasonOther +} + +func (r *governorPRModelCloseReasons) add(reason string) { + switch reason { + case governorPRCloseReasonSuperseded: + r.Superseded++ + case governorPRCloseReasonDuplicate: + r.Duplicate++ + default: + r.Other++ + } +} + +// governorPRModelsCutoff is the created-at cutoff for a normalized window; +// zero for "all". It mirrors the cutoff in aggregateGovernorPRModels. +func governorPRModelsCutoff(window string, now time.Time) time.Time { + switch normalizeGovernorPRModelsWindow(window) { + case governorPRModelsWindow7d: + return now.AddDate(0, 0, -governorPRModelsDays7) + case governorPRModelsWindow30d: + return now.AddDate(0, 0, -governorPRModelsDays30) + } + return time.Time{} +} + +// applyGovernorPRModelCloseReasons fills ClosedUnmergedReasons on each bucket +// of an aggregateGovernorPRModels response built from the same prs, window +// and now. It walks the PRs with the same filter and bucket key. +func applyGovernorPRModelCloseReasons(resp *governorPRModelsResponse, prs []ghpkg.PullRequest, superseded map[string]bool, now time.Time) { + if resp == nil || len(resp.Buckets) == 0 { + return + } + cutoff := governorPRModelsCutoff(resp.Window, now) + reasons := map[string]*governorPRModelCloseReasons{} + for _, pr := range prs { + if !pr.HiveAttributed { + continue + } + if !cutoff.IsZero() && pr.CreatedAt.Before(cutoff) { + continue + } + if governorPRModelOutcome(pr) != "closed_unmerged" { + continue + } + model := modelfamily.Canonical(pr.HiveModel) + r := reasons[model] + if r == nil { + r = &governorPRModelCloseReasons{} + reasons[model] = r + } + r.add(governorPRModelCloseReason(pr, superseded)) + } + for i := range resp.Buckets { + b := &resp.Buckets[i] + r := governorPRModelCloseReasons{} + if got := reasons[b.Model]; got != nil { + r = *got + } + // Other is the remainder, so the sum matches closed_unmerged even if + // a PR sits on the window edge between the two time.Now() calls. + r.Other = b.ClosedUnmerged - r.Superseded - r.Duplicate + if r.Other < 0 { + r.Other = 0 + } + b.ClosedUnmergedReasons = r + } +} diff --git a/src/pkg/dashboard/api_governor_pr_models_close_reasons_test.go b/src/pkg/dashboard/api_governor_pr_models_close_reasons_test.go new file mode 100644 index 0000000000..3019a83867 --- /dev/null +++ b/src/pkg/dashboard/api_governor_pr_models_close_reasons_test.go @@ -0,0 +1,125 @@ +package dashboard + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "strconv" + "testing" + "time" + + ghpkg "github.com/hivecommons/hive/pkg/github" + "github.com/hivecommons/hive/pkg/modelfamily" +) + +func closedReasonsFor(t *testing.T, resp governorPRModelsResponse, model string) governorPRModelBucket { + t.Helper() + for _, b := range resp.Buckets { + if b.Model == modelfamily.Canonical(model) { + return b + } + } + t.Fatalf("no bucket %q in %+v", model, resp.Buckets) + return governorPRModelBucket{} +} + +func assertCloseReasonsSum(t *testing.T, resp governorPRModelsResponse) { + t.Helper() + for _, b := range resp.Buckets { + r := b.ClosedUnmergedReasons + if r.Superseded+r.Duplicate+r.Other != b.ClosedUnmerged { + t.Fatalf("bucket %s: reasons %+v do not sum to closed_unmerged %d", b.Model, r, b.ClosedUnmerged) + } + } +} + +func TestGovernorPRModelsCloseReasons(t *testing.T) { + now := time.Date(2026, 11, 17, 12, 0, 0, 0, time.UTC) + closed := func(number int, model string, labels ...string) ghpkg.PullRequest { + return ghpkg.PullRequest{ + Repo: "org/repo", Number: number, URL: "https://github.com/org/repo/pull/" + strconv.Itoa(number), + HiveAttributed: true, HiveModel: model, HiveBackend: "claude", + CreatedAt: now.AddDate(0, 0, -1), State: "closed", ClosedAt: now, Labels: labels, + } + } + attributed := []ghpkg.PullRequest{ + closed(1, "opus", ghpkg.SupersededLabel), // superseded by label + closed(2, "opus"), // superseded by another PR (#10 below) + closed(3, "opus", "Duplicate"), // duplicate (case-insensitive) + closed(4, "opus"), // other + closed(5, "opus", "duplicate", "hive/superseded"), // both: superseded wins + closed(6, "sonnet"), // other; #20 in another repo supersedes a #6 there + {Repo: "org/repo", Number: 10, URL: "https://github.com/org/repo/pull/10", HiveAttributed: true, HiveModel: "opus", CreatedAt: now.AddDate(0, 0, -1), MergedAt: now, Supersedes: 2}, + {Repo: "org/repo", Number: 11, URL: "https://github.com/org/repo/pull/11", HiveAttributed: true, HiveModel: "opus", CreatedAt: now.AddDate(0, 0, -1), State: "open"}, + // Outside the 7d window: not counted anywhere. + {Repo: "org/repo", Number: 12, HiveAttributed: true, HiveModel: "opus", CreatedAt: now.AddDate(0, 0, -20), State: "closed", ClosedAt: now, Labels: []string{"duplicate"}}, + } + open := []ghpkg.PullRequest{ + {Repo: "org/other", Number: 20, URL: "https://github.com/org/other/pull/20", State: "open", Supersedes: 6}, + } + superseded := governorPRModelsSupersededKeys(attributed, open) + + resp := aggregateGovernorPRModels(attributed, "7d", now) + applyGovernorPRModelCloseReasons(&resp, attributed, superseded, now) + assertCloseReasonsSum(t, resp) + + opus := closedReasonsFor(t, resp, "opus") + if opus.ClosedUnmerged != 5 || opus.Merged != 1 || opus.Open != 1 { + t.Fatalf("opus outcomes changed: %+v", opus) + } + if want := (governorPRModelCloseReasons{Superseded: 3, Duplicate: 1, Other: 1}); opus.ClosedUnmergedReasons != want { + t.Fatalf("opus reasons = %+v, want %+v", opus.ClosedUnmergedReasons, want) + } + sonnet := closedReasonsFor(t, resp, "sonnet") + if want := (governorPRModelCloseReasons{Other: 1}); sonnet.ClosedUnmergedReasons != want { + t.Fatalf("sonnet reasons = %+v, want %+v (a Supersedes in another repo must not match)", sonnet.ClosedUnmergedReasons, want) + } + + all := aggregateGovernorPRModels(attributed, "all", now) + applyGovernorPRModelCloseReasons(&all, attributed, superseded, now) + assertCloseReasonsSum(t, all) + if got := closedReasonsFor(t, all, "opus").ClosedUnmergedReasons.Duplicate; got != 2 { + t.Fatalf("all-window opus duplicate = %d, want 2", got) + } +} + +func TestHandleGovernorPRModelsCloseReasonsFromOpenSuperseder(t *testing.T) { + s := covApiServer(t) + now := time.Now().UTC() + s.deps.Scheduler = metricsSchedulerStub{actionable: &ghpkg.ActionableResult{ + PRs: ghpkg.PRResult{ + Attributed: []ghpkg.PullRequest{ + {Repo: "org/repo", Number: 1, URL: "https://github.com/org/repo/pull/1", HiveAttributed: true, HiveModel: "opus", CreatedAt: now, State: "closed", ClosedAt: now}, + {Repo: "org/repo", Number: 2, URL: "https://github.com/org/repo/pull/2", HiveAttributed: true, HiveModel: "opus", CreatedAt: now, State: "closed", ClosedAt: now}, + }, + Items: []ghpkg.PullRequest{ + {Repo: "org/repo", Number: 3, URL: "https://github.com/org/repo/pull/3", State: "open", Supersedes: 1}, + }, + }, + }} + req := httptest.NewRequest(http.MethodGet, "/api/governor/pr-models?window=7d", nil) + rec := httptest.NewRecorder() + s.handleGovernorPRModels(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200", rec.Code) + } + var raw struct { + Buckets []map[string]json.RawMessage `json:"buckets"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &raw); err != nil { + t.Fatalf("decode: %v", err) + } + if len(raw.Buckets) != 1 { + t.Fatalf("buckets = %d, want 1", len(raw.Buckets)) + } + var reasons governorPRModelCloseReasons + if err := json.Unmarshal(raw.Buckets[0]["closed_unmerged_reasons"], &reasons); err != nil { + t.Fatalf("closed_unmerged_reasons missing or malformed: %v", err) + } + if want := (governorPRModelCloseReasons{Superseded: 1, Other: 1}); reasons != want { + t.Fatalf("closed_unmerged_reasons = %+v, want %+v", reasons, want) + } + if string(raw.Buckets[0]["closed_unmerged"]) != "2" { + t.Fatalf("closed_unmerged = %s, want 2 (unchanged)", raw.Buckets[0]["closed_unmerged"]) + } +} diff --git a/src/pkg/dashboard/pr_models_close_reasons.test.js b/src/pkg/dashboard/pr_models_close_reasons.test.js new file mode 100644 index 0000000000..53c6f6ee85 --- /dev/null +++ b/src/pkg/dashboard/pr_models_close_reasons.test.js @@ -0,0 +1,49 @@ +// "PRs by model" row tooltip splits closed-unmerged PRs by why they were +// closed (hivecommons/hive#18719): superseded, duplicate, other. A server +// that predates closed_unmerged_reasons, or a row with nothing closed, +// keeps the old line unchanged. +const test = require('node:test'); +const assert = require('node:assert'); +const fs = require('node:fs'); +const path = require('node:path'); + +const html = fs.readFileSync(path.join(__dirname, 'static', 'index.html'), 'utf8'); + +function pick(re, what) { + const m = html.match(re); + assert.ok(m, what + ' not found in index.html'); + return m[0]; +} + +const src = [ + 'let govPRModelsSort = "effectiveness"; let govPRModelsWindow = "7d"; let govPRModelsData = null;', + 'const escapeHtml = s => String(s).replace(/[&<>"\']/g, c => ({"&":"&","<":"<",">":">",\'"\':""","\'":"'"}[c]));', + 'const dashboardSectionHelpMark = () => "";', + 'const formatFixTime = () => "";', + pick(/function governorPRModelUnattributed\(b\) \{[\s\S]*?\n \}\n/, 'governorPRModelUnattributed'), + pick(/const GOV_PR_MODEL_COLUMNS = \[[\s\S]*?\n \];\n/, 'GOV_PR_MODEL_COLUMNS'), + pick(/function governorPRModelRows\(data\) \{[\s\S]*?\n \}\n/, 'governorPRModelRows'), + pick(/function renderGovernorPRModelsTile\(\) \{[\s\S]*?\n \}\n\n function renderGovernor\(/, 'renderGovernorPRModelsTile').replace(/\n\n function renderGovernor\($/, ''), +].join('\n'); +const lifted = new Function(src + '; return { render: data => { govPRModelsData = data; return renderGovernorPRModelsTile(); } };')(); + +const bucket = (model, closed, reasons) => ({ + model, backend: 'claude', backends: ['claude'], prs: 10, merged: 10 - closed, open: 0, closed_unmerged: closed, + ...(reasons ? { closed_unmerged_reasons: reasons } : {}), + rework: {}, effectiveness: {}, sources: [], +}); + +const render = b => lifted.render({ window: '7d', total: b.prs, min_merged_prs: 5, buckets: [b] }); + +test('row tooltip splits closed-unmerged PRs by close reason', () => { + const out = render(bucket('claude-opus-5.5', 6, { superseded: 3, duplicate: 1, other: 2 })); + assert.ok(out.includes('6 closed-unmerged (3 superseded · 1 duplicate · 2 other)'), 'split rendered after the closed-unmerged count'); +}); + +test('no split when the server predates the field or nothing was closed', () => { + const old = render(bucket('claude-opus-5.5', 2, null)); + assert.ok(old.includes('2 closed-unmerged\n'), 'old line kept'); + assert.ok(!old.includes('superseded'), 'no split without the field'); + const none = render(bucket('claude-opus-5.5', 0, { superseded: 0, duplicate: 0, other: 0 })); + assert.ok(!none.includes('superseded'), 'no split when nothing was closed'); +}); diff --git a/src/pkg/dashboard/static-src/js/governor-view.js b/src/pkg/dashboard/static-src/js/governor-view.js index f275439581..c9698f1632 100644 --- a/src/pkg/dashboard/static-src/js/governor-view.js +++ b/src/pkg/dashboard/static-src/js/governor-view.js @@ -168,9 +168,11 @@ const sources = (b.sources || []).map(src => `${src.model || 'unknown'} / ${src.backend || 'unknown'}: ${sliceRework(src)}`).join('\n'); const effortBreakdown = (b.efforts || []).map(e => `effort ${e.effort || 'unknown'}: ${sliceRework(e)}`).join('\n'); const agentBreakdown = (b.agents || []).map(a => `${a.agent || 'unknown'} / ${a.backend || b.backend || 'unknown'}: ${a.prs} PRs (${a.merged || 0} merged, ${a.open || 0} open, ${a.closed_unmerged || 0} closed-unmerged)`).join('\n'); + // Why the closed-unmerged PRs were closed (#18719); empty when the server predates the field. + const closeReasons = b.closed_unmerged_reasons && (b.closed_unmerged || 0) > 0 ? ` (${b.closed_unmerged_reasons.superseded || 0} superseded · ${b.closed_unmerged_reasons.duplicate || 0} duplicate · ${b.closed_unmerged_reasons.other || 0} other)` : ''; const scoreFormula = measured ? ' × (1 − fail) × (1 − no ship)' : ' (first-pass bound only: no runs recorded, so the run factors were skipped)'; const scoreLine = eff.rank ? `\nRank #${eff.rank} · score ${(Number(eff.score) || 0).toFixed(3)} = Wilson lower bound(${eff.first_pass_merged ?? '?'}/${eff.merged_prs ?? b.merged ?? 0} first pass)${scoreFormula}` : (unattributed ? '\nUnattributed: no model recorded, listed but not ranked' : `\nUnranked: fewer than ${data.min_merged_prs || 5} merged PRs`); - const tip = `${b.model || 'unknown'} via ${backends.join(', ')}${scoreLine}\n${prs} PRs (${share.toFixed(1)}%)\n${b.merged || 0} merged · ${b.open || 0} open · ${b.closed_unmerged || 0} closed-unmerged\nFirst-pass ${values.first_pass}${ci(eff.first_pass_merge_rate_ci || rw.first_pass_merge_rate_ci)} · PR run rate ${values.pr_runs}${measured ? ci(eff.verified_pr_run_rate_ci) : ''} · failure ${values.fail}${measured ? ci(eff.failure_rate_ci) : ''} · nothing-to-ship ${values.no_ship}${measured ? ci(eff.nothing_to_ship_rate_ci) : ''}${measured ? '' : '\n— = no runs recorded, not a measured 0%'}\nAvg rounds ${(rw.avg_review_rounds || 0).toFixed(1)} · avg fix attempts ${(rw.avg_fix_attempts || 0).toFixed(1)}${sources ? '\n\nRaw model ids folded into this row:\n' + sources : ''}${agentBreakdown ? '\n\n' + agentBreakdown : ''}${effortBreakdown ? '\n\nBy effort:\n' + effortBreakdown : ''}`; + const tip = `${b.model || 'unknown'} via ${backends.join(', ')}${scoreLine}\n${prs} PRs (${share.toFixed(1)}%)\n${b.merged || 0} merged · ${b.open || 0} open · ${b.closed_unmerged || 0} closed-unmerged${closeReasons}\nFirst-pass ${values.first_pass}${ci(eff.first_pass_merge_rate_ci || rw.first_pass_merge_rate_ci)} · PR run rate ${values.pr_runs}${measured ? ci(eff.verified_pr_run_rate_ci) : ''} · failure ${values.fail}${measured ? ci(eff.failure_rate_ci) : ''} · nothing-to-ship ${values.no_ship}${measured ? ci(eff.nothing_to_ship_rate_ci) : ''}${measured ? '' : '\n— = no runs recorded, not a measured 0%'}\nAvg rounds ${(rw.avg_review_rounds || 0).toFixed(1)} · avg fix attempts ${(rw.avg_fix_attempts || 0).toFixed(1)}${sources ? '\n\nRaw model ids folded into this row:\n' + sources : ''}${agentBreakdown ? '\n\n' + agentBreakdown : ''}${effortBreakdown ? '\n\nBy effort:\n' + effortBreakdown : ''}`; return `
${rank}${escapeHtml(b.model || 'unknown')}
${backendChips}
diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index 407c388633..aaa1780ce5 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -33868,9 +33868,11 @@

All labels

const sources = (b.sources || []).map(src => `${src.model || 'unknown'} / ${src.backend || 'unknown'}: ${sliceRework(src)}`).join('\n'); const effortBreakdown = (b.efforts || []).map(e => `effort ${e.effort || 'unknown'}: ${sliceRework(e)}`).join('\n'); const agentBreakdown = (b.agents || []).map(a => `${a.agent || 'unknown'} / ${a.backend || b.backend || 'unknown'}: ${a.prs} PRs (${a.merged || 0} merged, ${a.open || 0} open, ${a.closed_unmerged || 0} closed-unmerged)`).join('\n'); + // Why the closed-unmerged PRs were closed (#18719); empty when the server predates the field. + const closeReasons = b.closed_unmerged_reasons && (b.closed_unmerged || 0) > 0 ? ` (${b.closed_unmerged_reasons.superseded || 0} superseded · ${b.closed_unmerged_reasons.duplicate || 0} duplicate · ${b.closed_unmerged_reasons.other || 0} other)` : ''; const scoreFormula = measured ? ' × (1 − fail) × (1 − no ship)' : ' (first-pass bound only: no runs recorded, so the run factors were skipped)'; const scoreLine = eff.rank ? `\nRank #${eff.rank} · score ${(Number(eff.score) || 0).toFixed(3)} = Wilson lower bound(${eff.first_pass_merged ?? '?'}/${eff.merged_prs ?? b.merged ?? 0} first pass)${scoreFormula}` : (unattributed ? '\nUnattributed: no model recorded, listed but not ranked' : `\nUnranked: fewer than ${data.min_merged_prs || 5} merged PRs`); - const tip = `${b.model || 'unknown'} via ${backends.join(', ')}${scoreLine}\n${prs} PRs (${share.toFixed(1)}%)\n${b.merged || 0} merged · ${b.open || 0} open · ${b.closed_unmerged || 0} closed-unmerged\nFirst-pass ${values.first_pass}${ci(eff.first_pass_merge_rate_ci || rw.first_pass_merge_rate_ci)} · PR run rate ${values.pr_runs}${measured ? ci(eff.verified_pr_run_rate_ci) : ''} · failure ${values.fail}${measured ? ci(eff.failure_rate_ci) : ''} · nothing-to-ship ${values.no_ship}${measured ? ci(eff.nothing_to_ship_rate_ci) : ''}${measured ? '' : '\n— = no runs recorded, not a measured 0%'}\nAvg rounds ${(rw.avg_review_rounds || 0).toFixed(1)} · avg fix attempts ${(rw.avg_fix_attempts || 0).toFixed(1)}${sources ? '\n\nRaw model ids folded into this row:\n' + sources : ''}${agentBreakdown ? '\n\n' + agentBreakdown : ''}${effortBreakdown ? '\n\nBy effort:\n' + effortBreakdown : ''}`; + const tip = `${b.model || 'unknown'} via ${backends.join(', ')}${scoreLine}\n${prs} PRs (${share.toFixed(1)}%)\n${b.merged || 0} merged · ${b.open || 0} open · ${b.closed_unmerged || 0} closed-unmerged${closeReasons}\nFirst-pass ${values.first_pass}${ci(eff.first_pass_merge_rate_ci || rw.first_pass_merge_rate_ci)} · PR run rate ${values.pr_runs}${measured ? ci(eff.verified_pr_run_rate_ci) : ''} · failure ${values.fail}${measured ? ci(eff.failure_rate_ci) : ''} · nothing-to-ship ${values.no_ship}${measured ? ci(eff.nothing_to_ship_rate_ci) : ''}${measured ? '' : '\n— = no runs recorded, not a measured 0%'}\nAvg rounds ${(rw.avg_review_rounds || 0).toFixed(1)} · avg fix attempts ${(rw.avg_fix_attempts || 0).toFixed(1)}${sources ? '\n\nRaw model ids folded into this row:\n' + sources : ''}${agentBreakdown ? '\n\n' + agentBreakdown : ''}${effortBreakdown ? '\n\nBy effort:\n' + effortBreakdown : ''}`; return `
${rank}${escapeHtml(b.model || 'unknown')}
${backendChips}
diff --git a/src/pkg/github/attributed_closed_pr_supersedes_test.go b/src/pkg/github/attributed_closed_pr_supersedes_test.go new file mode 100644 index 0000000000..d9c807b91a --- /dev/null +++ b/src/pkg/github/attributed_closed_pr_supersedes_test.go @@ -0,0 +1,63 @@ +package github + +import ( + "context" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "testing" + "time" + + gh "github.com/google/go-github/v72/github" +) + +// #18719: closed attributed PR records carry the "Supersedes #N" number the +// open-PR path already records, so the PRs-by-model card can tell a PR that +// was replaced from one that failed. +func TestPullRequestAttributionRecordCopiesSupersedes(t *testing.T) { + for _, tc := range []struct { + name string + body string + want int + }{ + {name: "declares", body: "Re-authored.\n\nSupersedes #41\n\n— hive: agent=scanner backend=bob model=auto", want: 41}, + {name: "none", body: "done\n\n— hive: agent=scanner backend=bob model=auto", want: 0}, + } { + t.Run(tc.name, func(t *testing.T) { + pr := &gh.PullRequest{Number: gh.Ptr(42), Body: gh.Ptr(tc.body), State: gh.Ptr("closed")} + got := pullRequestAttributionRecord("r", pr, InvocationMeta{Agent: "scanner", Backend: "bob", Model: "auto"}) + if got.Supersedes != tc.want { + t.Fatalf("Supersedes = %d, want %d", got.Supersedes, tc.want) + } + }) + } +} + +func TestFetchAttributedClosedPRsRecordsSupersedesWithoutExtraCalls(t *testing.T) { + t.Setenv(attributedClosedPRLookbackEnv, "24h") + now := time.Now().UTC() + var calls int + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls++ + if r.URL.Path != "/repos/o/r/pulls" || r.URL.Query().Get("state") != "closed" { + t.Fatalf("unexpected request: %s?%s", r.URL.Path, r.URL.RawQuery) + } + superseder := attributedClosedPR(7, now.Add(-time.Hour)) + superseder["body"] = "Takes over the work.\n\nSupersedes #5\n\n— hive: agent=scanner backend=bob model=auto" + writeJSON(t, w, []map[string]any{superseder, attributedClosedPR(5, now.Add(-2*time.Hour))}) + })) + defer srv.Close() + + c := NewClientForTest(srv.URL, "o", []string{"r"}, slog.New(slog.NewTextHandler(io.Discard, nil))) + prs, err := c.fetchAttributedClosedPRs(context.Background(), "o", "r", "r") + if err != nil { + t.Fatalf("fetchAttributedClosedPRs: %v", err) + } + if calls != 1 { + t.Fatalf("GitHub calls = %d, want 1 (same as before #18719)", calls) + } + if len(prs) != 2 || prs[0].Number != 7 || prs[0].Supersedes != 5 || prs[1].Supersedes != 0 { + t.Fatalf("records = %+v, want #7 supersedes 5 and #5 none", prs) + } +} diff --git a/src/pkg/github/client.go b/src/pkg/github/client.go index 35c353ebe7..68fd737053 100644 --- a/src/pkg/github/client.go +++ b/src/pkg/github/client.go @@ -2581,6 +2581,10 @@ func pullRequestAttributionRecord(repo string, pr *gh.PullRequest, meta Invocati HiveBackend: meta.Backend, HiveModel: meta.Model, HiveEffort: meta.Effort, + // Supersedes lets the PRs-by-model card count a closed PR that + // another PR replaced as superseded (#18719); the body is already + // in hand, so this costs no GitHub call. + Supersedes: prSupersedesNumberOrZero(pr.GetBody()), } }