Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/fixed-19365-pr-models-closed-history.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- The PRs-by-model card and Most effective models panel now rank models over the same span: attributed closed PRs are kept in a durable store (`attributed-pr-store.json`, backfilled once to 30 days in bounded pages, `HIVE_ATTRIBUTED_PR_BACKFILL` to change or `off` to skip), so 30d windows no longer lose closed PRs last updated more than 14 days ago, and `coverage.closed_prs_partial` reflects the store's real reach ([#19365](https://github.com/hivecommons/hive/issues/19365)).
2 changes: 1 addition & 1 deletion src/pkg/dashboard/api_governor_pr_models.go
Original file line number Diff line number Diff line change
Expand Up @@ -269,7 +269,7 @@ func (s *Server) handleGovernorPRModels(w http.ResponseWriter, r *http.Request)
resp.FixerQueue = s.governorFixerQueue()
now := time.Now()
runs, rotated, _ := readEffectiveModelRuns(s.effectiveModelTaskRunLogPath(), governorPRModelsWindowDuration(window))
resp.Coverage = effectiveModelsCoverage(governorPRModelsWindowDuration(window), ghpkg.AttributedClosedPRLookback(), runs, rotated, now)
resp.Coverage = effectiveModelsCoverage(governorPRModelsWindowDuration(window), ghpkg.AttributedClosedPRCoverage(), runs, rotated, now)
effective := aggregateEffectiveModelsFromPRAggregation(resp, aggregateEffectiveRunStats(runs, effectiveModelsFilterAll), effectiveModelsFilterAll, minMergedPRs)
applyGovernorPRModelEffectiveness(&resp, effective, minMergedPRs)
launchCost, launchCostReady := s.launchCostSnapshotForPRModels()
Expand Down
30 changes: 30 additions & 0 deletions src/pkg/dashboard/api_governor_pr_models_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -412,3 +412,33 @@ func TestAggregateGovernorPRModelsReposAndWeeks(t *testing.T) {
}
}
}

// Closed PRs the live scan no longer lists (#19365) stay in a 30d window once
// the store reports coverage back to it.
func TestGovernorPRModels30dCountsStoredRecordsOlderThanLiveScan(t *testing.T) {
now := time.Date(2026, 11, 20, 12, 0, 0, 0, time.UTC)
var prs []ghpkg.PullRequest
for i, days := range []int{3, 16, 29} {
created := now.Add(-time.Duration(days) * 24 * time.Hour)
prs = append(prs, ghpkg.PullRequest{
Repo: "o/r", Number: i + 1, State: "closed", HiveAttributed: true,
HiveModel: "claude-opus-5.5", HiveBackend: "claude",
CreatedAt: created, ClosedAt: created.Add(time.Hour), MergedAt: created.Add(time.Hour),
})
}
got := aggregateGovernorPRModels(prs, "30d", now)
total := 0
for _, b := range got.Buckets {
total += b.PRs
}
if total != 3 {
t.Fatalf("30d PRs = %d, want all 3 (3, 16 and 29 days old)", total)
}
cov := effectiveModelsCoverage(30*24*time.Hour, 31*24*time.Hour, nil, false, now)
if cov.ClosedPRsPartial {
t.Fatalf("coverage = %+v, want closed PRs complete when the store reaches back 31d", cov)
}
if cov := effectiveModelsCoverage(30*24*time.Hour, 14*24*time.Hour, nil, false, now); !cov.ClosedPRsPartial {
t.Fatalf("coverage = %+v, want partial at the 14d scan lookback", cov)
}
}
2 changes: 1 addition & 1 deletion src/pkg/dashboard/contribute_effective_models.go
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ func (s *Server) handleContributeEffectiveModels(w http.ResponseWriter, r *http.
runs, rotated, _ := readEffectiveModelRuns(s.effectiveModelTaskRunLogPath(), windowDur)
now := time.Now()
resp := aggregateContributeEffectiveModels(prs, runs, window, filter, minMergedPRs, now)
resp.Coverage = effectiveModelsCoverage(windowDur, ghpkg.AttributedClosedPRLookback(), runs, rotated, now)
resp.Coverage = effectiveModelsCoverage(windowDur, ghpkg.AttributedClosedPRCoverage(), runs, rotated, now)
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(resp)
}
Expand Down
202 changes: 202 additions & 0 deletions src/pkg/github/attributed_pr_store.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,202 @@
package github

import (
"encoding/json"
"errors"
"os"
"path/filepath"
"reflect"
"strconv"
"strings"
"sync"
"time"
)

// The attributed-PR store keeps the parsed attribution and outcome fields of
// closed hive-attributed PRs on disk, so the PRs-by-model and effective-models
// windows are not limited to what the bounded live scan still lists
// (hivecommons/hive#19365). It holds no PR bodies.
const (
AttributedPRStoreFile = "attributed-pr-store.json"
// attributedPRBackfillEnv is how far back the one-time backfill reaches;
// "off" (or 0) skips the backfill.
attributedPRBackfillEnv = "HIVE_ATTRIBUTED_PR_BACKFILL"
attributedPRDefaultBackfill = 30 * 24 * time.Hour
// attributedPRBackfillPagesPerTick bounds the extra closed-PR listing
// pages one repo pays per enumeration while its backfill is unfinished.
attributedPRBackfillPagesPerTick = 5
attributedPRStoreSchemaVersion = 1
)

var (
AttributedPRStorePath = filepath.Join(ReviewLinksDir, AttributedPRStoreFile)
attributedPRStoreMu sync.Mutex
)

type attributedPRStoreRepo struct {
// CoveredFrom is the updated-time horizon the store fully covers for this
// repo: every attributed PR updated since then is recorded.
CoveredFrom time.Time `json:"covered_from"`
// BackfillDone is true once a backfill reached the target horizon.
BackfillDone bool `json:"backfill_done,omitempty"`
// BackfillNextPage resumes an unfinished backfill (0 = not started).
BackfillNextPage int `json:"backfill_next_page,omitempty"`
Items map[string]PullRequest `json:"items"`
}

type attributedPRStoreData struct {
SchemaVersion int `json:"schema_version"`
GeneratedAt time.Time `json:"generated_at"`
Repos map[string]*attributedPRStoreRepo `json:"repos"`
}

func attributedPRBackfillWindow() time.Duration {
raw := strings.TrimSpace(os.Getenv(attributedPRBackfillEnv))
if raw == "" {
return attributedPRDefaultBackfill
}
if raw == "off" || raw == "0" {
return 0
}
if d, err := time.ParseDuration(raw); err == nil && d > 0 {
return d
}
if days, err := strconv.Atoi(strings.TrimSuffix(raw, "d")); err == nil && days > 0 {
return time.Duration(days) * 24 * time.Hour
}
return attributedPRDefaultBackfill
}

func loadAttributedPRStoreLocked(path string) *attributedPRStoreData {
empty := &attributedPRStoreData{SchemaVersion: attributedPRStoreSchemaVersion, Repos: map[string]*attributedPRStoreRepo{}}
data, err := os.ReadFile(path)
if err != nil {
return empty
}
var f attributedPRStoreData
if err := json.Unmarshal(data, &f); err != nil || f.SchemaVersion != attributedPRStoreSchemaVersion || f.Repos == nil {
return empty
}
return &f
}

func saveAttributedPRStoreLocked(path string, f *attributedPRStoreData) {
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
return
}
f.GeneratedAt = time.Now().UTC()
data, err := json.Marshal(f)
if err != nil {
return
}
tmp := path + ".tmp"
if err := os.WriteFile(tmp, data, 0o644); err != nil {
return
}
if err := os.Rename(tmp, path); err != nil && !errors.Is(err, os.ErrNotExist) {
_ = os.Remove(tmp)
}
}

// attributedPRBackfillState is a repo's backfill position.
type attributedPRBackfillState struct {
Done bool
NextPage int
}

func attributedPRStoreBackfillState(path, repo string) attributedPRBackfillState {
attributedPRStoreMu.Lock()
defer attributedPRStoreMu.Unlock()
r := loadAttributedPRStoreLocked(path).Repos[repo]
if r == nil {
return attributedPRBackfillState{}
}
return attributedPRBackfillState{Done: r.BackfillDone, NextPage: r.BackfillNextPage}
}

// recordAttributedClosedPRs upserts scanned closed PRs into the store.
// coveredFrom only ever moves the repo's horizon earlier. backfill, when
// non-nil, replaces the repo's backfill progress.
func recordAttributedClosedPRs(path, repo string, prs []PullRequest, coveredFrom time.Time, backfill *attributedPRBackfillState) {
attributedPRStoreMu.Lock()
defer attributedPRStoreMu.Unlock()
f := loadAttributedPRStoreLocked(path)
r := f.Repos[repo]
if r == nil {
r = &attributedPRStoreRepo{}
f.Repos[repo] = r
}
if r.Items == nil {
r.Items = map[string]PullRequest{}
}
changed := false
for _, pr := range prs {
if !pr.HiveAttributed {
continue
}
key := strconv.Itoa(pr.Number)
if old, ok := r.Items[key]; !ok || !reflect.DeepEqual(old, pr) {
r.Items[key] = pr
changed = true
}
}
if !coveredFrom.IsZero() && (r.CoveredFrom.IsZero() || coveredFrom.Before(r.CoveredFrom)) {
r.CoveredFrom = coveredFrom
changed = true
}
if backfill != nil && (r.BackfillDone != backfill.Done || r.BackfillNextPage != backfill.NextPage) {
r.BackfillDone = backfill.Done
r.BackfillNextPage = backfill.NextPage
changed = true
}
if changed {
saveAttributedPRStoreLocked(path, f)
}
}

// storedAttributedClosedPRs returns the stored closed records for repo.
func storedAttributedClosedPRs(path, repo string) []PullRequest {
attributedPRStoreMu.Lock()
defer attributedPRStoreMu.Unlock()
r := loadAttributedPRStoreLocked(path).Repos[repo]
if r == nil {
return nil
}
out := make([]PullRequest, 0, len(r.Items))
for _, pr := range r.Items {
out = append(out, pr)
}
return out
}

// AttributedClosedPRCoverage is how far back the closed attributed-PR records
// reach: the live scan's lookback, extended to the store's horizon once every
// repo in the store has covered it. Consumers compare a window to it.
func AttributedClosedPRCoverage() time.Duration {
span := attributedClosedPRLookback()
attributedPRStoreMu.Lock()
f := loadAttributedPRStoreLocked(AttributedPRStorePath)
attributedPRStoreMu.Unlock()
now := time.Now()
least := time.Duration(-1)
for _, r := range f.Repos {
if r.CoveredFrom.IsZero() {
return span
}
if d := now.Sub(r.CoveredFrom); least < 0 || d < least {
least = d
}
}
if least > span {
return least
}
return span
}

// attributedPRStorePath is where this client keeps its attributed-PR store.
func (c *Client) attributedPRStorePath() string {
if c != nil && c.attributedPRStoreFile != "" {
return c.attributedPRStoreFile
}
return AttributedPRStorePath
}
131 changes: 131 additions & 0 deletions src/pkg/github/attributed_pr_store_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,131 @@
package github

import (
"context"
"io"
"log/slog"
"net/http"
"net/http/httptest"
"strconv"
"testing"
"time"
)

func TestAttributedClosedPRStoreKeepsPRsThePastScanNoLongerLists(t *testing.T) {
t.Setenv(attributedClosedPRLookbackEnv, "14d")
now := time.Now().UTC()
scan := 0
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
scan++
if scan == 1 {
writeJSON(t, w, []map[string]any{
attributedClosedPR(1, now.Add(-2*24*time.Hour)),
attributedClosedPR(2, now.Add(-13*24*time.Hour)),
})
return
}
// Later scans no longer list #2: it aged out of the live window.
writeJSON(t, w, []map[string]any{attributedClosedPR(1, now.Add(-time.Hour)), attributedClosedPR(3, now.Add(-20*24*time.Hour))})
}))
defer srv.Close()

c := NewClientForTest(srv.URL, "o", []string{"r"}, slog.New(slog.NewTextHandler(io.Discard, nil)))
if _, err := c.fetchAttributedClosedPRs(context.Background(), "o", "r", "r"); err != nil {
t.Fatal(err)
}
prs, err := c.fetchAttributedClosedPRs(context.Background(), "o", "r", "r")
if err != nil {
t.Fatal(err)
}
got := map[int]bool{}
for _, pr := range prs {
got[pr.Number] = true
}
if len(prs) != 2 || !got[1] || !got[2] {
t.Fatalf("PRs = %+v, want #1 (fresh) and #2 (stored); #3 is past the lookback", prs)
}
// A new client over the same store file still has them: restarts keep history.
c2 := NewClientForTest(srv.URL, "o", []string{"r"}, slog.New(slog.NewTextHandler(io.Discard, nil)))
c2.attributedPRStoreFile = c.attributedPRStoreFile
if stored := storedAttributedClosedPRs(c2.attributedPRStorePath(), "r"); len(stored) != 2 {
t.Fatalf("stored after restart = %+v, want 2", stored)
}
}

func TestAttributedClosedPRBackfillIsPagedAndResumes(t *testing.T) {
t.Setenv(attributedClosedPRLookbackEnv, "14d")
t.Setenv(attributedPRBackfillEnv, "30d")
now := time.Now().UTC()
// 12 pages of one PR each, newest first: pages 1-2 inside 14d, the rest
// older, with page 11 the first past the 30d horizon.
age := func(page int) time.Duration {
switch {
case page <= 2:
return time.Duration(page) * 24 * time.Hour
case page <= 10:
return time.Duration(14+page) * 24 * time.Hour
default:
return 40 * 24 * time.Hour
}
}
var calls int
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
calls++
page, _ := strconv.Atoi(r.URL.Query().Get("page"))
if page == 0 {
page = 1
}
if page < 12 {
w.Header().Set("Link", `<`+srvURL(r)+`/repos/o/r/pulls?state=closed&page=`+strconv.Itoa(page+1)+`>; rel="next"`)
}
writeJSON(t, w, []map[string]any{attributedClosedPR(page, now.Add(-age(page)))})
}))
defer srv.Close()

c := NewClientForTest(srv.URL, "o", []string{"r"}, slog.New(slog.NewTextHandler(io.Discard, nil)))
prs, err := c.fetchAttributedClosedPRs(context.Background(), "o", "r", "r")
if err != nil {
t.Fatal(err)
}
// Head: pages 1-3 (page 3 is past 14d). Backfill budget: 5 pages from 3.
if calls != 3+attributedPRBackfillPagesPerTick {
t.Fatalf("first tick calls = %d, want head 3 + backfill budget %d", calls, attributedPRBackfillPagesPerTick)
}
if st := attributedPRStoreBackfillState(c.attributedPRStorePath(), "r"); st.Done || st.NextPage != 3+attributedPRBackfillPagesPerTick {
t.Fatalf("backfill state after tick 1 = %+v, want unfinished, resume at page %d", st, 3+attributedPRBackfillPagesPerTick)
}
if len(prs) != 2+attributedPRBackfillPagesPerTick {
t.Fatalf("PRs after tick 1 = %d, want %d", len(prs), 2+attributedPRBackfillPagesPerTick)
}

calls = 0
if _, err := c.fetchAttributedClosedPRs(context.Background(), "o", "r", "r"); err != nil {
t.Fatal(err)
}
st := attributedPRStoreBackfillState(c.attributedPRStorePath(), "r")
if !st.Done {
t.Fatalf("backfill state after tick 2 = %+v, want done", st)
}
stored := storedAttributedClosedPRs(c.attributedPRStorePath(), "r")
if len(stored) != 10 {
t.Fatalf("stored = %d PRs, want the 10 within 30d", len(stored))
}

// Steady state: no backfill calls any more.
calls = 0
if _, err := c.fetchAttributedClosedPRs(context.Background(), "o", "r", "r"); err != nil {
t.Fatal(err)
}
if calls != 3 {
t.Fatalf("steady-state calls = %d, want 3 (head scan only)", calls)
}
}

func TestMergeAttributedClosedPRsPrefersFresh(t *testing.T) {
fresh := []PullRequest{{Number: 1, Title: "new"}}
stored := []PullRequest{{Number: 1, Title: "old"}, {Number: 2}}
got := mergeAttributedClosedPRs(fresh, stored)
if len(got) != 2 || got[0].Title != "new" {
t.Fatalf("merged = %+v", got)
}
}
Loading
Loading