Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/added-merge-hotspot-advisor.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- Merge-hotspot advisor (`governor.hotspot_*`): the merge train's own PR file lists and rebase conflicts now feed a per-file score of how often concurrent PRs collide on the same file. Files above `hotspot_min_prs` / `hotspot_min_overlaps` in a `hotspot_window_days` window are reported on `GET /api/hotspots` (`hivectl governor hotspots`, `/api/status.hotspots`); a code file of at least `hotspot_min_lines` with two or more top-level symbol groups gets one `refactor: <path> is a merge hotspot — split into <N> units` issue (labels `agent/ci-maintainer`, `kind/cleanup`) through the writeback gate, deduped by `<!-- hive:hotspot file=<path> -->` and refiled no sooner than `hotspot_refile_days` after close. Docs are only ever recommended for a by-section split, generated files point at their source, and the ci-maintainer prompt gains a "Merge Hotspots" section describing the mechanical, no-behaviour-change split.
132 changes: 132 additions & 0 deletions dashboard/openapi.json
Original file line number Diff line number Diff line change
Expand Up @@ -34114,6 +34114,138 @@
}
}
}
},
"/api/hotspots": {
"get": {
"tags": [
"Governor"
],
"summary": "List merge hotspots: files repeatedly touched by concurrent PRs",
"description": "The merge-hotspot advisor's cached report, derived from the merge train's PR file lists and the rebase follow-through's conflict records (no extra GitHub polling). A file is listed when at least governor.hotspot_min_prs distinct PRs touched it in the last governor.hotspot_window_days and at least governor.hotspot_min_overlaps pairs of them were open concurrently. Code files at or above governor.hotspot_min_lines with two or more top-level symbol groups get a split-code recommendation and one modularize issue for the ci-maintainer lane (labels agent/ci-maintainer, kind/cleanup) through the writeback gate; docs are reported and only recommended for a by-section split above 1500 lines; generated files point at their source; lock/data files are report-only. Filter with ?repo=owner/name.",
"parameters": [
{
"name": "repo",
"in": "query",
"schema": {
"type": "string"
},
"description": "Restrict to one owner/name"
}
],
"responses": {
"200": {
"description": "Hotspots, highest score first",
"content": {
"application/json": {
"schema": {
"type": "object",
"properties": {
"repo": {
"type": "string"
},
"count": {
"type": "integer",
"description": "Files at or above the thresholds across all repos"
},
"actionable": {
"type": "integer",
"description": "Files with a split recommendation"
},
"filed": {
"type": "integer",
"description": "Files with a modularize issue filed"
},
"hotspots": {
"type": "array",
"items": {
"type": "object",
"properties": {
"repo": {
"type": "string"
},
"file": {
"type": "string"
},
"kind": {
"type": "string",
"enum": [
"code",
"doc",
"lock",
"generated",
"data"
]
},
"prs": {
"type": "integer",
"description": "Distinct PRs that touched the file in the window"
},
"pr_numbers": {
"type": "array",
"items": {
"type": "integer"
}
},
"overlaps": {
"type": "integer",
"description": "PR pairs open at the same time that both touched the file"
},
"conflicts": {
"type": "integer",
"description": "Rebase/merge conflicts attributed to the file"
},
"lines": {
"type": "integer",
"description": "Current line count (0 when unreadable)"
},
"score": {
"type": "number",
"description": "overlaps × size weight + 2 × conflicts"
},
"symbol_groups": {
"type": "array",
"items": {
"type": "string"
},
"description": "Top-level funcs/types (Go), declarations (JS/TS/py) or ## headings (md)"
},
"recommendation": {
"type": "string",
"enum": [
"none",
"split-code",
"split-doc",
"split-source",
"report-only"
]
},
"reason": {
"type": "string"
},
"generated_from": {
"type": "string",
"description": "Source tree for a generated file"
},
"issue_number": {
"type": "integer"
},
"issue_url": {
"type": "string"
},
"last_updated": {
"type": "string",
"format": "date-time"
}
}
}
}
}
}
}
}
}
}
}
}
},
"components": {
Expand Down
217 changes: 217 additions & 0 deletions src/cmd/hive/hotspot_advisor.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,217 @@
package main

import (
"context"
"fmt"
"log/slog"
"os"
"path/filepath"
"strings"
"sync"
"time"

"github.com/hivecommons/hive/pkg/config"
"github.com/hivecommons/hive/pkg/github"
"github.com/hivecommons/hive/pkg/hotspot"
"github.com/hivecommons/hive/pkg/writeback"
)

// Merge-hotspot advisor wiring (pkg/hotspot). The merge train feeds PR file
// lists and sibling conflicts into one durable store; once an hour the eval
// cycle scores the window, publishes the result for GET /api/hotspots and
// files at most one modularize issue per hotspot through the writeback gate.

// hotspotStatePath is the advisor's durable state on the PVC; a var so
// tests can redirect it.
var hotspotStatePath = "/data/hotspots.json"

// hotspotAdvisorInterval bounds how often the analysis (and any issue
// filing) runs; the intake runs on every merge-train sweep regardless.
const hotspotAdvisorInterval = time.Hour

// hotspotUnseenAfter is how long a PR may go unobserved by the merge train
// before its open interval is closed at its last sighting. The train only
// lists candidates on bases with two or more PRs, so a PR left alone on its
// base simply stops colliding — which is the right outcome here too.
const hotspotUnseenAfter = 2 * time.Hour

var (
hotspotStoreOnce sync.Once
hotspotStoreInst *hotspot.Store
hotspotMu sync.Mutex
hotspotLastRun time.Time
)

// hotspotStore returns the process-wide store, creating it on first use.
func hotspotStore() *hotspot.Store {
hotspotStoreOnce.Do(func() {
st, err := hotspot.NewStore(hotspotStatePath)
if err != nil {
slog.Warn("hotspot: state unreadable; starting empty", "path", hotspotStatePath, "error", err)
st, _ = hotspot.NewStore("")
}
hotspotStoreInst = st
})
return hotspotStoreInst
}

// hotspotRecorder adapts the store to automerge.HotspotRecorder.
type hotspotRecorder struct{ store *hotspot.Store }

func (r hotspotRecorder) RecordPRFiles(repo string, number int, files []string, openedAt time.Time) {
r.store.ObservePR(repo, number, files, openedAt)
}

func (r hotspotRecorder) RecordConflict(repo string, number int, files []string) {
r.store.ObserveConflict(repo, number, files)
}

// hotspotDashSource adapts the store to dashboard.HotspotSource.
type hotspotDashSource struct{ store *hotspot.Store }

func (d hotspotDashSource) HotspotsJSON(repo string) any { return d.store.Results(repo) }

func (d hotspotDashSource) HotspotCounts() (int, int, int) { return d.store.Counts() }

func (d hotspotDashSource) ObserveConflict(repo string, number int, files []string) {
d.store.ObserveConflict(repo, number, files)
}

// hotspotGitHub adapts the GitHub client to the Filer's narrow interfaces.
type hotspotGitHub struct{ c *github.Client }

func (g hotspotGitHub) IssueFileAllowed(repo, file string) bool {
item := writeback.Item{HiveAuthored: true}
return g.c.WritebackAllowed(repo, item, writeback.KindIssueFile, "merge-hotspot issue for "+file).Allowed
}

func (g hotspotGitHub) CreateHotspotIssue(ctx context.Context, repo, title, body string, labels []string) (int, string, error) {
res, err := g.c.CreateIssue(ctx, github.CreateIssueRequest{ // writeback:gated
Repo: repo,
Title: title,
Body: body,
Labels: labels,
Origin: config.OriginSourceGovernor,
})
if err != nil {
return 0, "", err
}
return res.Number, res.URL, nil
}

func (g hotspotGitHub) HotspotIssueState(ctx context.Context, repo string, number int) (bool, time.Time, error) {
sum, ok, err := g.c.IssueSummary(ctx, repo, number)
if err != nil {
return false, time.Time{}, err
}
if !ok {
return false, time.Time{}, nil
}
return strings.EqualFold(sum.State, "open"), time.Time{}, nil
}

// hotspotSourceReader reads a repo file from the host-local checkout when
// project.checkouts_dir names one, else from the GitHub contents API, with
// a per-pass cache so the same file is fetched once.
func hotspotSourceReader(ctx context.Context, cfg *config.Config, ghClient *github.Client) hotspot.SourceReader {
cache := map[string]hotspot.FileInfo{}
var mu sync.Mutex
return func(repo, path string) (hotspot.FileInfo, bool) {
key := repo + ":" + path
mu.Lock()
if fi, ok := cache[key]; ok {
mu.Unlock()
return fi, fi.Lines > 0
}
mu.Unlock()
var src []byte
if root := cfg.Project.CheckoutRootFor(repo); root != "" {
clean := filepath.Clean("/" + path)
if data, err := os.ReadFile(filepath.Join(root, clean)); err == nil {
src = data
}
}
if src == nil && ghClient != nil {
if gh := ghClient.GoGitHub(); gh != nil {
owner, name, ok := strings.Cut(repo, "/")
if ok {
if fc, _, _, err := gh.Repositories.GetContents(ctx, owner, name, path, nil); err == nil && fc != nil {
if body, err := fc.GetContent(); err == nil {
src = []byte(body)
}
}
}
}
}
fi := hotspot.FileInfo{Lines: hotspot.CountLines(src), Source: src}
mu.Lock()
cache[key] = fi
mu.Unlock()
return fi, fi.Lines > 0
}
}

// runHotspotAdvisor is the eval-tick half of the advisor: at most once per
// hotspotAdvisorInterval it closes stale PR intervals, prunes the window,
// scores every repo the merge train has reported on, publishes the result
// and — unless governor.hotspot_enabled is false — files modularize issues
// through the writeback gate. It never lists PRs or files from GitHub; the
// only reads are the hotspot files' current contents (for size and
// top-level symbols) and the state of issues it filed earlier.
func runHotspotAdvisor(ctx context.Context, cfg *config.Config, ghClient *github.Client, actionable *github.ActionableResult, logger *slog.Logger) {
if cfg == nil || ghClient == nil {
return
}
if !hotspotMu.TryLock() {
return
}
defer hotspotMu.Unlock()
now := time.Now()
if !hotspotLastRun.IsZero() && now.Sub(hotspotLastRun) < hotspotAdvisorInterval {
return
}
hotspotLastRun = now

store := hotspotStore()
if actionable != nil {
open := map[string][]int{}
for _, pr := range actionable.PRs.Items {
open[pr.Repo] = append(open[pr.Repo], pr.Number)
}
for repo, nums := range open {
store.TouchOpen(repo, nums)
}
}
store.ExpireUnseen(hotspotUnseenAfter)
g := cfg.Governor
window := time.Duration(g.HotspotWindowDaysOrDefault()) * 24 * time.Hour
refile := time.Duration(g.HotspotRefileDaysOrDefault()) * 24 * time.Hour
store.Prune(window, refile)
opts := hotspot.Options{
Window: window,
MinPRs: g.HotspotMinPRsOrDefault(),
MinOverlaps: g.HotspotMinOverlapsOrDefault(),
MinLines: g.HotspotMinLinesOrDefault(),
}
read := hotspotSourceReader(ctx, cfg, ghClient)
ghAdapter := hotspotGitHub{c: ghClient}
filer := &hotspot.Filer{
Store: store, Gate: ghAdapter, Creator: ghAdapter, Stater: ghAdapter,
Opts: opts, Refile: refile,
Logf: func(format string, args ...any) { logger.Info(fmt.Sprintf(format, args...)) },
}
for _, repo := range store.Repos() {
if ctx.Err() != nil {
return
}
prs, conflicts := store.Snapshot(repo)
hs := hotspot.Analyze(repo, prs, conflicts, read, opts, now)
if g.HotspotIssuesEnabled() {
filer.File(ctx, repo, hs)
}
store.SetResults(repo, hs)
if len(hs) > 0 {
logger.Info("hotspot advisor: scored", "repo", repo, "hotspots", len(hs), "top", hs[0].File, "top_score", hs[0].Score, "top_recommendation", hs[0].Recommendation)
}
}
}
13 changes: 13 additions & 0 deletions src/cmd/hive/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -2588,6 +2588,9 @@ func (b *boot) bootAgentsWith(deps bootAgentsDeps) {
}
return automerge.MergeTrainPolicy{Enabled: b.cfg.AutoMerge.MergeTrain.IsEnabled(), MaxRebaseAttempts: b.cfg.AutoMerge.MergeTrain.EffectiveMaxRebaseAttempts()}
}
// Merge-hotspot advisor intake (hotspot_advisor.go): the train's file
// lists and sibling conflicts are the advisor's only evidence.
autoMergeOpts.HotspotRecorder = hotspotRecorder{store: hotspotStore()}
b.requestRelays = newRequestRelaySupervisor(b.ctx, func(ctx context.Context, client *github.Client) <-chan struct{} {
laneGate := relayLanePolicyGate(autoMergeOpts)(client)
setMergeEligibleLaneGate(laneGate)
Expand Down Expand Up @@ -2781,6 +2784,7 @@ func (b *boot) bootDashboardWith(deps bootDashboardDeps) {
b.dashSrv = deps.newServer(b.cfg.Dashboard.Port, b.cfg.Dashboard.AuthToken, b.logger)
worksource.SetRunStageAccessor(b.dashSrv.RunStageAccessor())
b.dashSrv.SetWritebackGate(b.writebackGate())
b.dashSrv.SetHotspotSource(hotspotDashSource{store: hotspotStore()})
b.dashSrv.SetMutationStats(func() interface{} {
if b.mutationStats == nil {
return nil
Expand Down Expand Up @@ -7388,6 +7392,15 @@ func runEvalCycle(
// issues only, never PRs; the watermark and dedupe index live on the PVC.
runUpstreamWatch(ctx, cfg, ghClient, logger)

// Merge-hotspot advisor: at most hourly, score the files concurrent
// PRs keep colliding on (from the merge train's own records) and file
// one modularize issue per hotspot for the ci-maintainer lane through
// the writeback gate. Skipped with the other optional sweeps when the
// GitHub budget is tight.
if !budgetDecision.SkipOptional {
runHotspotAdvisor(ctx, cfg, ghClient, actionable, logger)
}

if !budgetDecision.SkipUnstick {
shaResult, shaErr := ghClient.EnforceSHAHold(ctx, github.SHAHoldConfig{
PrimaryRepo: cfg.Project.PrimaryRepo,
Expand Down
Loading
Loading