diff --git a/changelog.d/added-merge-hotspot-advisor.md b/changelog.d/added-merge-hotspot-advisor.md new file mode 100644 index 0000000000..d1aa773a61 --- /dev/null +++ b/changelog.d/added-merge-hotspot-advisor.md @@ -0,0 +1 @@ +- Merge-hotspot advisor (`governor.hotspot_*`): the merge train's own PR file lists and rebase conflicts now feed a per-file score of how often concurrent PRs collide on the same file. Files above `hotspot_min_prs` / `hotspot_min_overlaps` in a `hotspot_window_days` window are reported on `GET /api/hotspots` (`hivectl governor hotspots`, `/api/status.hotspots`); a code file of at least `hotspot_min_lines` with two or more top-level symbol groups gets one `refactor: is a merge hotspot — split into units` issue (labels `agent/ci-maintainer`, `kind/cleanup`) through the writeback gate, deduped by `` and refiled no sooner than `hotspot_refile_days` after close. Docs are only ever recommended for a by-section split, generated files point at their source, and the ci-maintainer prompt gains a "Merge Hotspots" section describing the mechanical, no-behaviour-change split. diff --git a/dashboard/openapi.json b/dashboard/openapi.json index 4b5ed341fa..75e9a4ee36 100644 --- a/dashboard/openapi.json +++ b/dashboard/openapi.json @@ -34114,6 +34114,138 @@ } } } + }, + "/api/hotspots": { + "get": { + "tags": [ + "Governor" + ], + "summary": "List merge hotspots: files repeatedly touched by concurrent PRs", + "description": "The merge-hotspot advisor's cached report, derived from the merge train's PR file lists and the rebase follow-through's conflict records (no extra GitHub polling). A file is listed when at least governor.hotspot_min_prs distinct PRs touched it in the last governor.hotspot_window_days and at least governor.hotspot_min_overlaps pairs of them were open concurrently. Code files at or above governor.hotspot_min_lines with two or more top-level symbol groups get a split-code recommendation and one modularize issue for the ci-maintainer lane (labels agent/ci-maintainer, kind/cleanup) through the writeback gate; docs are reported and only recommended for a by-section split above 1500 lines; generated files point at their source; lock/data files are report-only. Filter with ?repo=owner/name.", + "parameters": [ + { + "name": "repo", + "in": "query", + "schema": { + "type": "string" + }, + "description": "Restrict to one owner/name" + } + ], + "responses": { + "200": { + "description": "Hotspots, highest score first", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "repo": { + "type": "string" + }, + "count": { + "type": "integer", + "description": "Files at or above the thresholds across all repos" + }, + "actionable": { + "type": "integer", + "description": "Files with a split recommendation" + }, + "filed": { + "type": "integer", + "description": "Files with a modularize issue filed" + }, + "hotspots": { + "type": "array", + "items": { + "type": "object", + "properties": { + "repo": { + "type": "string" + }, + "file": { + "type": "string" + }, + "kind": { + "type": "string", + "enum": [ + "code", + "doc", + "lock", + "generated", + "data" + ] + }, + "prs": { + "type": "integer", + "description": "Distinct PRs that touched the file in the window" + }, + "pr_numbers": { + "type": "array", + "items": { + "type": "integer" + } + }, + "overlaps": { + "type": "integer", + "description": "PR pairs open at the same time that both touched the file" + }, + "conflicts": { + "type": "integer", + "description": "Rebase/merge conflicts attributed to the file" + }, + "lines": { + "type": "integer", + "description": "Current line count (0 when unreadable)" + }, + "score": { + "type": "number", + "description": "overlaps × size weight + 2 × conflicts" + }, + "symbol_groups": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Top-level funcs/types (Go), declarations (JS/TS/py) or ## headings (md)" + }, + "recommendation": { + "type": "string", + "enum": [ + "none", + "split-code", + "split-doc", + "split-source", + "report-only" + ] + }, + "reason": { + "type": "string" + }, + "generated_from": { + "type": "string", + "description": "Source tree for a generated file" + }, + "issue_number": { + "type": "integer" + }, + "issue_url": { + "type": "string" + }, + "last_updated": { + "type": "string", + "format": "date-time" + } + } + } + } + } + } + } + } + } + } + } } }, "components": { diff --git a/src/cmd/hive/hotspot_advisor.go b/src/cmd/hive/hotspot_advisor.go new file mode 100644 index 0000000000..95a007fa04 --- /dev/null +++ b/src/cmd/hive/hotspot_advisor.go @@ -0,0 +1,217 @@ +package main + +import ( + "context" + "fmt" + "log/slog" + "os" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/github" + "github.com/hivecommons/hive/pkg/hotspot" + "github.com/hivecommons/hive/pkg/writeback" +) + +// Merge-hotspot advisor wiring (pkg/hotspot). The merge train feeds PR file +// lists and sibling conflicts into one durable store; once an hour the eval +// cycle scores the window, publishes the result for GET /api/hotspots and +// files at most one modularize issue per hotspot through the writeback gate. + +// hotspotStatePath is the advisor's durable state on the PVC; a var so +// tests can redirect it. +var hotspotStatePath = "/data/hotspots.json" + +// hotspotAdvisorInterval bounds how often the analysis (and any issue +// filing) runs; the intake runs on every merge-train sweep regardless. +const hotspotAdvisorInterval = time.Hour + +// hotspotUnseenAfter is how long a PR may go unobserved by the merge train +// before its open interval is closed at its last sighting. The train only +// lists candidates on bases with two or more PRs, so a PR left alone on its +// base simply stops colliding — which is the right outcome here too. +const hotspotUnseenAfter = 2 * time.Hour + +var ( + hotspotStoreOnce sync.Once + hotspotStoreInst *hotspot.Store + hotspotMu sync.Mutex + hotspotLastRun time.Time +) + +// hotspotStore returns the process-wide store, creating it on first use. +func hotspotStore() *hotspot.Store { + hotspotStoreOnce.Do(func() { + st, err := hotspot.NewStore(hotspotStatePath) + if err != nil { + slog.Warn("hotspot: state unreadable; starting empty", "path", hotspotStatePath, "error", err) + st, _ = hotspot.NewStore("") + } + hotspotStoreInst = st + }) + return hotspotStoreInst +} + +// hotspotRecorder adapts the store to automerge.HotspotRecorder. +type hotspotRecorder struct{ store *hotspot.Store } + +func (r hotspotRecorder) RecordPRFiles(repo string, number int, files []string, openedAt time.Time) { + r.store.ObservePR(repo, number, files, openedAt) +} + +func (r hotspotRecorder) RecordConflict(repo string, number int, files []string) { + r.store.ObserveConflict(repo, number, files) +} + +// hotspotDashSource adapts the store to dashboard.HotspotSource. +type hotspotDashSource struct{ store *hotspot.Store } + +func (d hotspotDashSource) HotspotsJSON(repo string) any { return d.store.Results(repo) } + +func (d hotspotDashSource) HotspotCounts() (int, int, int) { return d.store.Counts() } + +func (d hotspotDashSource) ObserveConflict(repo string, number int, files []string) { + d.store.ObserveConflict(repo, number, files) +} + +// hotspotGitHub adapts the GitHub client to the Filer's narrow interfaces. +type hotspotGitHub struct{ c *github.Client } + +func (g hotspotGitHub) IssueFileAllowed(repo, file string) bool { + item := writeback.Item{HiveAuthored: true} + return g.c.WritebackAllowed(repo, item, writeback.KindIssueFile, "merge-hotspot issue for "+file).Allowed +} + +func (g hotspotGitHub) CreateHotspotIssue(ctx context.Context, repo, title, body string, labels []string) (int, string, error) { + res, err := g.c.CreateIssue(ctx, github.CreateIssueRequest{ // writeback:gated + Repo: repo, + Title: title, + Body: body, + Labels: labels, + Origin: config.OriginSourceGovernor, + }) + if err != nil { + return 0, "", err + } + return res.Number, res.URL, nil +} + +func (g hotspotGitHub) HotspotIssueState(ctx context.Context, repo string, number int) (bool, time.Time, error) { + sum, ok, err := g.c.IssueSummary(ctx, repo, number) + if err != nil { + return false, time.Time{}, err + } + if !ok { + return false, time.Time{}, nil + } + return strings.EqualFold(sum.State, "open"), time.Time{}, nil +} + +// hotspotSourceReader reads a repo file from the host-local checkout when +// project.checkouts_dir names one, else from the GitHub contents API, with +// a per-pass cache so the same file is fetched once. +func hotspotSourceReader(ctx context.Context, cfg *config.Config, ghClient *github.Client) hotspot.SourceReader { + cache := map[string]hotspot.FileInfo{} + var mu sync.Mutex + return func(repo, path string) (hotspot.FileInfo, bool) { + key := repo + ":" + path + mu.Lock() + if fi, ok := cache[key]; ok { + mu.Unlock() + return fi, fi.Lines > 0 + } + mu.Unlock() + var src []byte + if root := cfg.Project.CheckoutRootFor(repo); root != "" { + clean := filepath.Clean("/" + path) + if data, err := os.ReadFile(filepath.Join(root, clean)); err == nil { + src = data + } + } + if src == nil && ghClient != nil { + if gh := ghClient.GoGitHub(); gh != nil { + owner, name, ok := strings.Cut(repo, "/") + if ok { + if fc, _, _, err := gh.Repositories.GetContents(ctx, owner, name, path, nil); err == nil && fc != nil { + if body, err := fc.GetContent(); err == nil { + src = []byte(body) + } + } + } + } + } + fi := hotspot.FileInfo{Lines: hotspot.CountLines(src), Source: src} + mu.Lock() + cache[key] = fi + mu.Unlock() + return fi, fi.Lines > 0 + } +} + +// runHotspotAdvisor is the eval-tick half of the advisor: at most once per +// hotspotAdvisorInterval it closes stale PR intervals, prunes the window, +// scores every repo the merge train has reported on, publishes the result +// and — unless governor.hotspot_enabled is false — files modularize issues +// through the writeback gate. It never lists PRs or files from GitHub; the +// only reads are the hotspot files' current contents (for size and +// top-level symbols) and the state of issues it filed earlier. +func runHotspotAdvisor(ctx context.Context, cfg *config.Config, ghClient *github.Client, actionable *github.ActionableResult, logger *slog.Logger) { + if cfg == nil || ghClient == nil { + return + } + if !hotspotMu.TryLock() { + return + } + defer hotspotMu.Unlock() + now := time.Now() + if !hotspotLastRun.IsZero() && now.Sub(hotspotLastRun) < hotspotAdvisorInterval { + return + } + hotspotLastRun = now + + store := hotspotStore() + if actionable != nil { + open := map[string][]int{} + for _, pr := range actionable.PRs.Items { + open[pr.Repo] = append(open[pr.Repo], pr.Number) + } + for repo, nums := range open { + store.TouchOpen(repo, nums) + } + } + store.ExpireUnseen(hotspotUnseenAfter) + g := cfg.Governor + window := time.Duration(g.HotspotWindowDaysOrDefault()) * 24 * time.Hour + refile := time.Duration(g.HotspotRefileDaysOrDefault()) * 24 * time.Hour + store.Prune(window, refile) + opts := hotspot.Options{ + Window: window, + MinPRs: g.HotspotMinPRsOrDefault(), + MinOverlaps: g.HotspotMinOverlapsOrDefault(), + MinLines: g.HotspotMinLinesOrDefault(), + } + read := hotspotSourceReader(ctx, cfg, ghClient) + ghAdapter := hotspotGitHub{c: ghClient} + filer := &hotspot.Filer{ + Store: store, Gate: ghAdapter, Creator: ghAdapter, Stater: ghAdapter, + Opts: opts, Refile: refile, + Logf: func(format string, args ...any) { logger.Info(fmt.Sprintf(format, args...)) }, + } + for _, repo := range store.Repos() { + if ctx.Err() != nil { + return + } + prs, conflicts := store.Snapshot(repo) + hs := hotspot.Analyze(repo, prs, conflicts, read, opts, now) + if g.HotspotIssuesEnabled() { + filer.File(ctx, repo, hs) + } + store.SetResults(repo, hs) + if len(hs) > 0 { + logger.Info("hotspot advisor: scored", "repo", repo, "hotspots", len(hs), "top", hs[0].File, "top_score", hs[0].Score, "top_recommendation", hs[0].Recommendation) + } + } +} diff --git a/src/cmd/hive/main.go b/src/cmd/hive/main.go index d8fe12e5ee..87da7ae2bd 100644 --- a/src/cmd/hive/main.go +++ b/src/cmd/hive/main.go @@ -2588,6 +2588,9 @@ func (b *boot) bootAgentsWith(deps bootAgentsDeps) { } return automerge.MergeTrainPolicy{Enabled: b.cfg.AutoMerge.MergeTrain.IsEnabled(), MaxRebaseAttempts: b.cfg.AutoMerge.MergeTrain.EffectiveMaxRebaseAttempts()} } + // Merge-hotspot advisor intake (hotspot_advisor.go): the train's file + // lists and sibling conflicts are the advisor's only evidence. + autoMergeOpts.HotspotRecorder = hotspotRecorder{store: hotspotStore()} b.requestRelays = newRequestRelaySupervisor(b.ctx, func(ctx context.Context, client *github.Client) <-chan struct{} { laneGate := relayLanePolicyGate(autoMergeOpts)(client) setMergeEligibleLaneGate(laneGate) @@ -2781,6 +2784,7 @@ func (b *boot) bootDashboardWith(deps bootDashboardDeps) { b.dashSrv = deps.newServer(b.cfg.Dashboard.Port, b.cfg.Dashboard.AuthToken, b.logger) worksource.SetRunStageAccessor(b.dashSrv.RunStageAccessor()) b.dashSrv.SetWritebackGate(b.writebackGate()) + b.dashSrv.SetHotspotSource(hotspotDashSource{store: hotspotStore()}) b.dashSrv.SetMutationStats(func() interface{} { if b.mutationStats == nil { return nil @@ -7388,6 +7392,15 @@ func runEvalCycle( // issues only, never PRs; the watermark and dedupe index live on the PVC. runUpstreamWatch(ctx, cfg, ghClient, logger) + // Merge-hotspot advisor: at most hourly, score the files concurrent + // PRs keep colliding on (from the merge train's own records) and file + // one modularize issue per hotspot for the ci-maintainer lane through + // the writeback gate. Skipped with the other optional sweeps when the + // GitHub budget is tight. + if !budgetDecision.SkipOptional { + runHotspotAdvisor(ctx, cfg, ghClient, actionable, logger) + } + if !budgetDecision.SkipUnstick { shaResult, shaErr := ghClient.EnforceSHAHold(ctx, github.SHAHoldConfig{ PrimaryRepo: cfg.Project.PrimaryRepo, diff --git a/src/docs/api-reference/auth.md b/src/docs/api-reference/auth.md index 5b08495c23..6850135715 100644 --- a/src/docs/api-reference/auth.md +++ b/src/docs/api-reference/auth.md @@ -24,6 +24,6 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/openrouter/models` | Dashboard auth/session | Open Router Models | `pkg/dashboard/openrouter.go:47` | | `GET` | `/api/openrouter/credit` | Dashboard auth/session | Open Router Credit | `pkg/dashboard/openrouter.go:48` | | `GET` | `/openrouter/callback` | Public | Open Router Callback | `pkg/dashboard/openrouter.go:49` | -| `POST` | `/api/github-app/recheck` | Dashboard auth/session | GitHub App Recheck | `pkg/dashboard/server.go:1613` | -| `POST` | `/api/github-app/install-clicked` | Dashboard auth/session | GitHub App Install Clicked | `pkg/dashboard/server.go:1614` | -| `GET` | `/gh-setup` | Public | GitHub App Setup Callback | `pkg/dashboard/server.go:1615` | +| `POST` | `/api/github-app/recheck` | Dashboard auth/session | GitHub App Recheck | `pkg/dashboard/server.go:1616` | +| `POST` | `/api/github-app/install-clicked` | Dashboard auth/session | GitHub App Install Clicked | `pkg/dashboard/server.go:1617` | +| `GET` | `/gh-setup` | Public | GitHub App Setup Callback | `pkg/dashboard/server.go:1618` | diff --git a/src/docs/api-reference/dashboard-misc.md b/src/docs/api-reference/dashboard-misc.md index 4c10107398..de36e21f0c 100644 --- a/src/docs/api-reference/dashboard-misc.md +++ b/src/docs/api-reference/dashboard-misc.md @@ -31,6 +31,7 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/upstream-watch` | Owner only | Upstream-watch divergence: per watched fork the upstream followed, the watermark and last-run time, surfaced/ported/dismissed/skipped counts, and the recent upstream refs with their upstream URL, diff URL, fork issue number and state. Read-only — it reads the config and the watch state file, never GitHub. Also carries `interval`, `project_repos` and each repo's configured entry (`config`) for the editor. | `pkg/dashboard/api.go:297` | | `PUT` | `/api/config/upstream-watch` | Owner only | Upstream Watch Config Update: edit `upstream_watch` from the Features tab. Body `{enabled?, interval?, repos?: {: {upstream?, sources?, pr_labels?, max_issues_per_run?, label?} \| null}}` — absent keys are unchanged, `null` removes a repo, keys are stored as the bare repo name. Validated with the loader's rules before the config changes (400 on failure), then persisted and audited as `config_upstream_watch`. | `pkg/dashboard/api.go:298` | +| `GET` | `/api/hotspots` | Dashboard auth/session | Merge-hotspot advisor report (`?repo=owner/name` optional): files repeatedly touched by concurrent PRs over the last `governor.hotspot_window_days`, each with `prs` (distinct PRs), `pr_numbers`, `overlaps` (PR pairs open at the same time), `conflicts` (rebase conflicts blamed on the file), `lines`, `kind` (`code`/`doc`/`lock`/`generated`/`data`), `score`, `symbol_groups`, `recommendation` (`split-code`/`split-doc`/`split-source`/`report-only`/`none`), `reason`, `generated_from`, `issue_number`/`issue_url` of the modularize issue filed, `last_updated`; top-level `count`/`actionable`/`filed`. Derived from the merge train's own PR file lists and the rebase follow-through's conflict records — never lists PRs or files from GitHub. See [operator-reference.md § Merge-hotspot advisor](../operator-reference.md#merge-hotspot-advisor-governorhotspot_). | `pkg/dashboard/api.go:531` | | `GET` | `/api/widget` | Dashboard auth/session | Widget | `pkg/dashboard/api.go:113` | | `GET` | `/api/pane/{agent}` | Dashboard auth/session | Pane | `pkg/dashboard/api.go:118` | | `GET` | `/api/role` | Dashboard auth/session | Role | `pkg/dashboard/api.go:132` | @@ -62,4 +63,4 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `POST` | `/api/hives/{id}/heartbeat` | Dashboard auth/session | Hives Heartbeat | `pkg/dashboard/api_contribute.go:275` | | `DELETE` | `/api/hives/{id}` | Owner only | Hives Delete | `pkg/dashboard/api_contribute.go:276` | | `POST` | `/api/hives/onboard` | Dashboard auth/session | Hives Onboard | `pkg/dashboard/api_contribute.go:277` | -| `GET` | `/sso` | Public | SSO | `pkg/dashboard/server.go:1620` | +| `GET` | `/sso` | Public | SSO | `pkg/dashboard/server.go:1623` | diff --git a/src/docs/api-reference/health-status-events.md b/src/docs/api-reference/health-status-events.md index f9cb4a1fca..415c13cd6b 100644 --- a/src/docs/api-reference/health-status-events.md +++ b/src/docs/api-reference/health-status-events.md @@ -8,17 +8,17 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/version` | Dashboard auth/session | Build/version metadata; includes `upgradeMarker` (`target`, `current`, `attempts`, `maxAttempts`, `failed`, `requestedAt`, `lastError`) while a self-upgrade is in flight or has failed ([#6765](https://github.com/hivecommons/hive/issues/6765)), the hub-delivered `upgradePolicy` (`hub_managed`, `spoke_managed`, `schedule`, `schedule_hour`, `schedule_timezone`, `schedule_weekday`, `paused`, `branch`, `channel`, `target_sha`, `target_resolved`, `armed_target`, and `next_update_at` — the hub's RFC3339 ETA for the next promotion into the `stable` channel, omitted when unknown ([#10256](https://github.com/hivecommons/hive/issues/10256))) when available, and an `autoUpdate` object (`enabled`, `state` — one of `disabled`/`up_to_date`/`behind`/`retrying`/`failed`/`unknown` — `healthy`, `period`, `targetBranch`, `targetCommit`, `currentCommit`, `commitsBehind`, `lastAttemptAt`, `lastError`, `detail`, and `nextUpdateAt` — the hub's RFC3339 ETA for the next update, omitted when unknown, disabled or paused ([#10257](https://github.com/hivecommons/hive/issues/10257))) that never reports a failed or unknown update as healthy ([#6962](https://github.com/hivecommons/hive/issues/6962), [#6963](https://github.com/hivecommons/hive/issues/6963)) | `pkg/dashboard/api.go:43` | | `GET` | `/api/version/release-notes` | Dashboard auth/session | Changelog gap between two revisions: `?from=&to=` returns `{from:{sha,version?}, to:{sha,ref,version?}, sections:[{version,date,categories:{added,changed,fixed,security,deprecated}}], unreleased?, truncated, source, error?}` — the `CHANGELOG.md` release sections present at `to` and absent at `from`, newest first, plus (when `to` is not a release tag) the `changelog.d/` fragments new at `to` under an `Unreleased (in this build)` section. Fetch failures return `source: "unavailable"` with `error` rather than an error status; results are cached per (from, to) for an hour ([#11065](https://github.com/hivecommons/hive/issues/11065)) | `pkg/dashboard/api.go:44` | | `POST` | `/api/release-channel` | Owner only | Hosted spoke self-service release-channel selector; relays `stable`/`candidate`/`edge` to the hub's existing switch-branch endpoint with the spoke dashboard-token proof, and reports the requested channel as pending until the Deployment image lands | `pkg/dashboard/api.go:65` | -| `GET` | `/api/health` | Public | Basic health probe | `pkg/dashboard/server.go:1593` | -| `GET` | `/api/health/deep` | Public | Deep health probe | `pkg/dashboard/server.go:1594` | -| `GET` | `/api/livez` | Public | Kubernetes liveness probe | `pkg/dashboard/server.go:1595` | -| `GET` | `/metrics` | Registered only when `HIVE_METRICS_ENABLED`; requires `Authorization: ****** (403 if the token is unset) | Prometheus metrics | `pkg/dashboard/server.go:1601` | -| `GET` | `/api/status` | Dashboard auth/session | Dashboard aggregate status; `labelPresets[]` (`{key,label,description,labels[],states[]}`) lists the built-in label-filter presets for the Projects cards, led by `needs-human` whose `labels[]` are the effective hard-suppress labels plus `needs-signal`, the hold labels, `stage/071-needs-author` and the reporter-trust awaiting label (#12172); accepts `?fields=a,b` for top-level selection and `?omit=repos` to drop heavy sections, and honors `Accept-Encoding: gzip`; `agents[]` includes busy-pane visibility fields (`kicksUndeliverable`, `busySince`, `lastTranscriptActivity`, `condition`, `conditionMessage`) when a Working pane cannot accept kicks; a working agent also carries `agents[].run` — the live per-run line (#14576): `{since, elapsedS, costUSD, tokens, priced, tests, filesChanged}`, the estimated spend/tokens since the kick was delivered (`lastKickAt`, falling back to `busySince`) (delta of the token collector's per-agent cumulative figure; `priced:false` when the agent has none yet), the last visible test-runner result (`pass`/`fail`/empty) and the last `N files changed` git summary (`-1` when none) parsed from the captured pane; top-level `fleetRun` `{running, hourUSD, hourPriced}` is the governor strip's `running N · $X this hour` figure; `sla` (fleet) and `repos[].sla` (per repo) carry the Delivery SLA block derived from the lifecycle timeline with no extra GitHub calls: `windows.{7d,30d}.{open_to_triage,triage_to_start,start_to_pr,pr_to_merge,open_to_merge}` as `{p50_h,p95_h,n}` (hours, nearest-rank), `history[]` of up to 30 daily buckets `{day,start_to_pr_p50,pr_to_merge_p50,open_to_merge_p50,n_started,n_pr,n_merged}` persisted under `/data/sla-history.json`, `stale_in_progress[]` (`{number,repo,started_at,age_h,agent}` — issues carrying `hive/in-progress` longer than the agent claim TTL with no linked PR), `stale_count`, `merged_7d`, `claim_ttl_s` and, on the fleet block, the effective `targets` (`start_to_pr_h`, `pr_to_merge_h`, `open_to_merge_h`); `staleSince` and `repos[].countsStale` / `countsStaleSince` mark totals served from last-good figures — the previous enumeration (#13382) or the ForgeState cache's last-good counts (#13376); `ci_failure_signals` `{count, last, kicks, unmatched}` is the webhook CI-failure signal store — distinct red (repo, head, check) signals retained, the most recent one `{repo, number, head_sha, check, conclusion, details_url, agent, at}`, webhook-triggered kicks/nudges since boot, and failure deliveries whose head matched no open PR | `pkg/dashboard/server.go:1606` | -| `GET` | `/api/status/summary` | Dashboard auth/session | Compact agent/governor/budget status summary for pollers; honors `Accept-Encoding: gzip` | `pkg/dashboard/server.go:1607` | -| `GET` | `/api/overview/issues.csv` | Dashboard auth/session | Cached Overview issue-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1608` | -| `GET` | `/api/overview/issues.json` | Dashboard auth/session | Cached Overview issue-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1609` | -| `GET` | `/api/overview/prs.csv` | Dashboard auth/session | Cached Overview PR-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1610` | -| `GET` | `/api/overview/prs.json` | Dashboard auth/session | Cached Overview PR-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1611` | -| `GET` | `/api/events` | Dashboard auth/session | Server-sent event stream | `pkg/dashboard/server.go:1612` | +| `GET` | `/api/health` | Public | Basic health probe | `pkg/dashboard/server.go:1596` | +| `GET` | `/api/health/deep` | Public | Deep health probe | `pkg/dashboard/server.go:1597` | +| `GET` | `/api/livez` | Public | Kubernetes liveness probe | `pkg/dashboard/server.go:1598` | +| `GET` | `/metrics` | Registered only when `HIVE_METRICS_ENABLED`; requires `Authorization: ****** (403 if the token is unset) | Prometheus metrics | `pkg/dashboard/server.go:1604` | +| `GET` | `/api/status` | Dashboard auth/session | Dashboard aggregate status; `labelPresets[]` (`{key,label,description,labels[],states[]}`) lists the built-in label-filter presets for the Projects cards, led by `needs-human` whose `labels[]` are the effective hard-suppress labels plus `needs-signal`, the hold labels, `stage/071-needs-author` and the reporter-trust awaiting label (#12172); accepts `?fields=a,b` for top-level selection and `?omit=repos` to drop heavy sections, and honors `Accept-Encoding: gzip`; `agents[]` includes busy-pane visibility fields (`kicksUndeliverable`, `busySince`, `lastTranscriptActivity`, `condition`, `conditionMessage`) when a Working pane cannot accept kicks; a working agent also carries `agents[].run` — the live per-run line (#14576): `{since, elapsedS, costUSD, tokens, priced, tests, filesChanged}`, the estimated spend/tokens since the kick was delivered (`lastKickAt`, falling back to `busySince`) (delta of the token collector's per-agent cumulative figure; `priced:false` when the agent has none yet), the last visible test-runner result (`pass`/`fail`/empty) and the last `N files changed` git summary (`-1` when none) parsed from the captured pane; top-level `fleetRun` `{running, hourUSD, hourPriced}` is the governor strip's `running N · $X this hour` figure; `sla` (fleet) and `repos[].sla` (per repo) carry the Delivery SLA block derived from the lifecycle timeline with no extra GitHub calls: `windows.{7d,30d}.{open_to_triage,triage_to_start,start_to_pr,pr_to_merge,open_to_merge}` as `{p50_h,p95_h,n}` (hours, nearest-rank), `history[]` of up to 30 daily buckets `{day,start_to_pr_p50,pr_to_merge_p50,open_to_merge_p50,n_started,n_pr,n_merged}` persisted under `/data/sla-history.json`, `stale_in_progress[]` (`{number,repo,started_at,age_h,agent}` — issues carrying `hive/in-progress` longer than the agent claim TTL with no linked PR), `stale_count`, `merged_7d`, `claim_ttl_s` and, on the fleet block, the effective `targets` (`start_to_pr_h`, `pr_to_merge_h`, `open_to_merge_h`); `staleSince` and `repos[].countsStale` / `countsStaleSince` mark totals served from last-good figures — the previous enumeration (#13382) or the ForgeState cache's last-good counts (#13376); `ci_failure_signals` `{count, last, kicks, unmatched}` is the webhook CI-failure signal store — distinct red (repo, head, check) signals retained, the most recent one `{repo, number, head_sha, check, conclusion, details_url, agent, at}`, webhook-triggered kicks/nudges since boot, and failure deliveries whose head matched no open PR; `hotspots` `{count, actionable, filed}` is the merge-hotspot advisor's one-line summary (files at/above the thresholds, those with a split recommendation, those with a modularize issue filed), absent when the advisor is not wired — details on `GET /api/hotspots` | `pkg/dashboard/server.go:1609` | +| `GET` | `/api/status/summary` | Dashboard auth/session | Compact agent/governor/budget status summary for pollers; honors `Accept-Encoding: gzip` | `pkg/dashboard/server.go:1610` | +| `GET` | `/api/overview/issues.csv` | Dashboard auth/session | Cached Overview issue-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1611` | +| `GET` | `/api/overview/issues.json` | Dashboard auth/session | Cached Overview issue-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1612` | +| `GET` | `/api/overview/prs.csv` | Dashboard auth/session | Cached Overview PR-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1613` | +| `GET` | `/api/overview/prs.json` | Dashboard auth/session | Cached Overview PR-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1614` | +| `GET` | `/api/events` | Dashboard auth/session | Server-sent event stream | `pkg/dashboard/server.go:1615` | | `GET` | `/api/swarm` | Dashboard auth/session | Current swarm status: display name, duration, active repo, expiry, persisted prep metrics when available, and idle-unlock fields. | `pkg/dashboard/swarm.go:310` | | `POST` | `/api/swarm` | Owner only | Start a 24h repo swarm for a configured repo, returning 409 when another swarm is active and 423 when a previous swarm keeps the next swarm locked until enough agents are idle unless body `{"repo":"owner/name","force":true}` is used; saves the active record before repo prep, persists prep metrics, and announces the start to Discord when configured. | `pkg/dashboard/swarm.go:311` | | `DELETE` | `/api/swarm` | Owner only | End the active swarm, score issues closed and PRs merged in the window, append it to history, and announce the result to Discord when configured. | `pkg/dashboard/swarm.go:312` | diff --git a/src/docs/api-reference/packs-and-acmm.md b/src/docs/api-reference/packs-and-acmm.md index 6688d01849..88d86e4b62 100644 --- a/src/docs/api-reference/packs-and-acmm.md +++ b/src/docs/api-reference/packs-and-acmm.md @@ -22,7 +22,7 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `POST` | `/api/repos/{owner}/{repo}/items/{number}/decide-delegated/plan` | Same as `…/decide-delegated` | Preview for *Let hive decide*: runs the same owner-delegate policy with **no side effects** (nothing written, no job) and answers `{would, rule, acting, title, summary, why, consequences[], ladder, mergeable_as_is, alternatives[], writes: false, prior, notes}` — `summary` is one verb-first sentence with no "Hive would:" prefix (*Send #15332 back to its author with one instruction: make CI green, nothing else.*), `why` one short line (*CI is red.*), `consequences` at most two items, `ladder` the rung that decided with its `ruled_out` list, and `mergeable_as_is` whether an as-is merge is honest (the merge-now alternative is dropped when it is not). The dialog shows **Cancel** / **Let hive do that**, plus **Approve & merge as-is** only when `mergeable_as_is`; only then does the SPA post `…/decide-delegated` with `chosen` naming the button. See [dashboard-sections.md](../dashboard-sections.md#let-hive-decide). | `pkg/dashboard/api.go:341`, `pkg/dashboard/api_repo_item_decide_delegated_plan.go` | | `POST` | `/api/repos/{owner}/{repo}/items/{number}/blocker/recheck` | Same as `…/hold` (owner or repo write), signed-in GitHub user | *Re-check permissions* on an issue parked on a capability blocker (#15099): probes the GitHub App installation's permissions; when the missing permission is granted, clears every ledger entry blocked on it and un-parks the item (`{result: {outcome: "released", …}}`), otherwise `{result: {outcome: "no-effect", reason: "blocker", options: [...]}}`; 404 when the item has no blocker, 409 when the blocker is not a capability gap | `pkg/dashboard/api.go:342` | | `POST` | `/api/repos/{owner}/{repo}/items/{number}/blocker/close-partial` | Same as `…/hold` (owner or repo write), signed-in GitHub user | Option (c) on a capability blocker (#15099): labels the issue `hive/partially-addressed`, posts the operator's `{"comment"}` naming what remains and closes it; the ledger entry is cleared | `pkg/dashboard/api.go:343` | -| `POST` | `/api/repos/{owner}/{repo}/items/{number}/accept-triage` | Owner or repo write/maintain/admin | Accept triage on an issue from the activity monitor's row menu — what a maintainer accepting triage on GitHub does by hand: adds the reporter-trust triage label (`triage/accepted`, or the first of `issue_filter.reporter_trust.untrusted_require_labels`; the label is created in the repo if missing) and removes the awaiting label (`needs-triage` / `reporter_trust.awaiting_label`) and a Hive-written `stage/015-triage-needed`. Empty body. Returns `{ok, repo, number, acceptLabel, added, removed, alreadyAccepted, minStatusSeq}`; an issue already carrying the accept label with nothing left to remove is a no-op (`alreadyAccepted: true`, no write, no audit). 404 for a pull request or an unknown item. The status snapshot's labels are patched so the next reconcile reads 010 Actionable before the engine's own 015 → 010 derivation. Every write is audited as `repo_item_accept_triage` under the dashboard user. | `pkg/dashboard/api.go:535`, `pkg/dashboard/api_repo_item_accept_triage.go` | +| `POST` | `/api/repos/{owner}/{repo}/items/{number}/accept-triage` | Owner or repo write/maintain/admin | Accept triage on an issue from the activity monitor's row menu — what a maintainer accepting triage on GitHub does by hand: adds the reporter-trust triage label (`triage/accepted`, or the first of `issue_filter.reporter_trust.untrusted_require_labels`; the label is created in the repo if missing) and removes the awaiting label (`needs-triage` / `reporter_trust.awaiting_label`) and a Hive-written `stage/015-triage-needed`. Empty body. Returns `{ok, repo, number, acceptLabel, added, removed, alreadyAccepted, minStatusSeq}`; an issue already carrying the accept label with nothing left to remove is a no-op (`alreadyAccepted: true`, no write, no audit). 404 for a pull request or an unknown item. The status snapshot's labels are patched so the next reconcile reads 010 Actionable before the engine's own 015 → 010 derivation. Every write is audited as `repo_item_accept_triage` under the dashboard user. | `pkg/dashboard/api.go:538`, `pkg/dashboard/api_repo_item_accept_triage.go` | | `GET` | `/api/acmm/evaluation` | Dashboard auth/session | ACMMEvaluation — combined codebase + operational result, cached server-side for 1 hour (`acmmEvalTTL`). `?refresh=1` (the dashboard's "🔄 Re-evaluate" button, #5877) bypasses the hourly TTL but is debounced server-side: requests within 1 minute of the last evaluation (`acmmRefreshDebounce`) still serve the cache, since a full refresh costs up to ~29 GitHub GetContents calls per repo. The response's `last_evaluated_at` timestamp reports when the cached evaluation was computed | `pkg/dashboard/api.go:345`, `pkg/dashboard/api_acmm_eval.go` | | `POST` | `/api/acmm/issue` | Owner only | ACMMCreate Issue — files on GitHub or, with `governor.acmm.issue_tracker: work_source` / body `tracker: "work_source"` on a Linear-sourced hive, on Linear; response `tracker` says which. See [ACMM policy matrix](../acmm-policy-matrix.md#where-acmm-gap-issues-are-filed) | `pkg/dashboard/api.go:346` | | `GET` | `/api/acmm-recommendation` | Dashboard auth/session | Advisory level-up recommendation (`acmmadvisor.Recommendation`, JSON): never changes the applied level — see [ACMM advisor](../acmm-advisor.md) | `pkg/dashboard/api.go:347` | diff --git a/src/docs/api-reference/snapshots-and-style.md b/src/docs/api-reference/snapshots-and-style.md index 448ff2b9c2..2530d0bbcf 100644 --- a/src/docs/api-reference/snapshots-and-style.md +++ b/src/docs/api-reference/snapshots-and-style.md @@ -6,7 +6,7 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | Method | Path | Auth | Purpose | Source | |---|---|---|---|---| | `GET` | `/api/style` | Public | Sanitized custom dashboard CSS | `pkg/dashboard/api.go:45` | -| `GET` | `/branding/custom.css` | Dashboard auth/session | Operator branding stylesheet override, read per request (see [branding](../branding.md)) | `pkg/dashboard/server.go:1673` | +| `GET` | `/branding/custom.css` | Dashboard auth/session | Operator branding stylesheet override, read per request (see [branding](../branding.md)) | `pkg/dashboard/server.go:1676` | | `GET` | `/api/snapshot/frame-ancestors` | Public | Snapshot framing allowlist | `pkg/dashboard/api.go:78` | | `GET` | `/api/snapshot` | Public | Snapshot data | `pkg/dashboard/api.go:79` | | `GET` | `/snapshot` | Public | Public read-only snapshot page | `pkg/dashboard/api.go:80` | diff --git a/src/docs/hivectl.md b/src/docs/hivectl.md index d0e33cee39..d1cc5fa06f 100644 --- a/src/docs/hivectl.md +++ b/src/docs/hivectl.md @@ -263,6 +263,7 @@ hivectl governor budget-set --file budget.yaml hivectl governor repos-set --file repos.yaml hivectl governor agent-add ux-discovery --backend copilot --model claude-sonnet-4-6 hivectl governor agent-remove ux-discovery --yes +hivectl governor hotspots --repo hivecommons/hive # merge hotspots: files concurrent PRs keep touching (GET /api/hotspots) ``` ### claim / unclaim / claims — issue claims diff --git a/src/docs/operator-reference.md b/src/docs/operator-reference.md index 4bef77f96b..f72be621ab 100644 --- a/src/docs/operator-reference.md +++ b/src/docs/operator-reference.md @@ -1307,6 +1307,25 @@ governor: Lower it to resume sooner after a reset at the cost of a rebuffed run per probe; raise it to waste less while noticing later. +## Merge-hotspot advisor (`governor.hotspot_*`) + +The hive already predicts PR file overlap for the merge train. The advisor reuses that evidence — the train's per-PR file lists and the conflicts it hands to the rebase resolver, plus the rebase follow-through's conflict records — to find files that concurrent PRs keep colliding on, and files one modularize issue per hotspot for the ci-maintainer lane. It never lists PRs or files from GitHub itself; its only reads are the hotspot file's current content (size and top-level symbols) and the state of issues it filed earlier. + +- A file is a **hotspot** when, over the rolling window, at least `hotspot_min_prs` distinct PRs touched it and at least `hotspot_min_overlaps` pairs of them were open at the same time. Score = concurrent pairs × a size weight (1 / 1.25 / 1.5 / 2 at 800 / 1500 / 4000 lines) + 2 × attributed conflicts. Hunk-level "same region" weighting is not available from the train's records and is skipped. +- **Don't splinter.** A split is only recommended for code when the file has at least `hotspot_min_lines` lines **and** two or more top-level symbol groups (Go: funcs and types via `go/ast`, methods grouped under their receiver; JS/TS/Python: top-level declarations; Markdown: `##` headings). The proposal targets at most 4 files of at least 150 lines, no behaviour change, exported API unchanged, tests move with the code. +- **By kind.** Docs (`*.md`, `*.rst`, `*.txt`) are reported but only recommended for a by-section split above 1500 lines (the `docs/api-reference/.md` pattern). Generated files (`// Code generated … DO NOT EDIT`, `*.pb.go`, the dashboard SPA under `pkg/dashboard/static/` → `static-src/`) point at their source. Lock and data files are report-only. +- **One issue per file.** Title `refactor: is a merge hotspot — split into units`, labels `agent/ci-maintainer` + `kind/cleanup`, origin `governor`, body marker ``, evidence table, the top-level units, a concrete split proposal and the don't-splinter rule. While that issue is open no second one is filed; after it closes the file waits `hotspot_refile_days` before it can be filed again. Issues go through the writeback gate as `issue_file` (a paused repo or `writeback.per_repo[].mode: off` stops them) and are skipped when the GitHub budget governor is skipping optional sweeps. +- **Where to look.** `GET /api/hotspots` (or `hivectl governor hotspots [--repo owner/name]`) lists every hotspot with its recommendation and issue; `/api/status` carries `hotspots: {count, actionable, filed}`. State lives in `/data/hotspots.json`; the analysis runs at most hourly in the eval cycle, the intake on every merge-train sweep. A PR the train has not seen for two hours has its interval closed at its last sighting. + +| Key | Default | Meaning | +|---|---|---| +| `governor.hotspot_enabled` | `true` | Set `false` to keep the report and `/api/hotspots` but never file an issue. | +| `governor.hotspot_window_days` | `7` | Rolling window of merged and open PRs considered. | +| `governor.hotspot_min_prs` | `6` | Distinct PRs touching a file in the window before it is reported. | +| `governor.hotspot_min_overlaps` | `3` | Concurrent PR pairs on the file before it is reported. | +| `governor.hotspot_min_lines` | `800` | Smallest code file a split is recommended for. | +| `governor.hotspot_refile_days` | `30` | Wait after a modularize issue closes before the same file may be filed again. | + ## Delivery SLA The **Throughput** section of the dashboard carries a **Delivery SLA** block diff --git a/src/pkg/config/config.go b/src/pkg/config/config.go index ae3c2cacad..3ccbb73bb2 100644 --- a/src/pkg/config/config.go +++ b/src/pkg/config/config.go @@ -1696,6 +1696,20 @@ type GovernorConfig struct { // the stage/085 decision park. Default on (hivecommons/hive#14965). AutoCloseSuperseded *bool `yaml:"auto_close_superseded,omitempty" json:"auto_close_superseded,omitempty"` + // Hotspot* tune the merge-hotspot advisor, which scores files that + // concurrent PRs keep touching (from merge-train file lists and + // rebase-conflict records the hive already holds — no extra GitHub + // polling) and files one modularize issue per hotspot for the + // ci-maintainer lane through the writeback gate. Zero means the + // default; see Hotspot*() accessors. Set hotspot_enabled: false to + // keep GET /api/hotspots reporting but never file an issue. + HotspotEnabled *bool `yaml:"hotspot_enabled,omitempty" json:"hotspot_enabled,omitempty"` + HotspotWindowDays int `yaml:"hotspot_window_days,omitempty" json:"hotspot_window_days,omitempty"` + HotspotMinPRs int `yaml:"hotspot_min_prs,omitempty" json:"hotspot_min_prs,omitempty"` + HotspotMinOverlaps int `yaml:"hotspot_min_overlaps,omitempty" json:"hotspot_min_overlaps,omitempty"` + HotspotMinLines int `yaml:"hotspot_min_lines,omitempty" json:"hotspot_min_lines,omitempty"` + HotspotRefileDays int `yaml:"hotspot_refile_days,omitempty" json:"hotspot_refile_days,omitempty"` + // ProjectObservability configures what the telemetry and operations agents // recommend for the MANAGED PROJECT. It is deliberately separate from // Config.OTel/Tracing, which export Hive's own telemetry. @@ -2195,6 +2209,52 @@ func (g GovernorConfig) AutoCloseSupersededEnabled() bool { return g.AutoCloseSuperseded == nil || *g.AutoCloseSuperseded } +// Defaults for the merge-hotspot advisor (GovernorConfig.Hotspot*). +const ( + DefaultHotspotWindowDays = 7 + DefaultHotspotMinPRs = 6 + DefaultHotspotMinOverlaps = 3 + DefaultHotspotMinLines = 800 + DefaultHotspotRefileDays = 30 +) + +// HotspotIssuesEnabled resolves GovernorConfig.HotspotEnabled (default on). +func (g GovernorConfig) HotspotIssuesEnabled() bool { + return g.HotspotEnabled == nil || *g.HotspotEnabled +} + +func hotspotOrDefault(v, def int) int { + if v <= 0 { + return def + } + return v +} + +// HotspotWindowDaysOrDefault resolves governor.hotspot_window_days. +func (g GovernorConfig) HotspotWindowDaysOrDefault() int { + return hotspotOrDefault(g.HotspotWindowDays, DefaultHotspotWindowDays) +} + +// HotspotMinPRsOrDefault resolves governor.hotspot_min_prs. +func (g GovernorConfig) HotspotMinPRsOrDefault() int { + return hotspotOrDefault(g.HotspotMinPRs, DefaultHotspotMinPRs) +} + +// HotspotMinOverlapsOrDefault resolves governor.hotspot_min_overlaps. +func (g GovernorConfig) HotspotMinOverlapsOrDefault() int { + return hotspotOrDefault(g.HotspotMinOverlaps, DefaultHotspotMinOverlaps) +} + +// HotspotMinLinesOrDefault resolves governor.hotspot_min_lines. +func (g GovernorConfig) HotspotMinLinesOrDefault() int { + return hotspotOrDefault(g.HotspotMinLines, DefaultHotspotMinLines) +} + +// HotspotRefileDaysOrDefault resolves governor.hotspot_refile_days. +func (g GovernorConfig) HotspotRefileDaysOrDefault() int { + return hotspotOrDefault(g.HotspotRefileDays, DefaultHotspotRefileDays) +} + // Bounds for AdvisoryConfig.UpdateIntervalS. Exported because the dashboard // PUT validates against them and pkg/hub pins the invariant that its // advisory-staleness threshold exceeds the maximum allowed posting cadence diff --git a/src/pkg/dashboard/api.go b/src/pkg/dashboard/api.go index b2676e43ff..c09e7ad9b5 100644 --- a/src/pkg/dashboard/api.go +++ b/src/pkg/dashboard/api.go @@ -526,6 +526,9 @@ func (s *Server) RegisterAPI(deps *Dependencies) { // and the sentinel findings held internally, written or not. s.mux.HandleFunc("GET /api/writeback/suppressed", s.handleWritebackSuppressed) s.mux.HandleFunc("GET /api/sentinel", s.handleSentinel) + // Merge-hotspot advisor (api_hotspots.go): files repeatedly touched by + // concurrent PRs and the modularize issues filed for them. + s.mux.HandleFunc("GET /api/hotspots", s.handleHotspots) // Knowledge connector status + manual sync (#11069, api_knowledge_connectors.go). s.registerKnowledgeConnectorRoutes() s.mux.HandleFunc("POST /api/system-alerts/dismiss", s.handleSystemAlertDismiss) diff --git a/src/pkg/dashboard/api_hotspots.go b/src/pkg/dashboard/api_hotspots.go new file mode 100644 index 0000000000..c14801cc1c --- /dev/null +++ b/src/pkg/dashboard/api_hotspots.go @@ -0,0 +1,78 @@ +package dashboard + +import ( + "net/http" + "strings" +) + +// Merge-hotspot advisor surfaces (pkg/hotspot, wired by cmd/hive): the +// per-file report on GET /api/hotspots and the one-line count block on +// /api/status. The dashboard holds the store behind a small interface so +// it does not import pkg/hotspot (import ratchet). + +// HotspotSource is what cmd/hive installs: the cached analysis, its counts +// and the conflict intake the rebase follow-through feeds. +type HotspotSource interface { + // HotspotsJSON returns the cached hotspots (all repos when repo is "") + // as a JSON-encodable slice, highest score first. + HotspotsJSON(repo string) any + // HotspotCounts summarises the cache: files at or above the thresholds, + // those with a split recommendation, and those with an issue filed. + HotspotCounts() (total, actionable, filed int) + // ObserveConflict records a rebase/merge conflict blamed on files. + ObserveConflict(repo string, number int, files []string) +} + +// HotspotStatus is the /api/status `hotspots` block. +type HotspotStatus struct { + Count int `json:"count"` + Actionable int `json:"actionable"` + Filed int `json:"filed"` +} + +// SetHotspotSource installs the advisor's store; nil removes it. +func (s *Server) SetHotspotSource(src HotspotSource) { + if s == nil { + return + } + s.hotspotMu.Lock() + s.hotspotSource = src + s.hotspotMu.Unlock() +} + +func (s *Server) hotspotSourceRef() HotspotSource { + if s == nil { + return nil + } + s.hotspotMu.RLock() + defer s.hotspotMu.RUnlock() + return s.hotspotSource +} + +// HotspotStatusBlock is nil when the advisor is not wired. +func (s *Server) HotspotStatusBlock() *HotspotStatus { + src := s.hotspotSourceRef() + if src == nil { + return nil + } + total, actionable, filed := src.HotspotCounts() + return &HotspotStatus{Count: total, Actionable: actionable, Filed: filed} +} + +// handleHotspots serves GET /api/hotspots?repo=owner/name: files repeatedly +// touched by concurrent PRs in the rolling window, with per-file PR count, +// concurrent-pair overlaps, attributed conflicts, line count, the +// recommendation and the modularize issue if one was filed. +func (s *Server) handleHotspots(w http.ResponseWriter, r *http.Request) { + src := s.hotspotSourceRef() + repo := strings.TrimSpace(r.URL.Query().Get("repo")) + resp := map[string]any{"repo": repo, "hotspots": []any{}} + if src != nil { + if hs := src.HotspotsJSON(repo); hs != nil { + resp["hotspots"] = hs + } + total, actionable, filed := src.HotspotCounts() + resp["count"], resp["actionable"], resp["filed"] = total, actionable, filed + } + jsonResponse(w, resp) +} diff --git a/src/pkg/dashboard/api_hotspots_test.go b/src/pkg/dashboard/api_hotspots_test.go new file mode 100644 index 0000000000..5091d671d0 --- /dev/null +++ b/src/pkg/dashboard/api_hotspots_test.go @@ -0,0 +1,62 @@ +package dashboard + +import ( + "encoding/json" + "net/http" + "strings" + "testing" +) + +type fakeHotspotSource struct { + conflicts map[int][]string +} + +func (f *fakeHotspotSource) HotspotsJSON(repo string) any { + if repo != "" && repo != "o/r" { + return []any{} + } + return []map[string]any{{"repo": "o/r", "file": "pkg/dashboard/api.go", "prs": 8, "overlaps": 11, "recommendation": "split-code", "issue_number": 42}} +} + +func (f *fakeHotspotSource) HotspotCounts() (int, int, int) { return 3, 1, 1 } + +func (f *fakeHotspotSource) ObserveConflict(_ string, number int, files []string) { + if f.conflicts == nil { + f.conflicts = map[int][]string{} + } + f.conflicts[number] = files +} + +func TestHotspotsEndpointAndStatusBlock(t *testing.T) { + s := covApiServer(t) + // Unwired: an empty list, never a 500, and no status block. + rec := doGet(s, "/api/hotspots") + if rec.Code != http.StatusOK || !strings.Contains(rec.Body.String(), `"hotspots":[]`) { + t.Fatalf("unwired = %d %s", rec.Code, rec.Body.String()) + } + if s.HotspotStatusBlock() != nil { + t.Fatal("status block must be nil when the advisor is not wired") + } + + s.SetHotspotSource(&fakeHotspotSource{}) + rec = doGet(s, "/api/hotspots?repo=o/r") + var resp struct { + Repo string `json:"repo"` + Count int `json:"count"` + Actionable int `json:"actionable"` + Filed int `json:"filed"` + Hotspots []map[string]any `json:"hotspots"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil || rec.Code != http.StatusOK { + t.Fatalf("hotspots = %d %s (%v)", rec.Code, rec.Body.String(), err) + } + if resp.Repo != "o/r" || resp.Count != 3 || resp.Actionable != 1 || resp.Filed != 1 || len(resp.Hotspots) != 1 || resp.Hotspots[0]["file"] != "pkg/dashboard/api.go" { + t.Fatalf("payload = %+v", resp) + } + if rec := doGet(s, "/api/hotspots?repo=x/y"); !strings.Contains(rec.Body.String(), `"hotspots":[]`) { + t.Fatalf("other repo = %s", rec.Body.String()) + } + if b := s.HotspotStatusBlock(); b == nil || b.Count != 3 || b.Actionable != 1 || b.Filed != 1 { + t.Fatalf("status block = %+v", b) + } +} diff --git a/src/pkg/dashboard/rebase_merge_followthrough.go b/src/pkg/dashboard/rebase_merge_followthrough.go index 9048d32c0c..f9b3f2d4f3 100644 --- a/src/pkg/dashboard/rebase_merge_followthrough.go +++ b/src/pkg/dashboard/rebase_merge_followthrough.go @@ -244,7 +244,13 @@ func (s *Server) driveRebaseMerge(ctx context.Context, store *rebasemerge.Store, } } } + wasFailed := f.State == rebasemerge.StateFailed action := rebasemerge.Advance(&f, obs, rebasemerge.Deadlines{}) + if !wasFailed && f.State == rebasemerge.StateFailed && f.FailReason == rebasemerge.FailConflict && len(f.ConflictFiles) > 0 { + if src := s.hotspotSourceRef(); src != nil { + src.ObserveConflict(f.Repo, f.Number, f.ConflictFiles) + } + } switch action { case rebasemerge.ActionMerge: head := nonEmptyString(f.RebasedHead, f.ApprovedHead) diff --git a/src/pkg/dashboard/server.go b/src/pkg/dashboard/server.go index 6cd8516923..d2ec4403d3 100644 --- a/src/pkg/dashboard/server.go +++ b/src/pkg/dashboard/server.go @@ -503,6 +503,9 @@ type Server struct { writebackMu sync.RWMutex writebackGate *writeback.Gate sentinelFindings sentinelFindings + // hotspotSource is the merge-hotspot advisor's store (api_hotspots.go). + hotspotMu sync.RWMutex + hotspotSource HotspotSource // forgeAppInventoryFn supplies the Forge App tab's key inventory from // cmd/hive (resolved active key path + per-app-id PVC keys, fingerprints @@ -3670,6 +3673,7 @@ func (s *Server) handleStatus(w http.ResponseWriter, r *http.Request) { } payload.UpgradeProgress = s.upgradeProgress(status, now) payload.Writeback = s.WritebackStatus() + payload.Hotspots = s.HotspotStatusBlock() statusJSONResponse(w, r, filterStatusPayload(&payload, r.URL.Query().Get("fields"), r.URL.Query().Get("omit"))) } @@ -3692,6 +3696,10 @@ type statusWithWebhooks struct { // writes in the last 24h, per-repo first_seen / writable / mode, and the // banners for repos the hive can now write to but left untouched. Writeback *writeback.Status `json:"writeback,omitempty"` + // Hotspots is the merge-hotspot advisor's one-line summary: files hit by + // concurrent PRs in the window, how many warrant a split, how many have + // a modularize issue filed. Absent when the advisor is not wired. + Hotspots *HotspotStatus `json:"hotspots,omitempty"` } func (s *Server) handleStatusSummary(w http.ResponseWriter, r *http.Request) { diff --git a/src/pkg/dashboard/settings_parity_allowlist.txt b/src/pkg/dashboard/settings_parity_allowlist.txt index e137214e25..c5d535e3dc 100644 --- a/src/pkg/dashboard/settings_parity_allowlist.txt +++ b/src/pkg/dashboard/settings_parity_allowlist.txt @@ -513,3 +513,9 @@ write_surface.enforce[] # TODO #11854 write_surface.neutralize_mentions[] # TODO #11854 writeback.kinds..own_only # policy-file knob; the Settings matrix shows it read-only as the kind's scope writeback.kinds..unlabeled_min_level # policy-file knob; the Settings matrix shows it read-only in the kind's scope +governor.hotspot_enabled # merge-hotspot advisor; yaml-only +governor.hotspot_window_days # merge-hotspot advisor; yaml-only +governor.hotspot_min_prs # merge-hotspot advisor; yaml-only +governor.hotspot_min_overlaps # merge-hotspot advisor; yaml-only +governor.hotspot_min_lines # merge-hotspot advisor; yaml-only +governor.hotspot_refile_days # merge-hotspot advisor; yaml-only diff --git a/src/pkg/dashboard/static-src/js/governor-view.js b/src/pkg/dashboard/static-src/js/governor-view.js index 6b0c53bb5b..7cc22b9f65 100644 --- a/src/pkg/dashboard/static-src/js/governor-view.js +++ b/src/pkg/dashboard/static-src/js/governor-view.js @@ -266,6 +266,15 @@
MTTR
${formatFixTime(itmMedian)}${itmSpark}
${fleetRunStatHtml()}
on hold
${holdTotal}${holdSpark}
+ ${(() => { + // Merge hotspots tile (GET /api/hotspots): files concurrent PRs + // keep colliding on; present only when the advisor is wired. + const hs = data && data.hotspots; + if (!hs || typeof hs.count !== 'number') return ''; + const tip = `MERGE HOTSPOTS — files concurrent PRs keep colliding on. WHAT: ${hs.count} file(s) above the hotspot thresholds in the window; ${hs.actionable || 0} warrant a split; ${hs.filed || 0} already have a modularize issue filed for the ci-maintainer. HOW: GET /api/hotspots (hivectl governor hotspots) lists each file with its PRs, concurrent pairs, conflicts, size, recommendation and issue.`; + const label = hs.filed > 0 ? `merge hotspots · ${hs.filed} filed` : 'merge hotspots'; + return `
${label}
${hs.count}
`; + })()} ${(() => { // Planning tile: active plan-trees, with plans-awaiting-review highlighted // amber (human-action-required, like on-hold). Reads /api/status planning diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index 3eae2157b3..06b454a896 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -23414,6 +23414,15 @@

All labels

MTTR
${formatFixTime(itmMedian)}${itmSpark}
${fleetRunStatHtml()}
on hold
${holdTotal}${holdSpark}
+ ${(() => { + // Merge hotspots tile (GET /api/hotspots): files concurrent PRs + // keep colliding on; present only when the advisor is wired. + const hs = data && data.hotspots; + if (!hs || typeof hs.count !== 'number') return ''; + const tip = `MERGE HOTSPOTS — files concurrent PRs keep colliding on. WHAT: ${hs.count} file(s) above the hotspot thresholds in the window; ${hs.actionable || 0} warrant a split; ${hs.filed || 0} already have a modularize issue filed for the ci-maintainer. HOW: GET /api/hotspots (hivectl governor hotspots) lists each file with its PRs, concurrent pairs, conflicts, size, recommendation and issue.`; + const label = hs.filed > 0 ? `merge hotspots · ${hs.filed} filed` : 'merge hotspots'; + return `
${label}
${hs.count}
`; + })()} ${(() => { // Planning tile: active plan-trees, with plans-awaiting-review highlighted // amber (human-action-required, like on-hold). Reads /api/status planning diff --git a/src/pkg/github/automerge/automerge_merge_train.go b/src/pkg/github/automerge/automerge_merge_train.go index dbc776af04..359bb75aef 100644 --- a/src/pkg/github/automerge/automerge_merge_train.go +++ b/src/pkg/github/automerge/automerge_merge_train.go @@ -76,6 +76,16 @@ type RebaseResolver interface { RequestRebase(ctx context.Context, req RebaseRequest) error } +// HotspotRecorder is the merge-hotspot advisor's intake (pkg/hotspot.Store +// satisfies it through cmd/hive). RecordPRFiles is called once per train +// candidate per sweep with the files the planner already fetched; +// RecordConflict once per sibling rebase the train requests, with the +// files that sibling shares with the front. +type HotspotRecorder interface { + RecordPRFiles(repo string, number int, files []string, openedAt time.Time) + RecordConflict(repo string, number int, files []string) +} + // RebaseResolverFunc adapts a function to RebaseResolver. type RebaseResolverFunc func(ctx context.Context, req RebaseRequest) error @@ -362,6 +372,9 @@ func (c *Engine) planMergeTrains(ctx context.Context, lane, displayRepo, owner, c.warn("merge train: could not list PR files; left out of the train", "repo", displayRepo, "pr", pr.GetNumber(), "error", err) continue } + if c.hotspotRecorder != nil { + c.hotspotRecorder.RecordPRFiles(owner+"/"+repo, pr.GetNumber(), files, pr.GetCreatedAt().Time) + } members = append(members, mergeTrainMember{ Number: pr.GetNumber(), Base: base, @@ -652,6 +665,9 @@ func (c *Engine) mergeTrainRequestSiblingRebase(ctx context.Context, plan *merge return } attempt := c.bumpMergeTrainAttempts(plan.owner, plan.repo, n, front) + if c.hotspotRecorder != nil { + c.hotspotRecorder.RecordConflict(plan.owner+"/"+plan.repo, n, mergeTrainSharedFiles(g, n)) + } req := RebaseRequest{Repo: plan.owner + "/" + plan.repo, Number: n, Base: g.Base, Head: live.GetHead().GetRef(), HeadSHA: live.GetHead().GetSHA(), Front: front, Attempt: attempt, MaxAttempts: policy.MaxRebaseAttempts, Lane: plan.lane} if err := c.rebaseResolver.RequestRebase(ctx, req); err != nil { c.warn("merge train: rebase request failed on sibling", "repo", plan.displayRepo, "pr", n, "front", front, "attempt", attempt, "error", err) @@ -660,6 +676,29 @@ func (c *Engine) mergeTrainRequestSiblingRebase(ctx context.Context, plan *merge c.info("merge train: rebase requested for sibling after the front merged", "repo", plan.displayRepo, "pr", n, "front", front, "attempt", attempt, "max_attempts", policy.MaxRebaseAttempts) } +// mergeTrainSharedFiles returns the overlap files the member itself +// touches — the paths a conflict on it can be blamed on. +func mergeTrainSharedFiles(g *mergeTrainGroup, number int) []string { + if g == nil { + return nil + } + m, ok := g.members[number] + if !ok { + return g.OverlapFiles + } + mine := make(map[string]bool, len(m.Files)) + for _, f := range m.Files { + mine[f] = true + } + var out []string + for _, f := range g.OverlapFiles { + if mine[f] { + out = append(out, f) + } + } + return out +} + // mergeTrainSkip is the per-cycle sibling skip: recorded on the stage // tracker (so the row reads "In merge train · 3rd of 4") and logged at // debug only — the train's INFO line is the transition, not the wait. diff --git a/src/pkg/github/automerge/automerge_merge_train_hotspot_test.go b/src/pkg/github/automerge/automerge_merge_train_hotspot_test.go new file mode 100644 index 0000000000..b8b40fc83c --- /dev/null +++ b/src/pkg/github/automerge/automerge_merge_train_hotspot_test.go @@ -0,0 +1,77 @@ +package automerge + +import ( + "context" + "sort" + "sync" + "testing" + "time" + + gh "github.com/google/go-github/v72/github" +) + +type recordedHotspot struct { + mu sync.Mutex + files map[int][]string + opened map[int]time.Time + conflicts map[int][]string +} + +func (r *recordedHotspot) RecordPRFiles(_ string, number int, files []string, openedAt time.Time) { + r.mu.Lock() + defer r.mu.Unlock() + if r.files == nil { + r.files, r.opened = map[int][]string{}, map[int]time.Time{} + } + r.files[number] = append([]string(nil), files...) + r.opened[number] = openedAt +} + +func (r *recordedHotspot) RecordConflict(_ string, number int, files []string) { + r.mu.Lock() + defer r.mu.Unlock() + if r.conflicts == nil { + r.conflicts = map[int][]string{} + } + r.conflicts[number] = append([]string(nil), files...) +} + +// The merge train hands its PR file lists and the conflicts it repairs to +// the hotspot recorder, so the advisor never lists files itself. +func TestMergeTrainFeedsHotspotRecorder(t *testing.T) { + t0 := time.Date(2026, 10, 20, 9, 0, 0, 0, time.UTC) + api := &mergeTrainAPI{ + files: map[int][]string{20: {"a.go", "b.go"}, 21: {"a.go", "c.go"}, 22: {"b.go", "d.go"}}, + state: map[int]string{20: "clean", 21: "clean", 22: "clean"}, + } + srv := newMergeTrainServer(t, api) + defer srv.Close() + resolver := RebaseResolverFunc(func(context.Context, RebaseRequest) error { return nil }) + c, _ := newMergeTrainEngine(t, srv.URL, DefaultMergeTrainPolicy, resolver) + rec := &recordedHotspot{} + c.hotspotRecorder = rec + prs := []*gh.PullRequest{mtPR(20, t0, ""), mtPR(21, t0.Add(time.Minute), ""), mtPR(22, t0.Add(2*time.Minute), "")} + plan := c.planMergeTrains(context.Background(), "self", "acme/widget", "acme", "widget", prs, func(*gh.PullRequest) string { return "" }) + if !plan.isFront(20) { + t.Fatalf("front = %v", plan.groups) + } + rec.mu.Lock() + if len(rec.files) != 3 || len(rec.files[21]) != 2 || !rec.opened[21].Equal(t0.Add(time.Minute)) { + t.Fatalf("recorded files = %v opened = %v", rec.files, rec.opened) + } + rec.mu.Unlock() + + // Front merged, #22 is dirty: the conflict is blamed on the overlap + // files #22 itself touches (b.go), not on #21's a.go. + api.mu.Lock() + api.state[21], api.state[22] = "behind", "dirty" + api.mu.Unlock() + c.mergeTrainAfterFrontMerge(context.Background(), plan, 20) + rec.mu.Lock() + defer rec.mu.Unlock() + got := rec.conflicts[22] + sort.Strings(got) + if len(rec.conflicts) != 1 || len(got) != 1 || got[0] != "b.go" { + t.Fatalf("conflicts = %v, want 22:[b.go]", rec.conflicts) + } +} diff --git a/src/pkg/github/automerge/automerge_sweep.go b/src/pkg/github/automerge/automerge_sweep.go index ccf3b813bd..caeb603966 100644 --- a/src/pkg/github/automerge/automerge_sweep.go +++ b/src/pkg/github/automerge/automerge_sweep.go @@ -195,6 +195,11 @@ type Options struct { // RebaseResolver receives the train's rebase-resolve requests for a // conflicting PR; nil leaves a dirty front on its conflicting path. RebaseResolver RebaseResolver + // HotspotRecorder receives the merge train's PR file lists and the + // conflicts it asks the resolver to repair, so the merge-hotspot + // advisor (pkg/hotspot) derives its evidence without polling GitHub + // again; nil records nothing. + HotspotRecorder HotspotRecorder } // Engine owns the automerge sweep policy state. @@ -317,6 +322,7 @@ type Engine struct { mergeTrainPolicy func() MergeTrainPolicy rebaseResolver RebaseResolver + hotspotRecorder HotspotRecorder mergeTrainState } @@ -369,6 +375,7 @@ func New(transport Transport, opts Options) *Engine { sweepTrigger: opts.SweepTrigger, mergeTrainPolicy: mergeTrainPolicyOption(opts), rebaseResolver: opts.RebaseResolver, + hotspotRecorder: opts.HotspotRecorder, evaluatedHeads: make(map[string]string), requiredChecksFallbackWarned: make(map[string]bool), } diff --git a/src/pkg/hivectl/commands/governor.go b/src/pkg/hivectl/commands/governor.go index 60278c0a10..b84e8e0539 100644 --- a/src/pkg/hivectl/commands/governor.go +++ b/src/pkg/hivectl/commands/governor.go @@ -27,9 +27,40 @@ func newGovernorCommand(env *commandEnv) *cobra.Command { governor.AddCommand(governorFileCommand(env, "repos-set", "/api/config/governor/repos", "Update governed repositories")) governor.AddCommand(governorAgentAddCommand(env)) governor.AddCommand(governorAgentRemoveCommand(env)) + governor.AddCommand(governorHotspotsCommand(env)) return governor } +// governorHotspotsCommand lists the merge-hotspot advisor's report: files +// repeatedly touched by concurrent PRs, with the split recommendation and +// the modularize issue filed for each (GET /api/hotspots). +func governorHotspotsCommand(env *commandEnv) *cobra.Command { + var repo string + command := &cobra.Command{ + Use: "hotspots", + Short: "List files repeatedly touched by concurrent PRs (merge hotspots)", + Args: argsNone(), + Example: " hivectl governor hotspots\n hivectl governor hotspots --repo hivecommons/hive --output yaml", + RunE: func(cmd *cobra.Command, _ []string) error { + client, err := env.client() + if err != nil { + return err + } + var query url.Values + if repo != "" { + query = url.Values{"repo": {repo}} + } + result, err := client.Do(cmd.Context(), http.MethodGet, "/api/hotspots", query, nil) + if err != nil { + return err + } + return env.print(cmd, result) + }, + } + command.Flags().StringVar(&repo, "repo", "", "restrict to one owner/name") + return command +} + func governorFileCommand(env *commandEnv, name, path, short string) *cobra.Command { var file string var stdin bool diff --git a/src/pkg/hivectl/commands/governor_hotspots_test.go b/src/pkg/hivectl/commands/governor_hotspots_test.go new file mode 100644 index 0000000000..9a25926035 --- /dev/null +++ b/src/pkg/hivectl/commands/governor_hotspots_test.go @@ -0,0 +1,28 @@ +package commands + +import ( + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +func TestGovernorHotspotsCommand(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/api/hotspots" || r.URL.Query().Get("repo") != "acme/widget" { + t.Fatalf("request = %s", r.URL.String()) + } + w.Header().Set("Content-Type", "application/json") + io.WriteString(w, `{"repo":"acme/widget","count":1,"hotspots":[{"file":"pkg/a.go","prs":7}]}`) + })) + defer server.Close() + + stdout, _, err := execute(t, server, "", "--output", "json", "governor", "hotspots", "--repo", "acme/widget") + if err != nil { + t.Fatal(err) + } + if !strings.Contains(stdout, `"file": "pkg/a.go"`) { + t.Fatalf("stdout = %q", stdout) + } +} diff --git a/src/pkg/hotspot/coverage_test.go b/src/pkg/hotspot/coverage_test.go new file mode 100644 index 0000000000..3b6b4cd341 --- /dev/null +++ b/src/pkg/hotspot/coverage_test.go @@ -0,0 +1,141 @@ +package hotspot + +import ( + "path/filepath" + "strings" + "testing" + "time" +) + +func TestClassifyGeneratedHeaderPointsAtSource(t *testing.T) { + src := []byte("// Code generated by gen/main.go from static-src; DO NOT EDIT.\npackage p\n") + kind, from := Classify("pkg/x/out.go", src) + if kind != KindGenerated || from != "gen/main" { + t.Fatalf("got %s/%q", kind, from) + } + if kind, _ := Classify("pkg/x/y.go", []byte("package p\n")); kind != KindCode { + t.Fatalf("plain Go file classified %s", kind) + } + if kind, _ := Classify("config.yaml", nil); kind != KindData { + t.Fatalf("yaml classified %s", kind) + } +} + +func TestCountLinesAndGroupsEdgeCases(t *testing.T) { + if n := CountLines([]byte("a\nb")); n != 2 { + t.Errorf("no trailing newline: %d", n) + } + if n := CountLines([]byte("a\nb\n")); n != 2 { + t.Errorf("trailing newline: %d", n) + } + goSrc := []byte("package p\n\ntype G[T any] struct{}\n\nfunc (g G[T]) A() {}\nfunc (g *G[T]) B() {}\nfunc F() {}\nvar x = 1\n") + g := Groups("pkg/p/g.go", goSrc) + if strings.Join(g, ",") != "F,G" { + t.Errorf("generic receiver groups=%v", g) + } + if g := Groups("pkg/p/bad.go", []byte("package p\nfunc {")); g != nil { + t.Errorf("unparsable Go should give nil, got %v", g) + } + if g := Groups("a.py", []byte("def a():\n pass\nclass B:\n pass\nasync def c(): pass\n")); len(g) != 3 { + t.Errorf("py groups=%v", g) + } + if g := Groups("a.rs", []byte("fn main() {}\n")); g != nil { + t.Errorf("unknown language should give nil, got %v", g) + } + if g := Groups("a.go", nil); g != nil { + t.Errorf("empty source should give nil, got %v", g) + } +} + +func TestUnitsAndTitle(t *testing.T) { + cases := []struct { + lines, groups, want int + }{ + {800, 10, 4}, // capped at MaxUnits + {400, 10, 2}, // floor of 2 + {800, 3, 3}, // never more units than groups + {800, 0, 4}, // no symbol info: size only + } + for _, c := range cases { + h := Hotspot{Lines: c.lines, SymbolGroups: make([]string, c.groups)} + if got := Units(h, Options{}); got != c.want { + t.Errorf("Units(lines=%d groups=%d)=%d want %d", c.lines, c.groups, got, c.want) + } + } + h := Hotspot{File: "pkg/dashboard/stage_tracker.go"} + if got := Title(h, 3); !strings.Contains(got, "stage_tracker.go") || !strings.Contains(got, "3 units") { + t.Errorf("Title=%q", got) + } + if got := Title(Hotspot{File: "docs/x.md", Recommendation: RecommendSplitDoc}, 3); !strings.Contains(got, "3 sections") { + t.Errorf("doc Title=%q", got) + } + h.Kind, h.GeneratedFrom, h.Recommendation = KindGenerated, "pkg/dashboard/static-src/", RecommendSplitSource + if got := Title(h, 3); !strings.Contains(got, "static-src") { + t.Errorf("generated Title should name the source: %q", got) + } +} + +func TestProposeBucketsShapes(t *testing.T) { + many := make([]string, 0, 12) + for i := 0; i < 12; i++ { + many = append(many, "Group"+string(rune('A'+i))) + } + code := proposeBuckets(Hotspot{File: "pkg/p/big_test.go", Kind: KindCode, SymbolGroups: many}, 2) + if len(code) != 2 || code[0].file != "pkg/p/big_test.go" || !strings.HasPrefix(code[1].file, "pkg/p/big_") { + t.Fatalf("code buckets=%+v", code) + } + if !strings.Contains(code[0].desc, "more") { + t.Errorf("bucket with >4 groups should elide: %q", code[0].desc) + } + sparse := proposeBuckets(Hotspot{File: "pkg/p/a.go", Kind: KindCode, SymbolGroups: []string{"Only"}}, 3) + if sparse[2].desc != "remaining helpers" { + t.Errorf("empty bucket desc=%q", sparse[2].desc) + } + doc := proposeBuckets(Hotspot{File: "docs/ops.md", Kind: KindDoc, SymbolGroups: []string{"Alpha / Beta", "Gamma", "Delta", "Epsilon", "Zeta"}}, 2) + if len(doc) != 3 || doc[0].file != "docs/ops/README.md" || doc[1].file != "docs/ops/alpha_beta.md" { + t.Fatalf("doc buckets=%+v", doc) + } + if !strings.Contains(doc[2].desc, "remaining sections") { + t.Errorf("doc tail should count the rest: %q", doc[2].desc) + } + if s := slug(" Ünïcode--Name That Is Far Too Long To Keep "); s != "n_code_name_that_is_far" && len(s) > 24 { + t.Errorf("slug=%q", s) + } + if partName(5, []string{"a"}, 6) != "part6" { + t.Error("partName fallback") + } +} + +func TestStoreClosePRReposAndResults(t *testing.T) { + s, err := NewStore(filepath.Join(t.TempDir(), "h.json")) + if err != nil { + t.Fatal(err) + } + clock := day(0) + s.SetClock(func() time.Time { return clock }) + s.ObservePR("o/r", 1, []string{"a.go"}, time.Time{}) + s.ObservePR("o/z", 7, []string{"z.go"}, time.Time{}) + s.ClosePR("o/r", 1, false, time.Time{}) + s.ClosePR("o/r", 1, true, day(3)) // already closed: no-op + s.ClosePR("o/r", 99, true, day(3)) // unknown: no-op + prs, _ := s.Snapshot("o/r") + if len(prs) != 1 || !prs[0].ClosedAt.Equal(clock) || prs[0].Merged { + t.Fatalf("ClosePR: %+v", prs) + } + if got := strings.Join(s.Repos(), ","); got != "o/r,o/z" { + t.Errorf("Repos=%q", got) + } + s.RecordIssue(IssueRecord{Repo: "o/r", File: "a.go", Number: 5, URL: "u", FiledAt: clock}) + s.SetResults("o/r", []Hotspot{{Repo: "o/r", File: "a.go", Score: 1}, {Repo: "o/r", File: "b.go", Score: 9}}) + s.SetResults("o/z", []Hotspot{{Repo: "o/z", File: "z.go", Score: 9}}) + all := s.Results("") + if len(all) != 3 || all[0].Score != 9 || all[1].Score != 9 || all[0].Repo != "o/r" || all[2].File != "a.go" { + t.Fatalf("Results order=%+v", all) + } + if all[2].IssueNumber != 5 || all[2].IssueURL != "u" { + t.Errorf("issue not joined onto result: %+v", all[2]) + } + if one := s.Results("o/z"); len(one) != 1 || one[0].File != "z.go" { + t.Errorf("Results(o/z)=%+v", one) + } +} diff --git a/src/pkg/hotspot/golden_test.go b/src/pkg/hotspot/golden_test.go new file mode 100644 index 0000000000..bd7e3b4275 --- /dev/null +++ b/src/pkg/hotspot/golden_test.go @@ -0,0 +1,28 @@ +package hotspot + +import ( + "os" + "path/filepath" + "testing" +) + +// compareGolden compares got with the file at path; HIVE_UPDATE_GOLDEN=1 +// rewrites it. +func compareGolden(t *testing.T, path, got string) { + t.Helper() + if os.Getenv("HIVE_UPDATE_GOLDEN") != "" { + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(got), 0o644); err != nil { + t.Fatal(err) + } + } + want, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read golden (set HIVE_UPDATE_GOLDEN=1 to create): %v", err) + } + if string(want) != got { + t.Errorf("golden mismatch for %s\n--- want\n%s\n--- got\n%s", path, want, got) + } +} diff --git a/src/pkg/hotspot/hotspot.go b/src/pkg/hotspot/hotspot.go new file mode 100644 index 0000000000..c3914a531b --- /dev/null +++ b/src/pkg/hotspot/hotspot.go @@ -0,0 +1,363 @@ +// Package hotspot finds files that concurrent pull requests keep touching +// and turns the worst of them into one modularize issue each for the +// ci-maintainer lane. +// +// It adds no GitHub polling of its own: the merge-train planner already +// fetches every candidate PR's file list to predict overlaps, and the +// rebase follow-through already records conflicting files. Both feed the +// Store here; Analyze reduces the window to per-file evidence and a +// recommendation; the Filer writes at most one open issue per file through +// the writeback gate. +package hotspot + +import ( + "sort" + "strings" + "time" +) + +// PRFiles is one pull request's file list as last observed. +type PRFiles struct { + Repo string `json:"repo"` + Number int `json:"number"` + Files []string `json:"files"` + OpenedAt time.Time `json:"opened_at"` + // ClosedAt is zero while the PR is open. Merged and closed-unmerged + // PRs both end the open interval; only Merged ones count as "merged". + ClosedAt time.Time `json:"closed_at,omitempty"` + Merged bool `json:"merged,omitempty"` + // SeenAt is when the file list was last refreshed. + SeenAt time.Time `json:"seen_at"` +} + +// Conflict records one merge-conflict or rebase-failure event and the files +// GitHub (or the rebase follow-through) blamed for it. +type Conflict struct { + Repo string `json:"repo"` + Number int `json:"number"` + Files []string `json:"files"` + At time.Time `json:"at"` +} + +// Kind classifies a path for the recommendation rules. +type Kind string + +const ( + KindCode Kind = "code" + KindDoc Kind = "doc" + KindLock Kind = "lock" + KindGenerated Kind = "generated" + KindData Kind = "data" +) + +// Recommendation is what the advisor suggests for a hotspot. +type Recommendation string + +const ( + // RecommendNone: below thresholds or guarded; reported only. + RecommendNone Recommendation = "none" + // RecommendSplitCode: file an issue to split the file along the symbol + // groups that collided. + RecommendSplitCode Recommendation = "split-code" + // RecommendSplitDoc: a large document whose sections collide; suggest + // splitting by section/area (like docs/api-reference/.md). + RecommendSplitDoc Recommendation = "split-doc" + // RecommendSplitSource: a generated file; the recommendation points at + // its source files instead. + RecommendSplitSource Recommendation = "split-source" + // RecommendReportOnly: lock/data files, or docs below the size bar. + RecommendReportOnly Recommendation = "report-only" +) + +// Hotspot is the per-file result. +type Hotspot struct { + Repo string `json:"repo"` + File string `json:"file"` + Kind Kind `json:"kind"` + PRs int `json:"prs"` + PRNumbers []int `json:"pr_numbers"` + // Overlaps counts PR pairs that were open at the same time and both + // touched File. + Overlaps int `json:"overlaps"` + // Conflicts counts conflict events attributable to File. + Conflicts int `json:"conflicts"` + // Lines is the file's line count, or 0 when the source was unreadable. + Lines int `json:"lines"` + Score float64 `json:"score"` + // SymbolGroups are the distinct top-level units of the file (funcs and + // types for Go, declarations for JS/TS/py, `##` headings for md). + SymbolGroups []string `json:"symbol_groups,omitempty"` + Recommendation Recommendation `json:"recommendation"` + // Reason explains why Recommendation is what it is, in one sentence. + Reason string `json:"reason"` + // GeneratedFrom names the source tree for generated files. + GeneratedFrom string `json:"generated_from,omitempty"` + // IssueNumber / IssueURL point at the open or most recent modularize + // issue for this file, if any. + IssueNumber int `json:"issue_number,omitempty"` + IssueURL string `json:"issue_url,omitempty"` + LastUpdated time.Time `json:"last_updated"` +} + +// Options are the thresholds; zero values take the documented defaults. +type Options struct { + Window time.Duration + MinPRs int + MinOverlaps int + MinLines int + // MinGroups is the don't-splinter guard: a split is only recommended + // when at least this many distinct symbol groups exist. Default 2. + MinGroups int + // MaxUnits caps the proposed split. Default 4. + MaxUnits int + // MinUnitLines is the smallest unit the proposal may create. Default 150. + MinUnitLines int +} + +// Defaults mirror governor.hotspot_* in pkg/config. +const ( + DefaultWindow = 7 * 24 * time.Hour + DefaultMinPRs = 6 + DefaultMinOverlaps = 3 + DefaultMinLines = 800 + DefaultMinGroups = 2 + DefaultMaxUnits = 4 + DefaultMinUnitLines = 150 + // DefaultDocMinLines is the size bar above which a document is + // recommended for a by-section split rather than merely reported. + DefaultDocMinLines = 1500 +) + +func (o Options) withDefaults() Options { + if o.Window <= 0 { + o.Window = DefaultWindow + } + if o.MinPRs <= 0 { + o.MinPRs = DefaultMinPRs + } + if o.MinOverlaps <= 0 { + o.MinOverlaps = DefaultMinOverlaps + } + if o.MinLines <= 0 { + o.MinLines = DefaultMinLines + } + if o.MinGroups <= 0 { + o.MinGroups = DefaultMinGroups + } + if o.MaxUnits <= 0 { + o.MaxUnits = DefaultMaxUnits + } + if o.MinUnitLines <= 0 { + o.MinUnitLines = DefaultMinUnitLines + } + return o +} + +// FileInfo is what Analyze needs to know about a file's current content. +// Source may be nil when the file could not be read; Lines is then 0 and +// no split is recommended (the guard needs a size). +type FileInfo struct { + Lines int + Source []byte +} + +// SourceReader resolves a repo-relative path to its current content. It may +// return (FileInfo{}, false) when the file is unavailable. +type SourceReader func(repo, path string) (FileInfo, bool) + +// Analyze reduces the PR and conflict records of one repo to hotspots at or +// above the thresholds. Files below the thresholds are omitted. The result +// is sorted by Score descending, then by path. +// +// Hunk-level attribution (which lines each PR changed) is not retained by +// the merge train, so the weighting by "same region" the owner suggested is +// skipped; symbol groups come from the file's current top-level +// declarations instead — see Groups. +func Analyze(repo string, prs []PRFiles, conflicts []Conflict, read SourceReader, opts Options, now time.Time) []Hotspot { + opts = opts.withDefaults() + since := now.Add(-opts.Window) + + type agg struct { + prs []PRFiles + conflicts int + } + byFile := map[string]*agg{} + for _, pr := range prs { + if pr.Repo != repo || !inWindow(pr, since) { + continue + } + seen := map[string]bool{} + for _, f := range pr.Files { + if f == "" || seen[f] { + continue + } + seen[f] = true + a := byFile[f] + if a == nil { + a = &agg{} + byFile[f] = a + } + a.prs = append(a.prs, pr) + } + } + for _, c := range conflicts { + if c.Repo != repo || c.At.Before(since) { + continue + } + for _, f := range c.Files { + if a := byFile[f]; a != nil { + a.conflicts++ + } + } + } + + var out []Hotspot + for file, a := range byFile { + if len(a.prs) < opts.MinPRs { + continue + } + overlaps := pairwiseOverlaps(a.prs) + if overlaps < opts.MinOverlaps { + continue + } + h := Hotspot{ + Repo: repo, + File: file, + PRs: len(a.prs), + Overlaps: overlaps, + Conflicts: a.conflicts, + LastUpdated: now, + } + for _, pr := range a.prs { + h.PRNumbers = append(h.PRNumbers, pr.Number) + } + sort.Ints(h.PRNumbers) + + var info FileInfo + if read != nil { + info, _ = read(repo, file) + } + h.Lines = info.Lines + h.Kind, h.GeneratedFrom = Classify(file, info.Source) + if h.Kind == KindCode || h.Kind == KindDoc { + h.SymbolGroups = Groups(file, info.Source) + } + h.Score = score(h) + h.Recommendation, h.Reason = recommend(h, opts) + out = append(out, h) + } + sort.Slice(out, func(i, j int) bool { + if out[i].Score != out[j].Score { + return out[i].Score > out[j].Score + } + return out[i].File < out[j].File + }) + return out +} + +// inWindow reports whether the PR was open at any point since `since`. +func inWindow(pr PRFiles, since time.Time) bool { + return pr.ClosedAt.IsZero() || !pr.ClosedAt.Before(since) +} + +// pairwiseOverlaps counts PR pairs whose open intervals intersect. +func pairwiseOverlaps(prs []PRFiles) int { + n := 0 + for i := 0; i < len(prs); i++ { + for j := i + 1; j < len(prs); j++ { + if intervalsIntersect(prs[i], prs[j]) { + n++ + } + } + } + return n +} + +func intervalsIntersect(a, b PRFiles) bool { + aEnd, bEnd := a.ClosedAt, b.ClosedAt + // Open PRs extend to +inf. + if aEnd.IsZero() { + aEnd = time.Unix(1<<40, 0) + } + if bEnd.IsZero() { + bEnd = time.Unix(1<<40, 0) + } + return !a.OpenedAt.After(bEnd) && !b.OpenedAt.After(aEnd) +} + +// score weights overlaps by file size (bigger files are costlier to +// collide on and to rebase) and counts a conflict as two overlaps. +func score(h Hotspot) float64 { + sizeWeight := 1.0 + switch { + case h.Lines >= 4000: + sizeWeight = 2.0 + case h.Lines >= 1500: + sizeWeight = 1.5 + case h.Lines >= 800: + sizeWeight = 1.25 + } + return float64(h.Overlaps)*sizeWeight + 2*float64(h.Conflicts) +} + +func recommend(h Hotspot, opts Options) (Recommendation, string) { + switch h.Kind { + case KindGenerated: + if h.GeneratedFrom != "" { + return RecommendSplitSource, "generated file; the collisions are in " + h.GeneratedFrom + " — look there, never edit the output by hand" + } + return RecommendReportOnly, "generated file with no known source; reported only" + case KindLock, KindData: + return RecommendReportOnly, "lock/data file; collisions resolve by regenerating, not by splitting" + case KindDoc: + if h.Lines < DefaultDocMinLines { + return RecommendReportOnly, "document below the size bar; reported only" + } + if len(h.SymbolGroups) < opts.MinGroups { + return RecommendReportOnly, "document has fewer than two top-level sections; nothing to split along" + } + return RecommendSplitDoc, "large document whose sections collide; split by section/area like docs/api-reference/.md" + } + if h.Lines == 0 { + return RecommendNone, "source unreadable; cannot size the file" + } + if h.Lines < opts.MinLines { + return RecommendNone, "below governor.hotspot_min_lines; splitting would splinter the code" + } + if len(h.SymbolGroups) < opts.MinGroups { + return RecommendNone, "fewer than two top-level symbol groups; the file is already one unit" + } + return RecommendSplitCode, "large file with several top-level groups hit by concurrent PRs" +} + +// Units proposes how many files the hotspot should become, honouring the +// don't-splinter rule: at most opts.MaxUnits, each at least +// opts.MinUnitLines lines, and never more units than symbol groups. +func Units(h Hotspot, opts Options) int { + opts = opts.withDefaults() + n := h.Lines / opts.MinUnitLines + if n > opts.MaxUnits { + n = opts.MaxUnits + } + if g := len(h.SymbolGroups); g > 0 && n > g { + n = g + } + if n < 2 { + n = 2 + } + return n +} + +// Actionable reports whether a hotspot warrants an issue. +func (h Hotspot) Actionable() bool { + switch h.Recommendation { + case RecommendSplitCode, RecommendSplitDoc, RecommendSplitSource: + return true + } + return false +} + +// Marker is the HTML comment that dedupes issues per file. +func Marker(path string) string { + return "" +} diff --git a/src/pkg/hotspot/hotspot_test.go b/src/pkg/hotspot/hotspot_test.go new file mode 100644 index 0000000000..a1a0abf225 --- /dev/null +++ b/src/pkg/hotspot/hotspot_test.go @@ -0,0 +1,442 @@ +package hotspot + +import ( + "context" + "errors" + "path/filepath" + "strings" + "testing" + "time" +) + +var t0 = time.Date(2026, 10, 20, 12, 0, 0, 0, time.UTC) + +func day(n int) time.Time { return t0.Add(time.Duration(n) * 24 * time.Hour) } + +const stillOpen = 1 << 30 + +// pr builds a PRFiles open from day(open) to day(close); close==stillOpen +// leaves it open. +func pr(n, open, close int, files ...string) PRFiles { + p := PRFiles{Repo: "o/r", Number: n, Files: files, OpenedAt: day(open), SeenAt: day(open)} + if close != stillOpen { + p.ClosedAt = day(close) + p.Merged = true + } + return p +} + +func goSource(groups int, lines int) []byte { + var b strings.Builder + b.WriteString("package p\n\n") + for i := 0; i < groups; i++ { + b.WriteString("type T" + string(rune('A'+i)) + " struct{}\n\n") + b.WriteString("func (t *T" + string(rune('A'+i)) + ") M() {}\n\n") + } + for CountLines([]byte(b.String())) < lines { + b.WriteString("//\n") + } + return []byte(b.String()) +} + +func reader(files map[string][]byte) SourceReader { + return func(_, p string) (FileInfo, bool) { + src, ok := files[p] + if !ok { + return FileInfo{}, false + } + return FileInfo{Lines: CountLines(src), Source: src}, true + } +} + +func TestAnalyzeThresholdsAndGuard(t *testing.T) { + now := day(7) + bigGo := goSource(6, 1000) + smallGo := goSource(6, 200) + oneGroup := goSource(1, 1000) + files := map[string][]byte{ + "big.go": bigGo, + "small.go": smallGo, + "one.go": oneGroup, + "serial.go": bigGo, + } + // Six PRs all open concurrently on big.go, small.go, one.go → 15 pairs. + var prs []PRFiles + for i := 1; i <= 6; i++ { + prs = append(prs, pr(i, 0, 5, "big.go", "small.go", "one.go")) + } + // serial.go: 6 PRs, none overlapping (each closed before the next opens). + for i := 0; i < 6; i++ { + prs = append(prs, pr(100+i, i, i, "serial.go")) + } + // few.go: only 3 PRs. + for i := 0; i < 3; i++ { + prs = append(prs, pr(200+i, 0, 5, "few.go")) + } + conflicts := []Conflict{{Repo: "o/r", Number: 1, Files: []string{"big.go"}, At: day(3)}} + + hs := Analyze("o/r", prs, conflicts, reader(files), Options{}, now) + got := map[string]Hotspot{} + for _, h := range hs { + got[h.File] = h + } + if _, ok := got["serial.go"]; ok { + t.Fatalf("serial.go has no concurrent pairs and must be excluded") + } + if _, ok := got["few.go"]; ok { + t.Fatalf("few.go is below min_prs and must be excluded") + } + cases := []struct { + file string + rec Recommendation + }{ + {"big.go", RecommendSplitCode}, + {"small.go", RecommendNone}, + {"one.go", RecommendNone}, + } + for _, c := range cases { + h, ok := got[c.file] + if !ok { + t.Fatalf("%s missing from %+v", c.file, hs) + } + if h.PRs != 6 || h.Overlaps != 15 { + t.Errorf("%s: prs=%d overlaps=%d, want 6/15", c.file, h.PRs, h.Overlaps) + } + if h.Recommendation != c.rec { + t.Errorf("%s: recommendation %s (%s), want %s", c.file, h.Recommendation, h.Reason, c.rec) + } + } + big := got["big.go"] + if big.Conflicts != 1 { + t.Errorf("big.go conflicts=%d want 1", big.Conflicts) + } + // 15 overlaps × 1.25 (≥800 lines) + 2×1 conflict. + if want := 15*1.25 + 2; big.Score != want { + t.Errorf("big.go score=%v want %v", big.Score, want) + } + if hs[0].File != "big.go" { + t.Errorf("highest score first, got %s", hs[0].File) + } + if u := Units(big, Options{}); u != 4 { + t.Errorf("Units=%d want 4 (capped)", u) + } + if u := Units(Hotspot{Lines: 400, SymbolGroups: []string{"a", "b", "c"}}, Options{}); u != 2 { + t.Errorf("Units for 400 lines=%d want 2 (150-line floor)", u) + } +} + +func TestAnalyzeWindowExcludesOldPRs(t *testing.T) { + var prs []PRFiles + for i := 1; i <= 6; i++ { + prs = append(prs, pr(i, -20, -15, "a.go")) // closed long before the window + } + hs := Analyze("o/r", prs, nil, nil, Options{}, day(0)) + if len(hs) != 0 { + t.Fatalf("expected nothing in window, got %+v", hs) + } + // A still-open PR counts even if opened before the window. + prs = nil + for i := 1; i <= 6; i++ { + prs = append(prs, pr(i, -20, stillOpen, "a.go")) + } + hs = Analyze("o/r", prs, nil, nil, Options{}, day(0)) + if len(hs) != 1 || hs[0].Recommendation != RecommendNone { + t.Fatalf("open PRs: got %+v", hs) + } +} + +func TestAnalyzeKinds(t *testing.T) { + md := []byte(strings.Repeat("## Section one\n\ntext\n\n## Section two\n\ntext\n", 300)) + smallMd := []byte("## A\n\n## B\n") + gen := []byte("// Code generated by protoc-gen-go. DO NOT EDIT.\npackage p\n") + files := map[string][]byte{ + "docs/api-reference.md": md, + "README.md": smallMd, + "pkg/dashboard/static/index.html": []byte(strings.Repeat("
\n", 2000)), + "pkg/x/thing.pb.go": gen, + "go.sum": []byte("x\n"), + "pkg/dashboard/static-src/js/a.js": []byte(strings.Repeat("function f(){}\n", 1000)), + "pkg/dashboard/static-src/js/b.js": []byte("export function a(){}\nexport const b = 1;\nclass C {}\n" + strings.Repeat("//\n", 1000)), + } + var prs []PRFiles + paths := make([]string, 0, len(files)) + for p := range files { + paths = append(paths, p) + } + for i := 1; i <= 6; i++ { + prs = append(prs, pr(i, 0, 5, paths...)) + } + hs := Analyze("o/r", prs, nil, reader(files), Options{}, day(7)) + got := map[string]Hotspot{} + for _, h := range hs { + got[h.File] = h + } + want := map[string]struct { + kind Kind + rec Recommendation + }{ + "docs/api-reference.md": {KindDoc, RecommendSplitDoc}, + "README.md": {KindDoc, RecommendReportOnly}, + "pkg/dashboard/static/index.html": {KindGenerated, RecommendSplitSource}, + "pkg/x/thing.pb.go": {KindGenerated, RecommendReportOnly}, + "go.sum": {KindLock, RecommendReportOnly}, + "pkg/dashboard/static-src/js/a.js": {KindCode, RecommendNone}, // one symbol group + "pkg/dashboard/static-src/js/b.js": {KindCode, RecommendSplitCode}, + } + for p, w := range want { + h, ok := got[p] + if !ok { + t.Errorf("%s missing", p) + continue + } + if h.Kind != w.kind || h.Recommendation != w.rec { + t.Errorf("%s: kind=%s rec=%s (%s), want %s/%s", p, h.Kind, h.Recommendation, h.Reason, w.kind, w.rec) + } + } + if g := got["pkg/dashboard/static/index.html"].GeneratedFrom; g != "pkg/dashboard/static-src/" { + t.Errorf("GeneratedFrom=%q", g) + } + if g := got["docs/api-reference.md"].SymbolGroups; len(g) != 2 { + t.Errorf("md sections=%v", g) + } + if g := got["pkg/dashboard/static-src/js/b.js"].SymbolGroups; len(g) != 3 { + t.Errorf("js groups=%v", g) + } +} + +func TestGoGroupsCollapseMethodsIntoReceiver(t *testing.T) { + src := []byte(`package p +type A struct{} +func (a *A) X() {} +func (a A) Y() {} +type B[T any] struct{} +func (b *B[T]) Z() {} +func Free() {} +var v = 1 +`) + g := Groups("x.go", src) + if strings.Join(g, ",") != "A,B,Free" { + t.Fatalf("groups=%v", g) + } + if Groups("x.go", []byte("package p\nfunc (")) != nil { + t.Fatal("unparsable source must yield nil groups") + } +} + +type fakeCreator struct { + calls int + title string + body string + labels []string + err error +} + +func (f *fakeCreator) CreateHotspotIssue(_ context.Context, _, title, body string, labels []string) (int, string, error) { + f.calls++ + f.title, f.body, f.labels = title, body, labels + if f.err != nil { + return 0, "", f.err + } + return 4000 + f.calls, "https://x/" + title, nil +} + +type fakeStater struct { + open bool + closedAt time.Time +} + +func (f fakeStater) HotspotIssueState(context.Context, string, int) (bool, time.Time, error) { + return f.open, f.closedAt, nil +} + +type gateFn func(repo, file string) bool + +func (g gateFn) IssueFileAllowed(repo, file string) bool { return g(repo, file) } + +func hot() Hotspot { + return Hotspot{ + Repo: "o/r", File: "pkg/dashboard/api.go", Kind: KindCode, PRs: 8, PRNumbers: []int{1, 2, 3, 4, 5, 6, 7, 8}, + Overlaps: 11, Conflicts: 2, Lines: 2400, Score: 20.5, + SymbolGroups: []string{"Server", "handleAgents", "handleConfig", "handleStatus", "statusWithWebhooks"}, + Recommendation: RecommendSplitCode, Reason: "large file with several top-level groups hit by concurrent PRs", + } +} + +func TestFilerDedupeAndRefileWindow(t *testing.T) { + store, err := NewStore("") + if err != nil { + t.Fatal(err) + } + clock := day(0) + now := func() time.Time { return clock } + store.SetClock(now) + cr := &fakeCreator{} + st := &fakeStater{open: true} + f := &Filer{Store: store, Creator: cr, Stater: st, Now: now, Refile: 30 * 24 * time.Hour} + ctx := context.Background() + h := hot() + + out := f.File(ctx, "o/r", []Hotspot{h}) + if out[0].Action != "filed" || cr.calls != 1 { + t.Fatalf("first pass: %+v calls=%d", out, cr.calls) + } + if !strings.HasPrefix(cr.body, Marker(h.File)) { + t.Errorf("body must start with dedupe marker, got %q", cr.body[:60]) + } + if cr.title != "refactor: pkg/dashboard/api.go is a merge hotspot — split into 4 units" { + t.Errorf("title=%q", cr.title) + } + if strings.Join(cr.labels, ",") != "agent/ci-maintainer,kind/cleanup" { + t.Errorf("labels=%v", cr.labels) + } + + // Still open → no second issue. + out = f.File(ctx, "o/r", []Hotspot{h}) + if out[0].Action != "open" || cr.calls != 1 { + t.Fatalf("open pass: %+v calls=%d", out, cr.calls) + } + + // Closed 10 days ago, refile window 30 days → wait. + clock = day(20) + st.open, st.closedAt = false, day(10) + out = f.File(ctx, "o/r", []Hotspot{h}) + if out[0].Action != "refile-wait" || cr.calls != 1 { + t.Fatalf("refile-wait pass: %+v calls=%d", out, cr.calls) + } + if rec, _ := store.Issue("o/r", h.File); rec.ClosedAt != day(10) { + t.Errorf("close not recorded: %+v", rec) + } + + // Past the window → refile. + clock = day(41) + out = f.File(ctx, "o/r", []Hotspot{h}) + if out[0].Action != "filed" || cr.calls != 2 { + t.Fatalf("refile pass: %+v calls=%d", out, cr.calls) + } + + // Not actionable → skip; gated → gated. + h2 := h + h2.Recommendation = RecommendNone + if out := f.File(ctx, "o/r", []Hotspot{h2}); out[0].Action != "skip" { + t.Errorf("non-actionable: %+v", out) + } + h3 := h + h3.File = "other.go" + f.Gate = gateFn(func(string, string) bool { return false }) + if out := f.File(ctx, "o/r", []Hotspot{h3}); out[0].Action != "gated" || cr.calls != 2 { + t.Errorf("gated: %+v calls=%d", out, cr.calls) + } + f.Gate = nil + cr.err = errors.New("boom") + if out := f.File(ctx, "o/r", []Hotspot{h3}); out[0].Action != "error" { + t.Errorf("error: %+v", out) + } + if _, ok := store.Issue("o/r", "other.go"); ok { + t.Error("failed create must not be recorded") + } +} + +func TestBodyGolden(t *testing.T) { + body := Body(hot(), Options{}, DefaultWindow) + golden := filepath.Join("testdata", "issue_body.golden.md") + compareGolden(t, golden, body) + // Plain-language part before the table stays short. + pre := body[:strings.Index(body, "## Evidence")] + if n := len(strings.Fields(pre)); n > 300 { + t.Errorf("summary is %d words, want ≤ 300", n) + } + for _, want := range []string{"At most 4 resulting files, each at least 150 lines", "No behaviour change", "Tests move with the code", "| distinct PRs | 8 |", "`Server`"} { + if !strings.Contains(body, want) { + t.Errorf("body missing %q", want) + } + } + gen := hot() + gen.File = "pkg/dashboard/static/index.html" + gen.Kind = KindGenerated + gen.GeneratedFrom = "pkg/dashboard/static-src/" + gen.Recommendation = RecommendSplitSource + gen.SymbolGroups = nil + gb := Body(gen, Options{}, DefaultWindow) + if !strings.Contains(gb, "**generated**") || strings.Contains(gb, "## Split proposal") { + t.Errorf("generated body wrong:\n%s", gb) + } + if got := Title(gen, 4); got != "refactor: pkg/dashboard/static/index.html is a merge hotspot — split its source pkg/dashboard/static-src into 4 units" { + t.Errorf("generated title=%q", got) + } +} + +func TestStorePersistAndPrune(t *testing.T) { + p := filepath.Join(t.TempDir(), "h.json") + s, err := NewStore(p) + if err != nil { + t.Fatal(err) + } + clock := day(0) + s.SetClock(func() time.Time { return clock }) + s.ObservePR("o/r", 1, []string{"a.go"}, time.Time{}) + s.ObservePR("o/r", 2, []string{"a.go", "b.go"}, day(-1)) + s.ObserveConflict("o/r", 2, []string{"a.go"}) + s.ReconcileOpen("o/r", map[int]bool{2: true}) + s.RecordIssue(IssueRecord{Repo: "o/r", File: "a.go", Number: 9, FiledAt: clock}) + + s2, err := NewStore(p) + if err != nil { + t.Fatal(err) + } + prs, cs := s2.Snapshot("o/r") + if len(prs) != 2 || len(cs) != 1 { + t.Fatalf("reload: prs=%d conflicts=%d", len(prs), len(cs)) + } + if prs[0].Number != 1 || prs[0].ClosedAt.IsZero() || !prs[0].Merged { + t.Errorf("PR 1 should be reconciled closed: %+v", prs[0]) + } + if prs[1].ClosedAt.IsZero() == false { + t.Errorf("PR 2 must stay open: %+v", prs[1]) + } + if _, ok := s2.Issue("o/r", "a.go"); !ok { + t.Error("issue record lost") + } + + // Prune: PR 1 closed at day0; at day 10 with a 7-day window it goes. + s2.SetClock(func() time.Time { return day(10) }) + s2.MarkIssueClosed("o/r", "a.go", day(1)) + s2.Prune(7*24*time.Hour, 30*24*time.Hour) + prs, cs = s2.Snapshot("o/r") + if len(prs) != 1 || prs[0].Number != 2 || len(cs) != 0 { + t.Errorf("after prune: prs=%+v conflicts=%d", prs, len(cs)) + } + if _, ok := s2.Issue("o/r", "a.go"); !ok { + t.Error("issue closed 9 days ago must survive a 30-day refile prune") + } + s2.SetClock(func() time.Time { return day(40) }) + s2.Prune(7*24*time.Hour, 30*24*time.Hour) + if _, ok := s2.Issue("o/r", "a.go"); ok { + t.Error("issue closed 39 days ago must be pruned") + } + + s2.SetResults("o/r", []Hotspot{{Repo: "o/r", File: "b.go", Score: 1, Recommendation: RecommendSplitCode}}) + if total, act, _ := s2.Counts(); total != 1 || act != 1 { + t.Errorf("counts=%d/%d", total, act) + } +} + +func TestStoreExpireUnseenAndTouch(t *testing.T) { + s, _ := NewStore("") + clock := day(0) + s.SetClock(func() time.Time { return clock }) + s.ObservePR("o/r", 1, []string{"a.go"}, time.Time{}) + s.ObservePR("o/r", 2, []string{"a.go"}, time.Time{}) + clock = day(0).Add(3 * time.Hour) + s.TouchOpen("o/r", []int{2, 99}) + s.ExpireUnseen(2 * time.Hour) + prs, _ := s.Snapshot("o/r") + if len(prs) != 2 || prs[0].ClosedAt != day(0) || !prs[1].ClosedAt.IsZero() { + t.Fatalf("expire: %+v", prs) + } + // Seen open again → interval reopens. + s.ObservePR("o/r", 1, []string{"a.go"}, time.Time{}) + prs, _ = s.Snapshot("o/r") + if !prs[0].ClosedAt.IsZero() || prs[0].OpenedAt != day(0) { + t.Fatalf("reopen: %+v", prs[0]) + } +} diff --git a/src/pkg/hotspot/issue.go b/src/pkg/hotspot/issue.go new file mode 100644 index 0000000000..9b0525c4b1 --- /dev/null +++ b/src/pkg/hotspot/issue.go @@ -0,0 +1,298 @@ +package hotspot + +import ( + "context" + "fmt" + "path" + "sort" + "strings" + "time" +) + +// Labels applied to every modularize issue. Both exist on hivecommons/hive; +// the writer's label-ensure step creates them elsewhere. +var Labels = []string{"agent/ci-maintainer", "kind/cleanup"} + +// Title is the issue title for a hotspot. +func Title(h Hotspot, units int) string { + switch h.Recommendation { + case RecommendSplitSource: + return fmt.Sprintf("refactor: %s is a merge hotspot — split its source %s into %d units", h.File, strings.TrimSuffix(h.GeneratedFrom, "/"), units) + case RecommendSplitDoc: + return fmt.Sprintf("refactor: %s is a merge hotspot — split into %d sections", h.File, units) + } + return fmt.Sprintf("refactor: %s is a merge hotspot — split into %d units", h.File, units) +} + +// Body renders the issue body: plain-language summary (≤ 300 words), +// evidence table, the symbol groups, a concrete split proposal and the +// don't-splinter rule. The dedupe Marker is the first line. +func Body(h Hotspot, opts Options, window time.Duration) string { + opts = opts.withDefaults() + units := Units(h, opts) + days := int(window.Hours() / 24) + if days <= 0 { + days = int(DefaultWindow.Hours() / 24) + } + var b strings.Builder + b.WriteString(Marker(h.File)) + b.WriteString("\n\n") + fmt.Fprintf(&b, "`%s` keeps being touched by pull requests that are open at the same time. In the last %d days %d distinct PRs changed it, %d pairs of them were open concurrently, and %d merge conflicts or rebase failures were blamed on it. Every concurrent pair is a rebase someone (or the merge train) has to redo, and a %d-line file makes each of those rebases slower and riskier.\n\n", + h.File, days, h.PRs, h.Overlaps, h.Conflicts, h.Lines) + + switch h.Recommendation { + case RecommendSplitSource: + fmt.Fprintf(&b, "This file is **generated** — do not edit it by hand. The collisions come from its source under `%s`; split *there* so that unrelated changes stop regenerating the same output hunks, then regenerate once per PR.\n\n", h.GeneratedFrom) + case RecommendSplitDoc: + b.WriteString("This is a document, so the split is by **section**: each top-level `##` area becomes its own file under a directory named after this one, with a short index page that links them (the way `docs/api-reference.md` became `docs/api-reference/.md`). Keep every heading anchor reachable from the index.\n\n") + default: + fmt.Fprintf(&b, "Proposed fix: a **mechanical split** into %d files in the same package. No behaviour change, exported API unchanged, tests move with the code they cover. Land it when no other PR is open against this file (check the merge-train overlap first) so the split itself does not become one more collision.\n\n", units) + } + + b.WriteString("## Evidence\n\n") + fmt.Fprintf(&b, "| metric | value |\n|---|---|\n| window | %d days |\n| distinct PRs | %d |\n| concurrent PR pairs | %d |\n| conflicts attributed | %d |\n| lines | %d |\n| kind | %s |\n| score | %.1f |\n\n", days, h.PRs, h.Overlaps, h.Conflicts, h.Lines, h.Kind, h.Score) + if len(h.PRNumbers) > 0 { + b.WriteString("PRs in window: ") + for i, n := range h.PRNumbers { + if i > 0 { + b.WriteString(", ") + } + fmt.Fprintf(&b, "#%d", n) + } + b.WriteString("\n\n") + } + + if len(h.SymbolGroups) > 0 { + if h.Kind == KindDoc { + b.WriteString("## Sections\n\n") + } else { + b.WriteString("## Top-level units\n\n") + } + for _, g := range h.SymbolGroups { + fmt.Fprintf(&b, "- `%s`\n", g) + } + b.WriteString("\n") + } + + if h.Recommendation != RecommendSplitSource { + b.WriteString("## Split proposal\n\n") + for i, bucket := range proposeBuckets(h, units) { + fmt.Fprintf(&b, "%d. `%s` — %s\n", i+1, bucket.file, bucket.desc) + } + b.WriteString("\n") + } + + b.WriteString("## Don't splinter\n\n") + fmt.Fprintf(&b, "- At most %d resulting files, each at least %d lines; fewer is better.\n", opts.MaxUnits, opts.MinUnitLines) + b.WriteString("- No behaviour change; the package, its exported API and every import path stay the same.\n") + b.WriteString("- Tests move with the code they cover; no new test helpers.\n") + b.WriteString("- One PR; land it when the file has no open PRs against it.\n") + b.WriteString("\n_Filed by the hive's merge-hotspot advisor (`GET /api/hotspots`). It will not file another issue for this file while this one is open._\n") + return b.String() +} + +type bucket struct { + file string + desc string +} + +// proposeBuckets spreads the symbol groups across `units` files named after +// the original, grouped by common name prefix where that helps. +func proposeBuckets(h Hotspot, units int) []bucket { + ext := path.Ext(h.File) + stem := strings.TrimSuffix(h.File, ext) + stem = strings.TrimSuffix(stem, "_test") + groups := append([]string(nil), h.SymbolGroups...) + sort.Strings(groups) + if h.Kind == KindDoc { + dir := stem + "/" + out := []bucket{{file: dir + "README.md", desc: "index linking every section"}} + for _, g := range groups { + out = append(out, bucket{file: dir + slug(g) + ".md", desc: "section “" + g + "”"}) + if len(out) >= units+1 { + break + } + } + if len(groups) > len(out)-1 { + out[len(out)-1].desc += fmt.Sprintf(" and the %d remaining sections", len(groups)-(len(out)-1)) + } + return out + } + out := make([]bucket, units) + out[0] = bucket{file: h.File, desc: "core types and the entry points other files call"} + for i := 1; i < units; i++ { + out[i] = bucket{file: fmt.Sprintf("%s_%s%s", stem, partName(i, groups, units), ext)} + } + // Round-robin the groups so every bucket names what moves into it. + perBucket := make([][]string, units) + for i, g := range groups { + perBucket[i%units] = append(perBucket[i%units], g) + } + for i := range out { + if len(perBucket[i]) == 0 { + if out[i].desc == "" { + out[i].desc = "remaining helpers" + } + continue + } + var names []string + for j, g := range perBucket[i] { + if j >= 4 { + names = append(names, fmt.Sprintf("… %d more", len(perBucket[i])-j)) + break + } + names = append(names, "`"+g+"`") + } + if out[i].desc != "" { + out[i].desc += "; keeps " + } else { + out[i].desc = "moves " + } + out[i].desc += strings.Join(names, ", ") + } + return out +} + +// partName picks a file suffix for bucket i from the first group it will +// hold, falling back to partN. +func partName(i int, groups []string, units int) string { + if i < len(groups) { + if s := slug(groups[i]); s != "" { + return s + } + } + return fmt.Sprintf("part%d", i+1) +} + +func slug(s string) string { + var b strings.Builder + lastDash := true + for _, r := range strings.ToLower(s) { + switch { + case r >= 'a' && r <= 'z', r >= '0' && r <= '9': + b.WriteRune(r) + lastDash = false + default: + if !lastDash { + b.WriteByte('_') + lastDash = true + } + } + } + out := strings.Trim(b.String(), "_") + if len(out) > 24 { + out = strings.Trim(out[:24], "_") + } + return out +} + +// Gate is the writeback decision the Filer consults before creating an +// issue. pkg/github.Client.WritebackAllowed(repo, item, kind, preview) +// satisfies it through a small adapter in cmd/hive. +type Gate interface { + IssueFileAllowed(repo, file string) bool +} + +// IssueCreator creates the issue. It is satisfied by an adapter around +// github.Client.CreateIssue so that this package never imports pkg/github. +type IssueCreator interface { + CreateHotspotIssue(ctx context.Context, repo, title, body string, labels []string) (number int, url string, err error) +} + +// IssueStater reports whether a previously filed issue is still open and, +// when closed, when it closed. +type IssueStater interface { + HotspotIssueState(ctx context.Context, repo string, number int) (open bool, closedAt time.Time, err error) +} + +// Filer turns actionable hotspots into issues with the dedupe and refile +// rules applied. +type Filer struct { + Store *Store + Gate Gate + Creator IssueCreator + Stater IssueStater + Opts Options + Refile time.Duration + Now func() time.Time + Logf func(format string, args ...any) +} + +// Outcome is what File decided for one hotspot. +type Outcome struct { + File string + Action string // "filed", "open", "refile-wait", "gated", "skip", "error" + Number int + Err error +} + +// File walks hotspots and files at most one new issue per actionable file. +// It is safe to call every eval cycle: existing issues are checked, closed +// ones wait Refile before a new one may be opened. +func (f *Filer) File(ctx context.Context, repo string, hs []Hotspot) []Outcome { + now := time.Now + if f.Now != nil { + now = f.Now + } + refile := f.Refile + if refile <= 0 { + refile = time.Duration(DefaultHotspotRefileDays) * 24 * time.Hour + } + logf := f.Logf + if logf == nil { + logf = func(string, ...any) {} + } + var out []Outcome + for _, h := range hs { + if !h.Actionable() { + out = append(out, Outcome{File: h.File, Action: "skip"}) + continue + } + if rec, ok := f.Store.Issue(repo, h.File); ok { + if rec.ClosedAt.IsZero() && f.Stater != nil { + open, closedAt, err := f.Stater.HotspotIssueState(ctx, repo, rec.Number) + if err != nil { + out = append(out, Outcome{File: h.File, Action: "error", Number: rec.Number, Err: err}) + continue + } + if !open { + if closedAt.IsZero() { + closedAt = now() + } + f.Store.MarkIssueClosed(repo, h.File, closedAt) + rec.ClosedAt = closedAt + } + } + if rec.ClosedAt.IsZero() { + out = append(out, Outcome{File: h.File, Action: "open", Number: rec.Number}) + continue + } + if now().Sub(rec.ClosedAt) < refile { + out = append(out, Outcome{File: h.File, Action: "refile-wait", Number: rec.Number}) + continue + } + } + if f.Gate != nil && !f.Gate.IssueFileAllowed(repo, h.File) { + out = append(out, Outcome{File: h.File, Action: "gated"}) + continue + } + if f.Creator == nil { + out = append(out, Outcome{File: h.File, Action: "skip"}) + continue + } + units := Units(h, f.Opts) + num, url, err := f.Creator.CreateHotspotIssue(ctx, repo, Title(h, units), Body(h, f.Opts, f.Opts.withDefaults().Window), Labels) + if err != nil { + logf("hotspot: file issue for %s %s: %v", repo, h.File, err) + out = append(out, Outcome{File: h.File, Action: "error", Err: err}) + continue + } + f.Store.RecordIssue(IssueRecord{Repo: repo, File: h.File, Number: num, URL: url, FiledAt: now()}) + logf("hotspot: filed %s#%d for %s", repo, num, h.File) + out = append(out, Outcome{File: h.File, Action: "filed", Number: num}) + } + return out +} + +// DefaultHotspotRefileDays mirrors config.DefaultHotspotRefileDays. +const DefaultHotspotRefileDays = 30 diff --git a/src/pkg/hotspot/kinds.go b/src/pkg/hotspot/kinds.go new file mode 100644 index 0000000000..ea24207d16 --- /dev/null +++ b/src/pkg/hotspot/kinds.go @@ -0,0 +1,180 @@ +package hotspot + +import ( + "bytes" + "go/ast" + "go/parser" + "go/token" + "path" + "regexp" + "sort" + "strings" +) + +// Classify decides a file's Kind from its path and (when available) its +// first lines. Generated files also return the source tree that produces +// them, when known. +func Classify(p string, src []byte) (Kind, string) { + base := path.Base(p) + lower := strings.ToLower(p) + ext := strings.ToLower(path.Ext(base)) + + switch base { + case "go.sum", "package-lock.json", "yarn.lock", "pnpm-lock.yaml", "Cargo.lock", "poetry.lock", "Gemfile.lock", "composer.lock": + return KindLock, "" + } + if strings.HasSuffix(lower, ".pb.go") || strings.HasSuffix(lower, "_generated.go") || strings.HasSuffix(lower, ".gen.go") { + return KindGenerated, "" + } + // hive's dashboard SPA: pkg/dashboard/static/** is `go generate`d from + // pkg/dashboard/static-src/**. + if i := strings.Index(p, "/static/"); i >= 0 && strings.Contains(p, "dashboard") { + return KindGenerated, p[:i] + "/static-src/" + } + if strings.HasPrefix(p, "static/") && strings.Contains(p, "dashboard") { + return KindGenerated, "static-src/" + } + if head := firstLines(src, 5); bytes.Contains(head, []byte("Code generated")) && bytes.Contains(head, []byte("DO NOT EDIT")) { + return KindGenerated, generatedFromComment(head) + } + switch ext { + case ".md", ".rst", ".txt", ".adoc": + return KindDoc, "" + case ".json", ".yaml", ".yml", ".toml", ".csv", ".txt.golden", ".golden": + return KindData, "" + } + return KindCode, "" +} + +func firstLines(src []byte, n int) []byte { + idx := 0 + for i := 0; i < n && idx < len(src); i++ { + j := bytes.IndexByte(src[idx:], '\n') + if j < 0 { + return src + } + idx += j + 1 + } + return src[:idx] +} + +var generatedByRe = regexp.MustCompile(`(?i)generated (?:by|from) ([^\s;.]+)`) + +func generatedFromComment(head []byte) string { + if m := generatedByRe.FindSubmatch(head); m != nil { + return string(m[1]) + } + return "" +} + +// Groups lists the file's distinct top-level units: funcs and types for Go +// (methods are grouped under their receiver type), top-level declarations +// for JS/TS/Python, and `##` headings for Markdown. The result is sorted. +// It is nil when the source is unavailable or the language is unknown. +func Groups(p string, src []byte) []string { + if len(src) == 0 { + return nil + } + ext := strings.ToLower(path.Ext(p)) + var groups map[string]bool + switch ext { + case ".go": + groups = goGroups(src) + case ".js", ".mjs", ".cjs", ".ts", ".tsx", ".jsx": + groups = regexGroups(src, jsDeclRe) + case ".py": + groups = regexGroups(src, pyDeclRe) + case ".md", ".rst": + groups = regexGroups(src, mdHeadingRe) + default: + return nil + } + if len(groups) == 0 { + return nil + } + out := make([]string, 0, len(groups)) + for g := range groups { + out = append(out, g) + } + sort.Strings(out) + return out +} + +func goGroups(src []byte) map[string]bool { + fset := token.NewFileSet() + f, err := parser.ParseFile(fset, "", src, parser.SkipObjectResolution) + if err != nil || f == nil { + return nil + } + groups := map[string]bool{} + for _, d := range f.Decls { + switch d := d.(type) { + case *ast.FuncDecl: + if d.Recv != nil && len(d.Recv.List) > 0 { + if name := receiverName(d.Recv.List[0].Type); name != "" { + groups[name] = true + continue + } + } + if d.Name != nil { + groups[d.Name.Name] = true + } + case *ast.GenDecl: + if d.Tok != token.TYPE { + continue + } + for _, s := range d.Specs { + if ts, ok := s.(*ast.TypeSpec); ok && ts.Name != nil { + groups[ts.Name.Name] = true + } + } + } + } + return groups +} + +func receiverName(e ast.Expr) string { + switch t := e.(type) { + case *ast.StarExpr: + return receiverName(t.X) + case *ast.Ident: + return t.Name + case *ast.IndexExpr: + return receiverName(t.X) + case *ast.IndexListExpr: + return receiverName(t.X) + } + return "" +} + +var ( + // Top-level JS/TS declarations. The dashboard static-src fragments are + // spliced into a