diff --git a/changelog.d/added-15964-hub-activity-events.md b/changelog.d/added-15964-hub-activity-events.md new file mode 100644 index 0000000000..feae12d5c6 --- /dev/null +++ b/changelog.d/added-15964-hub-activity-events.md @@ -0,0 +1 @@ +- The heartbeat `activity` block now carries the stage hops, opens and merges a spoke saw since the hub last acknowledged them (at most 200, oldest dropped first, drops disclosed as `events_dropped`), and `GET /api/hub/activity` serves them as `events` from a fleet-wide, memory-only ring of the last 1,000 events, filtered per user and per hive like items, so the hub board can animate movement and show merge toasts ([#15964](https://github.com/hivecommons/hive/issues/15964), RFC [#15861](https://github.com/hivecommons/hive/issues/15861)). diff --git a/src/cmd/hive/main.go b/src/cmd/hive/main.go index 207bf617db..e304f38c60 100644 --- a/src/cmd/hive/main.go +++ b/src/cmd/hive/main.go @@ -5686,6 +5686,14 @@ func (b *boot) bootHeartbeatWith(deps bootHeartbeatDeps) { } return b.dashSrv.HubActivityItems(ctx) }, + // Snapshot supersedes Items and adds recent merges, from + // which the publisher derives merge events (#15964). + Snapshot: func(ctx context.Context) (hubactivity.Snapshot, bool) { + if b.dashSrv == nil { + return hubactivity.Snapshot{}, false + } + return b.dashSrv.HubActivitySnapshot(ctx) + }, Subscribe: func(notify func()) { if b.dashSrv != nil { b.dashSrv.OnStageTrackerPublish(notify) diff --git a/src/docs/api-reference/hub-server.md b/src/docs/api-reference/hub-server.md index ab623d6bee..3fba7b9f13 100644 --- a/src/docs/api-reference/hub-server.md +++ b/src/docs/api-reference/hub-server.md @@ -38,7 +38,7 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `POST` | `/api/shared-agent-pool/leases` | Hub handler-specific | Acquire or renew a shared CLI session lease | `pkg/hub/server.go:1708` | | `PUT` | `/api/shared-agent-pool/leases` | Hub handler-specific | Renew an active shared CLI session lease | `pkg/hub/server.go:1709` | | `DELETE` | `/api/shared-agent-pool/leases` | Hub handler-specific | Release a shared CLI session lease | `pkg/hub/server.go:1710` | -| `POST` | `/api/heartbeat` | Hub handler-specific | Heartbeat. A request carrying `X-Hive-Activity-Update` is an activity-only update ([#15963](https://github.com/hivecommons/hive/issues/15963)): body `{"activity_update":{"hive_id","activity"}}`, the same per-hive bearer, accepted only for a hive already in the registry (`404` otherwise); it replaces the in-memory activity digest and nothing else, and the signed response carries `ok` and `activity_etag` | `pkg/hub/server.go:1711` | +| `POST` | `/api/heartbeat` | Hub handler-specific | Heartbeat. A request carrying `X-Hive-Activity-Update` is an activity-only update ([#15963](https://github.com/hivecommons/hive/issues/15963)): body `{"activity_update":{"hive_id","activity"}}`, the same per-hive bearer, accepted only for a hive already in the registry (`404` otherwise); it replaces the in-memory activity digest and nothing else, and the signed response carries `ok`, `activity_etag` and `activity_events_ack` (the highest activity event sequence number the hub holds for the hive, [#15964](https://github.com/hivecommons/hive/issues/15964); full heartbeat responses carry it too) | `pkg/hub/server.go:1711` | | `POST` | `/api/task-status` | Hub handler-specific | Task Status | `pkg/hub/server.go:1712` | | `GET` | `/api/registry` | Hub handler-specific | Registry | `pkg/hub/server.go:1720` | | `GET` | `/api/hub/leaderboard` | Hub handler-specific | Leaderboard | `pkg/hub/server.go:1721` | @@ -46,7 +46,7 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/fleet-stats` | Hub handler-specific | Fleet Stats | `pkg/hub/server.go:1723` | | `GET` | `/api/hub/evidence` | Hub admin | Read-only cross-hive evidence drill-down from each spoke's stored heartbeat `evidence` summary ([#12116](https://github.com/hivecommons/hive/issues/12116)); non-admins get `403`. Returns `spokes`, `reporting`, pooled `calibration` (bucket 0–5 → `{n, bad, rate, lo, hi}`; the hub view folds the six buckets into four reviewer-confidence ranges), `findings[]` (the plain-language *What this says* sentences, generated from the same fields; [#12456](https://github.com/hivecommons/hive/issues/12456)), `by_acmm_level` (`level`, `hives`, `merged_90d`, `revert_rate`, and `time_to_merge` — `median_of_hive_medians_s`, `hives`, `n` — the median of the level's per-hive median time-to-merge, omitted when no hive at the level reports one; [#12464](https://github.com/hivecommons/hive/issues/12464)), `family_false_approve_rate` (`same`/`cross`/`mixed`, hives classed by their reviewer vs author family sets), fleet `record_coverage` (merged PRs with an evidence record over `prs_merged_90d`), fleet panels ([#12370](https://github.com/hivecommons/hive/issues/12370)) `outcomes_by_author_kind[]` (`kind`, `hives`, `n`, `merged`, `reverted`, `merge_rate`, `revert_rate`), `write_counters[]` (`tier`, `hives`, `attempted`, `blocked`, `block_rate`) and `shadow_gate` (`review_gate` / `intent` → `held` / `passed` outcomes with `revert_rate`), each omitted while no spoke reports it and repeated per `by_acmm_level[]` row, and per hive `hives[]` with `acmm_level`, `prs_merged_90d`, `record_coverage`, `family_mix`, false-approve / false-block / revert rates, families, a calibration series the raw `counts` behind them (including `by_author_kind`) so the drill-down can re-pool a hive / ACMM-level selection client-side, and `criteria[]`: the per-axis criteria table derived hub-side from the quadrant criteria (`axis`, `label`, `scored`, `criteria[]` with `name`, `label`, `status` `met`/`unmet`/`not_reported`, `display`/`score` when measured, `reason` otherwise; offline, evidence-less or unscored hives are `not_reported`, never `unmet`; [#12463](https://github.com/hivecommons/hive/issues/12463)). Every rate is `{n, rate, lo, hi}` with a 95% Wilson interval; `rate`/`lo`/`hi` are null when `n` < 10. Window selection ([RFC #12062](https://github.com/hivecommons/hive/issues/12062)): `?hours=24|168|720` pools each hive's matching heartbeat `windows` block instead of its 90-day block (any other or absent `hours` → 90 days); `?from=YYYY-MM-DD|RFC3339[&to=…]` (date-only `to` is inclusive, `to` defaults to now, `to` ≥ `from`, span ≤ 366 days, else `400`) is served from the smallest reported window that reaches back to `from` and flagged `window.approximate`. The response carries `window` (`hours`, `key`, `label`, `from`, `to`, `approximate`, `coverage_available` — `record_coverage` is only computed for the 90-day window because `prs_merged_90d` is a 90-day count), `window_unreported` and per-hive `window_unreported` for hives whose spoke predates per-window reporting. `?hive=[,…]` (repeatable) and `?level=` restrict the registry server-side to those hives / that ACMM level before pooling. Hub admins open it from the Fleet quadrant card or its **🔬 Evidence** button ([#12117](https://github.com/hivecommons/hive/issues/12117)), pick a window (24h / 7d / 30d / 90d / custom), filter it by hive and ACMM level and read each section's ⓘ explainer; the per-hive author-kind revert counts behind the *Revert rate by author kind* panel are `hives[].counts.by_author_kind` ([#12326](https://github.com/hivecommons/hive/issues/12326)). Never consulted by merge decisions | `pkg/hub/evidence.go` | | `GET` | `/api/hub/evidence/report` | Hub admin | Plain-language report of the hub evidence view ([#12456](https://github.com/hivecommons/hive/issues/12456)), rendered server-side from the same `EvidenceView` as `/api/hub/evidence` so every figure reconciles with the modal; non-admins get `403`. Same `hours=` / `from=` / `to=` window and `hive=` / `level=` filter parameters. `format=md` (default, `text/markdown`) or `format=html` (standalone page, inline CSS, no scripts or external assets, hive names escaped); any other value → `400`. `download=1` sets `Content-Disposition: attachment; filename="hive-evidence---.md|html"`, otherwise inline. Sections: Summary (the generated findings), What we measured and why, The numbers, How to read these, What is not reported yet and why, Grade per section, footer link to `review-evidence.md`. `Cache-Control: no-store`. Read-only; never consulted by merge decisions | `pkg/hub/evidence_report.go` | -| `GET` | `/api/hub/activity` | Signed-in user | Read-only hub-wide activity view ([#15951](https://github.com/hivecommons/hive/issues/15951), RFC [#15861](https://github.com/hivecommons/hive/issues/15861)) merged from each spoke's heartbeat `activity` digest, held in hub memory only. Authorization is per user and per hive before aggregation: hub admins see every registered hive; everyone else only hives they own or hold an unexpired grant for; anonymous → `401`. Returns `hives[]` (`hive_id`, `name`, `status` `ok`/`stale`/`not_reporting`/`missing`, `reporting`, `stale` once the block is older than two heartbeats, `last_heartbeat`, `received_at`, `generated_at`, `totals` — `items`, `by_stage`, `by_modifier`, counting private and truncated items too — and `truncated`), `items[]` (`hive_id`, `repo`, `n`, `kind`, `title`, `author_kind`, `stage`, `stage_n`, `since`, `mods`, `private`) ordered needs-human/needs-decision, stalled, furthest stage, oldest, and a response-level `truncated` (cap 2000). Each digest carries at most 100 items by default and never more than the 250-item hub ceiling; private-repository items appear only when the spoke opts in. Filters: `hive`, `repo`, `stage`, `mod`, `author_kind`, `q` (title substring or `#`), each ≤ 200 characters (else `400`). `ETag` + `If-None-Match` → `304`; `Cache-Control: private, no-cache`, `Vary: Cookie, Authorization`. No actions | `pkg/hub/activity.go` | +| `GET` | `/api/hub/activity` | Signed-in user | Read-only hub-wide activity view ([#15951](https://github.com/hivecommons/hive/issues/15951), RFC [#15861](https://github.com/hivecommons/hive/issues/15861)) merged from each spoke's heartbeat `activity` digest, held in hub memory only. Authorization is per user and per hive before aggregation: hub admins see every registered hive; everyone else only hives they own or hold an unexpired grant for; anonymous → `401`. Returns `hives[]` (`hive_id`, `name`, `status` `ok`/`stale`/`not_reporting`/`missing`, `reporting`, `stale` once the block is older than two heartbeats, `last_heartbeat`, `received_at`, `generated_at`, `totals` — `items`, `by_stage`, `by_modifier`, counting private and truncated items too — `truncated` and `events_dropped`), `items[]` (`hive_id`, `repo`, `n`, `kind`, `title`, `author_kind`, `stage`, `stage_n`, `since`, `mods`, `private`) ordered needs-human/needs-decision, stalled, furthest stage, oldest, a response-level `truncated` (cap 2000), and `events[]` ([#15964](https://github.com/hivecommons/hive/issues/15964): `hive_id`, `received_at`, `seq`, `type` `opened`/`stage`/`merged`, `repo`, `n`, `kind`, `title`, `from`, `stage`, `at`, `private`), oldest first, from a fleet-wide in-memory ring of the last 1,000 events. Events are filtered per user and per hive exactly like items, before aggregation, so an event for a hive the viewer cannot see is never delivered; `mod`/`author_kind` filters keep only events whose item is still on the hive's board. A spoke block carries at most 200 events; drops are disclosed per hive as `events_dropped`. New events change the `ETag`. Each digest carries at most 100 items by default and never more than the 250-item hub ceiling; private-repository items appear only when the spoke opts in. Filters: `hive`, `repo`, `stage`, `mod`, `author_kind`, `q` (title substring or `#`), each ≤ 200 characters (else `400`). `ETag` + `If-None-Match` → `304`; `Cache-Control: private, no-cache`, `Vary: Cookie, Authorization`. No actions | `pkg/hub/activity.go` | | `GET` | `/api/hub/version` | Hub handler-specific | Hub Version | `pkg/hub/server.go:1731` | | `DELETE` | `/api/hub/registry/{id}` | Hub handler-specific | Registry Delete | `pkg/hub/server.go:1741` | | `POST` | `/api/contribute/register` | Hub handler-specific | Contribute Proxy | `pkg/hub/server.go:1742` | diff --git a/src/docs/telemetry.md b/src/docs/telemetry.md index ce3713ab55..9f5a56a4cb 100644 --- a/src/docs/telemetry.md +++ b/src/docs/telemetry.md @@ -72,7 +72,7 @@ hub: share_private: false # private-repository items are dropped (still counted in totals) unless true ``` -When the cap applies, items needing a human or a decision are kept first, then stalled ones, then the furthest stage, and the number left out is reported as `truncated`. A repository counts as public only when the hub lists the hive as public (`hub.is_public`, its project repos) or it is on `HIVE_PUBLIC_REPOS`. Once the hub acknowledges a digest's etag, later beats send only `etag` and `generated_at` until the tracker changes. A tracker change also sends an activity-only update over the heartbeat endpoint and bearer, at most one every 15 seconds; changes arriving meanwhile are merged into it. `hub.heartbeat_omit` applies: `repos` withholds every item (totals remain) and `task_titles` blanks titles. +When the cap applies, items needing a human or a decision are kept first, then stalled ones, then the furthest stage, and the number left out is reported as `truncated`. A repository counts as public only when the hub lists the hive as public (`hub.is_public`, its project repos) or it is on `HIVE_PUBLIC_REPOS`. Once the hub acknowledges a digest's etag, later beats send only `etag` and `generated_at` until the tracker changes. A tracker change also sends an activity-only update over the heartbeat endpoint and bearer, at most one every 15 seconds; changes arriving meanwhile are merged into it. The block also carries `events` ([#15964](https://github.com/hivecommons/hive/issues/15964)): the stage hops, opens and merges the spoke saw since the hub last acknowledged them, derived by diffing successive tracker snapshots (the first one only seeds the baseline), each with a sequence number. At most 200 are buffered; the oldest are dropped first and `events_dropped` discloses how many. Events stay buffered, and are resent, until a heartbeat or update response's `activity_events_ack` covers them. Private-repository items yield no events unless `share_private` is on. The hub keeps the last 1,000 events across the fleet in memory only. `hub.heartbeat_omit` applies: `repos` withholds every item and event (totals remain) and `task_titles` blanks titles. ## Telemetry agent diff --git a/src/pkg/dashboard/hub_activity.go b/src/pkg/dashboard/hub_activity.go index ed575d880a..a278242a58 100644 --- a/src/pkg/dashboard/hub_activity.go +++ b/src/pkg/dashboard/hub_activity.go @@ -46,26 +46,48 @@ func (s *Server) notifyStageTrackerPublish() { // while the spoke has no status or the tracker cannot be built. Reading // does not count as a tracker client, so it adds no SSE nudges. func (s *Server) HubActivityItems(ctx context.Context) ([]hubactivity.Item, bool) { + snap, ok := s.HubActivitySnapshot(ctx) + return snap.Items, ok +} + +// HubActivitySnapshot is HubActivityItems plus the tracker's recently +// merged pull requests (RecentlyMerged), which the heartbeat publisher +// turns into merge events (#15964). Merges carry Private the same way. +func (s *Server) HubActivitySnapshot(ctx context.Context) (hubactivity.Snapshot, bool) { if s == nil { - return nil, false + return hubactivity.Snapshot{}, false } s.statusMu.RLock() status := s.status s.statusMu.RUnlock() if status == nil { - return nil, false + return hubactivity.Snapshot{}, false } body, _, _, _ := s.stageTrackerPayloadFrom(ctx, status, time.Now().UTC(), false) if body == nil { - return nil, false + return hubactivity.Snapshot{}, false } var tracker struct { - Items []StageTrackerItem `json:"items"` + Items []StageTrackerItem `json:"items"` + RecentlyMerged []StageTrackerMerged `json:"recently_merged"` } if err := json.Unmarshal(body, &tracker); err != nil { - return nil, false + return hubactivity.Snapshot{}, false + } + public := s.publicRepos() + return hubactivity.Snapshot{ + Items: hubActivityItems(tracker.Items, public), + Merged: hubActivityMerges(tracker.RecentlyMerged, public), + }, true +} + +// hubActivityMerges maps the tracker's recently merged list to merge rows. +func hubActivityMerges(merged []StageTrackerMerged, public publicRepoSet) []hubactivity.Merge { + out := make([]hubactivity.Merge, 0, len(merged)) + for _, m := range merged { + out = append(out, hubactivity.Merge{Repo: m.Repo, Number: m.Number, MergedAt: m.MergedAt, Private: !public.allows(m.Repo)}) } - return hubActivityItems(tracker.Items, s.publicRepos()), true + return out } // hubActivityItems maps tracker rows to digest rows. A row the tracker diff --git a/src/pkg/hub/activity.go b/src/pkg/hub/activity.go index abe58319e5..415e5ecbaf 100644 --- a/src/pkg/hub/activity.go +++ b/src/pkg/hub/activity.go @@ -64,6 +64,8 @@ type storedActivity struct { type activityStore struct { mu sync.Mutex byHive map[string]storedActivity + // events is the fleet-wide ring of recent activity events (#15964). + events activityEventRing } // record applies a heartbeat's activity block and returns the etag the hub @@ -85,6 +87,9 @@ func (st *activityStore) record(hiveID string, block *hubactivity.Digest, now ti } return "" } + // Events ride on markers and full digests alike, and are taken even + // when the marker's etag misses. + st.events.ingest(hiveID, block.Events, block.EventsDropped, now) if block.IsUnchangedMarker() { // Compare in the stored (cleaned) form, or a marker whose etag // changes under cleaning would never match and wedge the handshake. @@ -124,6 +129,7 @@ func (st *activityStore) forget(hiveID string) { st.mu.Lock() defer st.mu.Unlock() delete(st.byHive, hiveID) + st.events.forget(hiveID) } func (st *activityStore) forgetOldLocked(now time.Time) { @@ -132,6 +138,7 @@ func (st *activityStore) forgetOldLocked(now time.Time) { delete(st.byHive, id) } } + st.events.forgetOld(now, activityForgetAfter) } // snapshot copies the blocks of the given hives only, so data of hives the @@ -262,6 +269,9 @@ type ActivityHiveView struct { Totals *hubactivity.Totals `json:"totals,omitempty"` // Truncated is how many of this hive's items its digest left out. Truncated int `json:"truncated,omitempty"` + // EventsDropped is how many of this hive's events the spoke's or the + // hub's per-block cap dropped before they reached the ring (#15964). + EventsDropped int `json:"events_dropped,omitempty"` } // ActivityViewItem is one merged item tagged with its hive. @@ -274,6 +284,9 @@ type ActivityViewItem struct { type ActivityView struct { Hives []ActivityHiveView `json:"hives"` Items []ActivityViewItem `json:"items"` + // Events are the visible hives' recent stage hops, opens and merges + // from the fleet-wide ring, oldest first (#15964). + Events []ActivityViewEvent `json:"events"` // Truncated is how many matching items the response cap left out, on // top of what each hive's own digest already left out. Truncated int `json:"truncated,omitempty"` @@ -282,7 +295,7 @@ type ActivityView struct { // buildActivityView merges the blocks of the visible hives only. Hives must // already be authorized for the viewer. func buildActivityView(hives []activityHive, blocks map[string]storedActivity, f activityFilter, now time.Time) ActivityView { - view := ActivityView{Hives: []ActivityHiveView{}, Items: []ActivityViewItem{}} + view := ActivityView{Hives: []ActivityHiveView{}, Items: []ActivityViewItem{}, Events: []ActivityViewEvent{}} for _, h := range hives { hv := ActivityHiveView{HiveID: h.ID, Name: h.Name, LastHeartbeat: h.LastHeartbeat} block, ok := blocks[h.ID] @@ -359,7 +372,9 @@ func (s *HubServer) handleHubActivity(w http.ResponseWriter, r *http.Request) { for i, h := range hives { ids[i] = h.ID } - view := buildActivityView(hives, s.activity.snapshot(ids), f, now) + blocks := s.activity.snapshot(ids) + view := buildActivityView(hives, blocks, f, now) + addActivityEvents(&view, hives, blocks, s.activity.eventsSnapshot(ids), f) body, err := json.Marshal(view) if err != nil { s.logger.Error("hub activity: encode view", "error", err) diff --git a/src/pkg/hub/activity_events.go b/src/pkg/hub/activity_events.go new file mode 100644 index 0000000000..e5aa5ffa83 --- /dev/null +++ b/src/pkg/hub/activity_events.go @@ -0,0 +1,227 @@ +package hub + +import ( + "slices" + "strconv" + "strings" + "time" + + "github.com/hivecommons/hive/pkg/hubactivity" +) + +// Fleet-wide activity event ring (RFC hivecommons/hive#15861 §A/§B, child +// #15964). Spokes attach the stage hops, opens and merges they saw since +// the hub last acknowledged to their activity block; the hub keeps the most +// recent activityEventRingSize of them across the fleet IN MEMORY ONLY and +// serves them on GET /api/hub/activity, so the hub board can animate +// movement and show merge toasts. Events are copied out per hive only for +// hives the viewer is authorized for, exactly like digests, and they are +// part of the response body, so its ETag changes when new events arrive. + +// activityEventRingSize bounds the fleet-wide ring; the oldest event is +// dropped first. +const activityEventRingSize = 1000 + +// activityEvent is one stored event tagged with its hive. +type activityEvent struct { + hiveID string + event hubactivity.Event + receivedAt time.Time +} + +// activityEventHive is the per-hive acknowledgement state. +type activityEventHive struct { + // lastSeq is the highest event sequence number accepted (the ack). + lastSeq uint64 + // dropped sums the drops the spoke disclosed and the hub's own cap + // applied to its batches. + dropped int + // disclosed is the drop count the latest counted batch claimed (the + // spoke's disclosure plus the hub's cap). The spoke's disclosure is + // cumulative until an ack covers its latest build, so a batch resent + // after a lost ack restates the drops already counted; only the + // increase over disclosed is added then. + disclosed int + lastSeen time.Time +} + +// activityEventRing is guarded by activityStore.mu. The zero value is +// ready to use. +type activityEventRing struct { + entries []activityEvent + byHive map[string]*activityEventHive +} + +// ingest takes a spoke's events. Only events newer than the hive's +// acknowledged sequence number are kept, so a batch resent because an +// acknowledgement was lost is not stored (or its drops counted) twice. +func (r *activityEventRing) ingest(hiveID string, events []hubactivity.Event, dropped int, now time.Time) { + clean, capped := hubactivity.SanitizeEvents(events) + h := r.byHive[hiveID] + if h == nil { + if len(clean) == 0 { + return + } + if r.byHive == nil { + r.byHive = map[string]*activityEventHive{} + } + h = &activityEventHive{} + r.byHive[hiveID] = h + } + h.lastSeen = now + fresh, restated := 0, false + for _, e := range clean { + if e.Seq <= h.lastSeq { + // The spoke trims acknowledged events, so holding one proves + // it never saw the ack and its drop disclosure is cumulative + // over the one already counted. Known limit: if the spoke + // buffered more than hubactivity.MaxEvents new events before + // the resend, the earlier events were trimmed for room and + // nothing here repeats a Seq, so the restatement goes + // undetected and the previous batch's drops are counted once + // more. The over-count is bounded by one batch's disclosure + // and affects only the drop counter, never stored events. + restated = true + continue + } + h.lastSeq = e.Seq + r.entries = append(r.entries, activityEvent{hiveID: hiveID, event: e, receivedAt: now}) + fresh++ + } + if fresh == 0 { + return + } + claimed := hubactivity.ClampCount(hubactivity.ClampCount(dropped) + capped) + add := claimed + if restated { + add = max(claimed-h.disclosed, 0) + } + h.disclosed = claimed + h.dropped = hubactivity.ClampCount(h.dropped + add) + if over := len(r.entries) - activityEventRingSize; over > 0 { + r.entries = slices.Delete(r.entries, 0, over) + } +} + +// ack is the highest sequence number held for the hive (0 = none). +func (r *activityEventRing) ack(hiveID string) uint64 { + if h := r.byHive[hiveID]; h != nil { + return h.lastSeq + } + return 0 +} + +// forget drops a hive's events and acknowledgement state. +func (r *activityEventRing) forget(hiveID string) { + if _, ok := r.byHive[hiveID]; !ok { + return + } + delete(r.byHive, hiveID) + r.entries = slices.DeleteFunc(r.entries, func(e activityEvent) bool { return e.hiveID == hiveID }) +} + +// forgetOld forgets hives that sent nothing for maxAge. +func (r *activityEventRing) forgetOld(now time.Time, maxAge time.Duration) { + for id, h := range r.byHive { + if now.Sub(h.lastSeen) > maxAge { + r.forget(id) + } + } +} + +// activityEventsSnapshot is the events and drop counts of a set of hives. +type activityEventsSnapshot struct { + events []activityEvent + dropped map[string]int +} + +// eventsAck is the highest event sequence number the hub holds for hiveID, +// returned to the spoke so it can trim its buffer. +func (st *activityStore) eventsAck(hiveID string) uint64 { + if st == nil { + return 0 + } + st.mu.Lock() + defer st.mu.Unlock() + return st.events.ack(hiveID) +} + +// eventsSnapshot copies the events of the given hives only, oldest first, +// so another hive's events are never copied out of the store. +func (st *activityStore) eventsSnapshot(hiveIDs []string) activityEventsSnapshot { + st.mu.Lock() + defer st.mu.Unlock() + want := make(map[string]bool, len(hiveIDs)) + for _, id := range hiveIDs { + want[id] = true + } + out := activityEventsSnapshot{dropped: map[string]int{}} + for _, e := range st.events.entries { + if want[e.hiveID] { + out.events = append(out.events, e) + } + } + for id := range want { + if h := st.events.byHive[id]; h != nil && h.dropped > 0 { + out.dropped[id] = h.dropped + } + } + return out +} + +// ActivityViewEvent is one event tagged with its hive. +type ActivityViewEvent struct { + HiveID string `json:"hive_id"` + ReceivedAt time.Time `json:"received_at"` + hubactivity.Event +} + +// matchesEvent applies the view filter to an event. Filters the event does +// not carry (modifier, author kind) are checked against the item's current +// row in the hive's digest; an event whose item is not there is left out. +func (f activityFilter) matchesEvent(hiveID string, e hubactivity.Event, current map[string]hubactivity.Item) bool { + it := hubactivity.Item{Repo: e.Repo, Number: e.Number, Kind: e.Kind, Title: e.Title, Stage: e.Stage} + if f.Mod != "" || f.AuthorKind != "" { + cur, ok := current[activityItemKey(e.Repo, e.Number)] + if !ok { + return false + } + it.Mods, it.AuthorKind = cur.Mods, cur.AuthorKind + } + return f.matches(hiveID, it) +} + +func activityItemKey(repo string, n int) string { + return strings.ToLower(repo) + "#" + strconv.Itoa(n) +} + +// addActivityEvents adds the filtered events of the visible hives to view. +// Hives must already be authorized for the viewer, and the snapshot taken +// for them only. +func addActivityEvents(view *ActivityView, hives []activityHive, blocks map[string]storedActivity, snap activityEventsSnapshot, f activityFilter) { + visible := make(map[string]bool, len(hives)) + for _, h := range hives { + if !h.Missing { + visible[h.ID] = true + } + } + current := map[string]map[string]hubactivity.Item{} + if f.Mod != "" || f.AuthorKind != "" { + for id, b := range blocks { + m := make(map[string]hubactivity.Item, len(b.digest.Items)) + for _, it := range b.digest.Items { + m[activityItemKey(it.Repo, it.Number)] = it + } + current[id] = m + } + } + for _, e := range snap.events { + if !visible[e.hiveID] || !f.matchesEvent(e.hiveID, e.event, current[e.hiveID]) { + continue + } + view.Events = append(view.Events, ActivityViewEvent{HiveID: e.hiveID, ReceivedAt: e.receivedAt.UTC(), Event: e.event}) + } + for i := range view.Hives { + view.Hives[i].EventsDropped = snap.dropped[view.Hives[i].HiveID] + } +} diff --git a/src/pkg/hub/activity_events_test.go b/src/pkg/hub/activity_events_test.go new file mode 100644 index 0000000000..a83b1a5220 --- /dev/null +++ b/src/pkg/hub/activity_events_test.go @@ -0,0 +1,277 @@ +package hub + +import ( + "fmt" + "net/http" + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/hubactivity" +) + +func activityEv(seq uint64, typ, repo string, n int) hubactivity.Event { + return hubactivity.Event{Seq: seq, Type: typ, Repo: repo, Number: n, Kind: "pr", Title: fmt.Sprintf("%s #%d", repo, n), Stage: "ci_review", At: activityT0} +} + +func withEvents(d *hubactivity.Digest, events ...hubactivity.Event) *hubactivity.Digest { + out := *d + out.Events = events + return &out +} + +func viewEventKeys(v ActivityView) []string { + out := make([]string, 0, len(v.Events)) + for _, e := range v.Events { + out = append(out, fmt.Sprintf("%s:%s#%d:%s", e.HiveID, e.Repo, e.Number, e.Type)) + } + return out +} + +// Acceptance (#15964): a merge a spoke reports on a heartbeat is served by +// /api/hub/activity right after that one heartbeat, and the ETag changes +// so a poller picks it up. +func TestHubActivityEvents_MergeAppearsWithinOneHeartbeat(t *testing.T) { + cleanup := helperSetupTempDirs(t) + defer cleanup() + t.Setenv(hubAdminsEnv, "github:root") + s := newActivityHub() + mkUser(t, "root") + + d := activityDigest("o/a", 1, 2) + postActivityHeartbeat(t, s, "hive-a", d) + first := getActivity(t, s, "root", "", "") + if v := decodeActivity(t, first); len(v.Events) != 0 { + t.Fatalf("events before any movement = %v", viewEventKeys(v)) + } + etag := first.Header().Get("ETag") + + marker := &hubactivity.Digest{ETag: d.ETag, GeneratedAt: activityT0.Add(time.Minute)} + merge := activityEv(7, hubactivity.EventMerged, "o/a", 2) + resp := postActivityHeartbeat(t, s, "hive-a", withEvents(marker, merge)) + if resp.ActivityEventsAck != 7 || resp.ActivityETag != d.ETag { + t.Fatalf("heartbeat ack = %q / %d, want the held etag and event seq 7", resp.ActivityETag, resp.ActivityEventsAck) + } + + rec := getActivity(t, s, "root", "", etag) + if rec.Code != http.StatusOK || rec.Header().Get("ETag") == etag { + t.Fatalf("new events must change the etag: status %d etag %q (was %q)", rec.Code, rec.Header().Get("ETag"), etag) + } + v := decodeActivity(t, rec) + if got := viewEventKeys(v); len(got) != 1 || got[0] != "hive-a:o/a#2:merged" { + t.Fatalf("events = %v, want the merge", got) + } + if len(v.Items) != 2 { + t.Fatalf("a marker carrying events must keep the held digest: items %d", len(v.Items)) + } +} + +// An event of a hive the user is not authorized for is never delivered; +// filtering happens before the response (and its etag) is built. +func TestHubActivityEvents_PerUserScoping(t *testing.T) { + cleanup := helperSetupTempDirs(t) + defer cleanup() + t.Setenv(hubAdminsEnv, "github:root") + s := newActivityHub() + + postActivityHeartbeat(t, s, "hive-a", withEvents(activityDigest("o/a", 1), activityEv(1, hubactivity.EventOpened, "o/a", 1))) + postActivityHeartbeat(t, s, "hive-b", withEvents(activityDigest("o/b", 3), activityEv(1, hubactivity.EventMerged, "o/b", 3))) + if err := saveSaaSUser(&SaaSUser{GitHubUsername: "alice", Hives: map[string]string{"hive-a": "viewer"}}); err != nil { + t.Fatal(err) + } + mkUser(t, "mallory") + mkUser(t, "root") + + rec := getActivity(t, s, "alice", "", "") + if got := viewEventKeys(decodeActivity(t, rec)); len(got) != 1 || got[0] != "hive-a:o/a#1:opened" { + t.Fatalf("alice events = %v, want only hive-a's", got) + } + if strings.Contains(rec.Body.String(), "o/b") { + t.Fatalf("hive-b leaked to alice: %s", rec.Body.String()) + } + aliceETag := rec.Header().Get("ETag") + + // Movement on a hive alice cannot see must not even change her etag. + postActivityHeartbeat(t, s, "hive-b", withEvents(activityDigest("o/b", 3), activityEv(2, hubactivity.EventStage, "o/b", 3))) + if again := getActivity(t, s, "alice", "", aliceETag); again.Code != http.StatusNotModified { + t.Fatalf("hive-b movement changed alice's view: status %d", again.Code) + } + + if v := decodeActivity(t, getActivity(t, s, "mallory", "", "")); len(v.Events) != 0 { + t.Fatalf("ungranted user saw events %v", viewEventKeys(v)) + } + if got := viewEventKeys(decodeActivity(t, getActivity(t, s, "root", "", ""))); len(got) != 3 { + t.Fatalf("admin events = %v, want all three", got) + } + + // Filters apply to events too. + if got := viewEventKeys(decodeActivity(t, getActivity(t, s, "root", "?hive=hive-b", ""))); len(got) != 2 { + t.Fatalf("hive filter events = %v, want hive-b's two", got) + } + if got := viewEventKeys(decodeActivity(t, getActivity(t, s, "root", "?mod=stalled", ""))); len(got) != 0 { + t.Fatalf("modifier filter must drop events of items without it: %v", got) + } +} + +// Private items the spoke does not share never become events (the spoke's +// exclusion, tested in pkg/hub/spoke); an event the hub receives for a hive +// it then forgets is purged with it. +func TestHubActivityEvents_ForgetPurgesEvents(t *testing.T) { + var st activityStore + st.record("hive-a", withEvents(activityDigest("o/a", 1), activityEv(1, hubactivity.EventOpened, "o/a", 1)), activityT0) + st.record("hive-b", withEvents(activityDigest("o/b", 1), activityEv(1, hubactivity.EventOpened, "o/b", 1)), activityT0) + + st.forget("hive-a") + + snap := st.eventsSnapshot([]string{"hive-a", "hive-b"}) + if len(snap.events) != 1 || snap.events[0].hiveID != "hive-b" { + t.Fatalf("events after forget = %+v", snap.events) + } + if st.eventsAck("hive-a") != 0 { + t.Fatal("a forgotten hive must lose its event acknowledgement too") + } +} + +func TestActivityEventRing_DedupesByAcknowledgedSeq(t *testing.T) { + var st activityStore + batch := withEvents(activityDigest("o/a", 1), activityEv(10, hubactivity.EventOpened, "o/a", 1), activityEv(11, hubactivity.EventStage, "o/a", 1)) + batch.EventsDropped = 3 + + st.record("hive-a", batch, activityT0) + // The spoke lost the ack and resends the same batch plus one new event; + // its cumulative disclosure restates the 3 drops already counted. + resend := withEvents(batch, append(batch.Events, activityEv(12, hubactivity.EventMerged, "o/a", 1))...) + st.record("hive-a", resend, activityT0.Add(time.Minute)) + // A batch with nothing new does not count its drops again. + st.record("hive-a", batch, activityT0.Add(2*time.Minute)) + + snap := st.eventsSnapshot([]string{"hive-a"}) + if len(snap.events) != 3 { + t.Fatalf("stored %d events, want 3 (no duplicates)", len(snap.events)) + } + if got := st.eventsAck("hive-a"); got != 12 { + t.Fatalf("ack = %d, want 12", got) + } + if snap.dropped["hive-a"] != 3 { + t.Fatalf("disclosed drops = %d, want 3 (a resend restates, not repeats, them)", snap.dropped["hive-a"]) + } + + // A resend whose disclosure grew counts only the increase. + grown := withEvents(resend, append(resend.Events, activityEv(13, hubactivity.EventStage, "o/a", 1))...) + grown.EventsDropped = 5 + st.record("hive-a", grown, activityT0.Add(3*time.Minute)) + if got := st.eventsSnapshot([]string{"hive-a"}).dropped["hive-a"]; got != 5 { + t.Fatalf("drops after grown resend = %d, want 5", got) + } + + // After the ack lands the spoke trims and resets its disclosure, so a + // batch of only new events discloses only new drops, counted in full. + acked := withEvents(activityDigest("o/a", 1), activityEv(14, hubactivity.EventStage, "o/a", 1)) + acked.EventsDropped = 2 + st.record("hive-a", acked, activityT0.Add(4*time.Minute)) + if got := st.eventsSnapshot([]string{"hive-a"}).dropped["hive-a"]; got != 7 { + t.Fatalf("drops after acked batch = %d, want 7", got) + } +} + +// Known limit, pinned so a change is deliberate: a lost ack followed by a +// full buffer turnover (more than hubactivity.MaxEvents new events) leaves +// no repeated Seq in the resend, so the hub cannot see that the spoke's +// cumulative disclosure restates drops already counted and counts the +// earlier batch's drops once more. The over-count is bounded by that one +// batch's disclosure; stored events are never duplicated. +func TestActivityEventRing_RestatedDropsAfterFullTurnoverOverCountOnce(t *testing.T) { + var st activityStore + first := withEvents(activityDigest("o/a", 1), activityEv(1, hubactivity.EventOpened, "o/a", 1), activityEv(2, hubactivity.EventStage, "o/a", 1)) + first.EventsDropped = 3 + st.record("hive-a", first, activityT0) + + // The ack is lost and the spoke buffers MaxEvents newer events, so the + // resend holds only Seq 3.. and its disclosure 3 (old) + 2 (new) = 5. + turned := make([]hubactivity.Event, 0, hubactivity.MaxEvents) + for i := range hubactivity.MaxEvents { + turned = append(turned, activityEv(uint64(3+i), hubactivity.EventStage, "o/a", 1)) + } + resend := withEvents(first, turned...) + resend.EventsDropped = 5 + st.record("hive-a", resend, activityT0.Add(time.Minute)) + + snap := st.eventsSnapshot([]string{"hive-a"}) + if len(snap.events) != 2+hubactivity.MaxEvents { + t.Fatalf("stored %d events, want %d (no duplicates)", len(snap.events), 2+hubactivity.MaxEvents) + } + if got := st.eventsAck("hive-a"); got != uint64(2+hubactivity.MaxEvents) { + t.Fatalf("ack = %d, want %d", got, 2+hubactivity.MaxEvents) + } + // Exact accounting would be 5; the undetectable restatement adds the + // first batch's 3 once more, and never more than that. + if snap.dropped["hive-a"] != 8 { + t.Fatalf("disclosed drops = %d, want 8 (5 exact + the 3 restated once, undetectable without a repeated Seq)", snap.dropped["hive-a"]) + } + + // A further resend that does repeat a Seq is detected again and only + // the increase over the last counted disclosure is added. + again := withEvents(resend, turned[len(turned)-1], activityEv(uint64(3+hubactivity.MaxEvents), hubactivity.EventMerged, "o/a", 1)) + again.EventsDropped = 6 + st.record("hive-a", again, activityT0.Add(2*time.Minute)) + if got := st.eventsSnapshot([]string{"hive-a"}).dropped["hive-a"]; got != 9 { + t.Fatalf("drops after detected resend = %d, want 9", got) + } +} + +// The fleet-wide ring keeps the last 1,000 events across all hives, oldest +// dropped first; each block is also capped at hubactivity.MaxEvents with +// the overflow disclosed. +func TestActivityEventRing_FleetCapAndPerBlockCap(t *testing.T) { + var st activityStore + seq := uint64(0) + batch := func(repo string, n int) []hubactivity.Event { + out := make([]hubactivity.Event, 0, n) + for range n { + seq++ + out = append(out, activityEv(seq, hubactivity.EventStage, repo, int(seq))) + } + return out + } + + // 250 events in one block: the hub keeps the newest 200 and discloses 50. + st.record("hive-a", withEvents(activityDigest("o/a", 1), batch("o/a", 250)...), activityT0) + snap := st.eventsSnapshot([]string{"hive-a"}) + if len(snap.events) != hubactivity.MaxEvents || snap.events[0].event.Seq != 51 || snap.dropped["hive-a"] != 50 { + t.Fatalf("per-block cap: %d events from seq %d, dropped %d; want 200 from 51, 50 dropped", len(snap.events), snap.events[0].event.Seq, snap.dropped["hive-a"]) + } + + // Five more full blocks across two hives overflow the fleet ring. + for i := range 5 { + hive, repo := "hive-a", "o/a" + if i%2 == 1 { + hive, repo = "hive-b", "o/b" + } + st.record(hive, withEvents(activityDigest(repo, 1), batch(repo, 200)...), activityT0.Add(time.Duration(i+1)*time.Minute)) + } + all := st.eventsSnapshot([]string{"hive-a", "hive-b"}) + if len(all.events) != activityEventRingSize { + t.Fatalf("ring holds %d events, want %d", len(all.events), activityEventRingSize) + } + if first, last := all.events[0].event.Seq, all.events[len(all.events)-1].event.Seq; first != seq-activityEventRingSize+1 || last != seq { + t.Fatalf("ring spans seq %d..%d, want the newest %d ending at %d", first, last, activityEventRingSize, seq) + } +} + +func TestActivityEventRing_DropsMalformedEvents(t *testing.T) { + var st activityStore + st.record("hive-a", withEvents(activityDigest("o/a", 1), + hubactivity.Event{Seq: 1, Type: "exploded", Repo: "o/a", Number: 1}, + hubactivity.Event{Seq: 2, Type: hubactivity.EventMerged, Repo: "not a repo", Number: 1}, + hubactivity.Event{Seq: 0, Type: hubactivity.EventMerged, Repo: "o/a", Number: 1}, + hubactivity.Event{Seq: 3, Type: hubactivity.EventMerged, Repo: "o/a", Number: 1, Title: strings.Repeat("x", 500)}, + ), activityT0) + + snap := st.eventsSnapshot([]string{"hive-a"}) + if len(snap.events) != 1 || snap.events[0].event.Seq != 3 { + t.Fatalf("events = %+v, want only the valid merge", snap.events) + } + if n := len([]rune(snap.events[0].event.Title)); n != hubactivity.MaxTitleRunes { + t.Fatalf("title runes = %d, want bounded to %d", n, hubactivity.MaxTitleRunes) + } +} diff --git a/src/pkg/hub/activity_update.go b/src/pkg/hub/activity_update.go index 11f5ed7e49..31da1e7761 100644 --- a/src/pkg/hub/activity_update.go +++ b/src/pkg/hub/activity_update.go @@ -76,6 +76,7 @@ func (s *HubServer) handleActivityUpdate(w http.ResponseWriter, r *http.Request) } resp := HeartbeatResponse{OK: true} resp.ActivityETag = s.activity.record(hiveID, payload.Update.Activity, time.Now().UTC()) + resp.ActivityEventsAck = s.activity.eventsAck(hiveID) w.Header().Set("Content-Type", "application/json") s.signHeartbeatResponse(w, &resp, hiveID) data, _ := json.Marshal(resp) diff --git a/src/pkg/hub/heartbeat.go b/src/pkg/hub/heartbeat.go index 9f6ce240af..1545db6c2d 100644 --- a/src/pkg/hub/heartbeat.go +++ b/src/pkg/hub/heartbeat.go @@ -476,6 +476,10 @@ type HeartbeatResponse struct { // this hive ("" = none). A spoke whose etag differs sends its full // digest instead of an unchanged marker on the next beat. ActivityETag string `json:"activity_etag,omitempty"` + // ActivityEventsAck is the highest activity event sequence number the + // hub holds for this hive (#15964); the spoke drops buffered events up + // to it. 0/absent = none. Decoded by spoke.HeartbeatResponse. + ActivityEventsAck uint64 `json:"activity_events_ack,omitempty"` } // HubBanner is a message from the hub admin displayed on spoke dashboards. diff --git a/src/pkg/hub/server.go b/src/pkg/hub/server.go index 0d2686b9ed..64eb5abe4c 100644 --- a/src/pkg/hub/server.go +++ b/src/pkg/hub/server.go @@ -2824,6 +2824,7 @@ func (s *HubServer) handleHeartbeat(w http.ResponseWriter, r *http.Request) { } resp.SelfExperimentFlag = s.selfExperimentFlagForHeartbeat(payload.HiveID, payload.SelfExperimentsOptIn) resp.ActivityETag = activityAck + resp.ActivityEventsAck = s.activity.eventsAck(payload.HiveID) if entry.IsPublic != payload.IsPublic { resp.IsPublic = &entry.IsPublic diff --git a/src/pkg/hub/spoke/activity_events_test.go b/src/pkg/hub/spoke/activity_events_test.go new file mode 100644 index 0000000000..ba501cb00a --- /dev/null +++ b/src/pkg/hub/spoke/activity_events_test.go @@ -0,0 +1,265 @@ +package spoke + +import ( + "context" + "fmt" + "sync" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/hubactivity" +) + +// activitySnapshotBox is a tracker snapshot (items and merges) a test can +// swap. +type activitySnapshotBox struct { + mu sync.Mutex + snap hubactivity.Snapshot +} + +func (b *activitySnapshotBox) Set(items []hubactivity.Item, merged ...hubactivity.Merge) { + b.mu.Lock() + b.snap = hubactivity.Snapshot{Items: items, Merged: merged} + b.mu.Unlock() +} + +func (b *activitySnapshotBox) Snapshot(context.Context) (hubactivity.Snapshot, bool) { + b.mu.Lock() + defer b.mu.Unlock() + return hubactivity.Snapshot{Items: append([]hubactivity.Item(nil), b.snap.Items...), Merged: append([]hubactivity.Merge(nil), b.snap.Merged...)}, true +} + +func testEventPublisher(opts ActivityOptions, box *activitySnapshotBox, clock *activityClock) *activityPublisher { + p := newActivityPublisher(ActivitySource{ + Options: func() ActivityOptions { return opts }, + Snapshot: box.Snapshot, + }, "", nil2Logger()) + p.now = clock.Now + return p +} + +func stagedItem(repo string, n int, stage string) hubactivity.Item { + return hubactivity.Item{Repo: repo, Number: n, Kind: "pr", Title: fmt.Sprintf("%s #%d", repo, n), Stage: stage, StageNumber: 10} +} + +func eventKeys(events []hubactivity.Event) []string { + out := make([]string, 0, len(events)) + for _, e := range events { + out = append(out, fmt.Sprintf("%s#%d:%s:%s>%s", e.Repo, e.Number, e.Type, e.From, e.Stage)) + } + return out +} + +func sameKeys(got, want []string) bool { + return fmt.Sprint(got) == fmt.Sprint(want) +} + +// Acceptance (#15964): a merge on the spoke rides on the very next +// heartbeat's activity block, alongside the stage hops and opens seen +// since the previous one. The first snapshot only seeds the baseline. +func TestActivityEventsStageHopsOpensAndMergeOnTheNextBeat(t *testing.T) { + box := &activitySnapshotBox{} + box.Set([]hubactivity.Item{stagedItem("o/r", 1, "ci_review"), stagedItem("o/r", 2, "ci_review")}) + clock := &activityClock{now: activityTestT0} + p := testEventPublisher(ActivityOptions{Enabled: true}, box, clock) + ctx := context.Background() + + if seed := p.heartbeatDigestLocked(ctx); len(seed.Events) != 0 { + t.Fatalf("first beat replayed the board as events: %v", eventKeys(seed.Events)) + } + + mergedAt := activityTestT0.Add(time.Minute) + box.Set([]hubactivity.Item{stagedItem("o/r", 1, "merge_queue"), stagedItem("o/r", 3, "actionable")}, + hubactivity.Merge{Repo: "o/r", Number: 2, MergedAt: mergedAt}) + clock.Advance(2 * time.Minute) + d := p.heartbeatDigestLocked(ctx) + + want := []string{"o/r#1:stage:ci_review>merge_queue", "o/r#3:opened:>actionable", "o/r#2:merged:ci_review>"} + if got := eventKeys(d.Events); !sameKeys(got, want) { + t.Fatalf("events = %v, want %v", got, want) + } + merge := d.Events[2] + if !merge.At.Equal(mergedAt) || merge.Title != "o/r #2" { + t.Fatalf("merge event = %+v, want the merge time and the item's last title", merge) + } + for i := 1; i < len(d.Events); i++ { + if d.Events[i].Seq <= d.Events[i-1].Seq { + t.Fatalf("sequence numbers not increasing: %v", d.Events) + } + } + + // The merge stays in the tracker's recent list: no second event. + clock.Advance(2 * time.Minute) + p.acknowledgeEventsLocked(d.Events[len(d.Events)-1].Seq) + if again := p.heartbeatDigestLocked(ctx); len(again.Events) != 0 { + t.Fatalf("an already reported merge was reported again: %v", eventKeys(again.Events)) + } +} + +// Events stay buffered (and are resent) until the hub acknowledges them; +// an acknowledgement drops exactly the events up to its sequence number. +func TestActivityEventsTrimmedOnlyByAcknowledgement(t *testing.T) { + box := &activitySnapshotBox{} + box.Set(nil) + p := testEventPublisher(ActivityOptions{Enabled: true}, box, &activityClock{now: activityTestT0}) + ctx := context.Background() + p.heartbeatDigestLocked(ctx) + + box.Set([]hubactivity.Item{stagedItem("o/r", 1, "a")}) + first := p.heartbeatDigestLocked(ctx) + box.Set([]hubactivity.Item{stagedItem("o/r", 1, "a"), stagedItem("o/r", 2, "a")}) + second := p.heartbeatDigestLocked(ctx) + if len(first.Events) != 1 || len(second.Events) != 2 { + t.Fatalf("unacknowledged events must be resent: %d then %d", len(first.Events), len(second.Events)) + } + + // An accepted response without an events ack (a hub before #15964, + // or one that restarted) keeps the buffer. + p.acknowledgeLocked(second.ETag) + p.acknowledgeEventsLocked(0) + marker := p.heartbeatDigestLocked(ctx) + if !marker.IsUnchangedMarker() || len(marker.Events) != 2 { + t.Fatalf("marker = %+v, want the etag only plus both pending events", marker) + } + + p.acknowledgeEventsLocked(first.Events[0].Seq) + if rest := p.heartbeatDigestLocked(ctx); len(rest.Events) != 1 || rest.Events[0].Number != 2 { + t.Fatalf("after a partial ack events = %v, want only #2", eventKeys(rest.Events)) + } + p.acknowledgeEventsLocked(second.Events[1].Seq) + if done := p.heartbeatDigestLocked(ctx); len(done.Events) != 0 { + t.Fatalf("after a full ack events = %v, want none", eventKeys(done.Events)) + } +} + +// The buffer holds at most hubactivity.MaxEvents: the oldest go first and +// the drop count is disclosed until an acknowledgement covers it. +func TestActivityEventsCapAt200DisclosesDrops(t *testing.T) { + box := &activitySnapshotBox{} + box.Set(nil) + p := testEventPublisher(ActivityOptions{Enabled: true, MaxItems: 1}, box, &activityClock{now: activityTestT0}) + ctx := context.Background() + p.heartbeatDigestLocked(ctx) + + var items []hubactivity.Item + for n := 1; n <= 250; n++ { + items = append(items, stagedItem("o/r", n, "a")) + } + box.Set(items) + d := p.heartbeatDigestLocked(ctx) + + if len(d.Events) != hubactivity.MaxEvents || d.EventsDropped != 50 { + t.Fatalf("events = %d dropped = %d, want %d and 50", len(d.Events), d.EventsDropped, hubactivity.MaxEvents) + } + if d.Events[0].Number != 51 || d.Events[len(d.Events)-1].Number != 250 { + t.Fatalf("kept #%d..#%d, want the newest (#51..#250)", d.Events[0].Number, d.Events[len(d.Events)-1].Number) + } + + p.acknowledgeEventsLocked(d.Events[len(d.Events)-1].Seq) + if after := p.heartbeatDigestLocked(ctx); len(after.Events) != 0 || after.EventsDropped != 0 { + t.Fatalf("after the ack: %d events, %d dropped; want both cleared", len(after.Events), after.EventsDropped) + } +} + +// Private-repository items yield no events unless the spoke shares them. +func TestActivityEventsExcludePrivateItemsUnlessShared(t *testing.T) { + for _, share := range []bool{false, true} { + box := &activitySnapshotBox{} + box.Set(nil) + p := testEventPublisher(ActivityOptions{Enabled: true, SharePrivate: share}, box, &activityClock{now: activityTestT0}) + ctx := context.Background() + p.heartbeatDigestLocked(ctx) + + secret := stagedItem("o/secret", 9, "a") + secret.Private = true + box.Set([]hubactivity.Item{secret, stagedItem("o/public", 1, "a")}, + hubactivity.Merge{Repo: "o/secret", Number: 8, MergedAt: activityTestT0, Private: true}) + d := p.heartbeatDigestLocked(ctx) + + var private int + for _, e := range d.Events { + if e.Repo == "o/secret" { + private++ + if !e.Private { + t.Fatalf("shared private event not marked private: %+v", e) + } + } + } + if want := map[bool]int{false: 0, true: 2}[share]; private != want { + t.Fatalf("share_private=%v: %d private events, want %d (%v)", share, private, want, eventKeys(d.Events)) + } + } +} + +// Turning publishing off forgets the buffer, so turning it back on starts +// from a fresh baseline instead of replaying stale movement. +func TestActivityEventsResetWhenDisabled(t *testing.T) { + opts := ActivityOptions{Enabled: true} + box := &activitySnapshotBox{} + box.Set(nil) + p := newActivityPublisher(ActivitySource{Options: func() ActivityOptions { return opts }, Snapshot: box.Snapshot}, "", nil2Logger()) + ctx := context.Background() + p.heartbeatDigestLocked(ctx) + box.Set([]hubactivity.Item{stagedItem("o/r", 1, "a")}) + p.heartbeatDigestLocked(ctx) + + opts.Enabled = false + if d := p.heartbeatDigestLocked(ctx); d != nil { + t.Fatalf("disabled publisher sent %+v", d) + } + opts.Enabled = true + box.Set([]hubactivity.Item{stagedItem("o/r", 1, "a"), stagedItem("o/r", 2, "a")}) + if d := p.heartbeatDigestLocked(ctx); len(d.Events) != 0 { + t.Fatalf("re-enabled publisher replayed %v", eventKeys(d.Events)) + } +} + +func TestRedactActivityAppliesToEvents(t *testing.T) { + d := hubactivity.Digest{Events: []hubactivity.Event{{Seq: 1, Type: hubactivity.EventMerged, Repo: "o/r", Number: 1, Title: "secret"}}, EventsDropped: 4} + + repos := redactActivity(d, map[string]bool{config.HeartbeatOmitRepos: true}) + if len(repos.Events) != 0 || repos.EventsDropped != 0 { + t.Fatalf("omit repos left events: %+v", repos) + } + titles := redactActivity(d, map[string]bool{config.HeartbeatOmitTaskTitles: true}) + if len(titles.Events) != 1 || titles.Events[0].Title != "" { + t.Fatalf("omit task titles left a title: %+v", titles.Events) + } + if d.Events[0].Title != "secret" { + t.Fatal("redaction modified the input") + } +} + +// An out-of-band update carries new events even when the digest itself is +// unchanged, as an unchanged marker, and applies the event ack. +func TestActivityUpdateSendsPendingEventsOnAMarker(t *testing.T) { + box := &activitySnapshotBox{} + box.Set([]hubactivity.Item{stagedItem("o/r", 1, "a")}) + clock := &activityClock{now: activityTestT0} + p := testEventPublisher(ActivityOptions{Enabled: true}, box, clock) + rec := &recordingPost{sent: make(chan *hubactivity.Digest, 4), status: 200, ok: true} + p.post = rec.post + ctx := context.Background() + + if !p.update(ctx) { + t.Fatal("first update not sent") + } + <-rec.sent + if p.update(ctx) { + t.Fatal("nothing changed: no update expected") + } + + // A merge of an item no longer tracked leaves the digest unchanged. + box.Set([]hubactivity.Item{stagedItem("o/r", 1, "a")}, hubactivity.Merge{Repo: "o/r", Number: 5, MergedAt: activityTestT0}) + if !p.update(ctx) { + t.Fatal("pending merge event not sent") + } + sent := <-rec.sent + if !sent.IsUnchangedMarker() || len(sent.Events) != 1 || sent.Events[0].Type != hubactivity.EventMerged { + t.Fatalf("update = %+v, want a marker carrying the merge", sent) + } + if len(p.events) != 0 { + t.Fatalf("acknowledged events still buffered: %v", eventKeys(p.events)) + } +} diff --git a/src/pkg/hub/spoke/activity_publisher.go b/src/pkg/hub/spoke/activity_publisher.go index 4e2ed4ec6e..40fdf1023c 100644 --- a/src/pkg/hub/spoke/activity_publisher.go +++ b/src/pkg/hub/spoke/activity_publisher.go @@ -8,6 +8,8 @@ import ( "log/slog" "net/http" "slices" + "strconv" + "strings" "sync" "sync/atomic" "time" @@ -29,6 +31,11 @@ import ( // activityUpdateMinInterval. Publishes that arrive while one is waiting // or in flight merge into it; the rest are dropped (the next heartbeat // carries the digest anyway). +// +// Both also carry the activity events (#15964): stage hops, opens and +// merges the publisher derives by diffing successive tracker snapshots, +// buffered (at most hubactivity.MaxEvents, oldest dropped first, the drop +// count disclosed) until the hub acknowledges them by sequence number. // ActivityUpdateHeader marks an activity-only update on POST /api/heartbeat. const ActivityUpdateHeader = "X-Hive-Activity-Update" @@ -57,11 +64,14 @@ type ActivityOptions struct { // ActivitySource is the StartHeartbeat callback that feeds the activity // digest. Options is read on every build so a config reload applies to the // next beat. Items returns the tracked items with Private set for -// private-repository rows (ok=false: no tracker yet). Subscribe, optional, -// registers a non-blocking hook run on every tracker publish. +// private-repository rows (ok=false: no tracker yet). Snapshot, optional, +// supersedes Items and also reports recently merged pull requests, which +// the publisher turns into merge events. Subscribe, optional, registers a +// non-blocking hook run on every tracker publish. type ActivitySource struct { Options func() ActivityOptions Items func(ctx context.Context) (items []hubactivity.Item, ok bool) + Snapshot func(ctx context.Context) (snap hubactivity.Snapshot, ok bool) Subscribe func(notify func()) } @@ -79,9 +89,16 @@ type ActivityUpdate struct { Activity *hubactivity.Digest `json:"activity"` } +// activityAck is what an accepted response acknowledged: the digest etag +// the hub holds and the highest event sequence number it holds. +type activityAck struct { + ETag string + Events uint64 +} + // activityPostFunc sends one activity-only update. ok is true only for an // accepted, verified response; status is the HTTP status (0 = no response). -type activityPostFunc func(ctx context.Context, d *hubactivity.Digest) (ack string, status int, ok bool) +type activityPostFunc func(ctx context.Context, d *hubactivity.Digest) (ack activityAck, status int, ok bool) type activityPublisher struct { src ActivitySource @@ -99,6 +116,18 @@ type activityPublisher struct { acked string lastUpdate time.Time backoff time.Time + + // Event state (#15964), also guarded by mu. prevItems is nil until the + // first snapshot seeds it, so startup never replays the board as opens. + events []hubactivity.Event + eventsDropped int + // sentDropped and sentMaxSeq are what the latest build carried, so an + // acknowledgement covering it clears the disclosed drops. + sentDropped int + sentMaxSeq uint64 + nextSeq uint64 + prevItems map[string]hubactivity.Item + prevMerges map[string]bool } // activityPub is the running heartbeat loop's publisher (nil when no @@ -115,31 +144,56 @@ func newActivityPublisher(src ActivitySource, hubURL string, logger *slog.Logger now: time.Now, after: time.After, kick: make(chan struct{}, 1), + // Sequence numbers start from the wall clock so a restarted spoke + // numbers past what the hub acknowledged from its previous run. + nextSeq: uint64(max(time.Now().UnixMilli(), 1)) * 1000, } - p.post = func(ctx context.Context, d *hubactivity.Digest) (string, int, bool) { - return postActivityUpdate(ctx, hubURL, heartbeatHiveID(), d, logger) + p.post = func(ctx context.Context, d *hubactivity.Digest) (activityAck, int, bool) { + return postActivityUpdateAck(ctx, hubURL, heartbeatHiveID(), d, logger) } return p } // build returns the current digest, redacted for hub.heartbeat_omit. // ok is false when publishing is off or the tracker has nothing to give. +// It also records the events since the previous build and attaches the +// unacknowledged ones. Callers hold p.mu. func (p *activityPublisher) build(ctx context.Context) (hubactivity.Digest, bool) { - if p.src.Options == nil || p.src.Items == nil { + if p.src.Options == nil || (p.src.Items == nil && p.src.Snapshot == nil) { return hubactivity.Digest{}, false } opts := p.src.Options() if !opts.Enabled { + p.resetEventsLocked() return hubactivity.Digest{}, false } - items, ok := activityItemsWithTimeout(ctx, p.src.Items, activityItemsTimeout, p.logger) + snap, ok := p.readSnapshot(ctx) if !ok { return hubactivity.Digest{}, false } - d := hubactivity.Build(items, hubactivity.BuildOptions{MaxItems: opts.MaxItems, SharePrivate: opts.SharePrivate, Now: p.now()}) + now := p.now() + p.observeLocked(snap, opts.SharePrivate, now) + d := hubactivity.Build(snap.Items, hubactivity.BuildOptions{MaxItems: opts.MaxItems, SharePrivate: opts.SharePrivate, Now: now}) + d.Events = slices.Clone(p.events) + d.EventsDropped = p.eventsDropped + p.sentDropped = p.eventsDropped + p.sentMaxSeq = 0 + if n := len(p.events); n > 0 { + p.sentMaxSeq = p.events[n-1].Seq + } return redactActivity(d, omitSet()), true } +// readSnapshot reads the tracker through Snapshot when set, else Items, +// bounded by activityItemsTimeout. +func (p *activityPublisher) readSnapshot(ctx context.Context) (hubactivity.Snapshot, bool) { + if p.src.Snapshot != nil { + return callWithTimeout(ctx, p.src.Snapshot, activityItemsTimeout, p.logger) + } + items, ok := activityItemsWithTimeout(ctx, p.src.Items, activityItemsTimeout, p.logger) + return hubactivity.Snapshot{Items: items}, ok +} + // heartbeatDigestLocked is the activity block for a heartbeat: nil when // there is nothing to publish, an unchanged marker when the hub already // holds this etag, else the full digest. Callers hold p.mu. @@ -149,17 +203,119 @@ func (p *activityPublisher) heartbeatDigestLocked(ctx context.Context) *hubactiv return nil } if d.ETag != "" && d.ETag == p.acked { - return &hubactivity.Digest{ETag: d.ETag, GeneratedAt: d.GeneratedAt} + return unchangedMarker(d) } return &d } +// unchangedMarker is d reduced to its etag and generated_at, keeping the +// events it carries. +func unchangedMarker(d hubactivity.Digest) *hubactivity.Digest { + return &hubactivity.Digest{ETag: d.ETag, GeneratedAt: d.GeneratedAt, Events: d.Events, EventsDropped: d.EventsDropped} +} + // acknowledgeLocked records the etag the hub reported holding. Callers hold // p.mu and call it only for an accepted, verified response. func (p *activityPublisher) acknowledgeLocked(etag string) { p.acked = etag } +// acknowledgeEventsLocked drops the buffered events up to the sequence +// number the hub reported holding (0 = none, e.g. a hub before #15964 +// or one that restarted: the buffer is kept and resent). Once the ack +// covers the latest build, the drops it disclosed are cleared. Callers +// hold p.mu and call it only for an accepted, verified response. +func (p *activityPublisher) acknowledgeEventsLocked(seq uint64) { + if seq == 0 { + return + } + p.events = slices.DeleteFunc(p.events, func(e hubactivity.Event) bool { return e.Seq <= seq }) + if seq >= p.sentMaxSeq { + p.eventsDropped = max(p.eventsDropped-p.sentDropped, 0) + p.sentDropped = 0 + } +} + +// resetEventsLocked forgets all event state, so re-enabling publishing +// starts from a fresh baseline instead of replaying stale movement. +func (p *activityPublisher) resetEventsLocked() { + p.events, p.eventsDropped, p.sentDropped, p.sentMaxSeq = nil, 0, 0, 0 + p.prevItems, p.prevMerges = nil, nil +} + +// observeLocked diffs snap against the previous snapshot and buffers the +// stage hops, opens and merges it finds. Private-repository items yield no +// events unless sharePrivate. The first snapshot only seeds the baseline. +func (p *activityPublisher) observeLocked(snap hubactivity.Snapshot, sharePrivate bool, now time.Time) { + cur := make(map[string]hubactivity.Item, len(snap.Items)) + for _, it := range snap.Items { + cur[activityEventKey(it.Repo, it.Number)] = it + } + merges := make(map[string]bool, len(snap.Merged)) + for _, m := range snap.Merged { + merges[activityEventKey(m.Repo, m.Number)] = true + } + seeded := p.prevItems != nil + prev, prevMerges := p.prevItems, p.prevMerges + p.prevItems, p.prevMerges = cur, merges + if !seeded { + return + } + for _, it := range snap.Items { + if it.Private && !sharePrivate { + continue + } + e := hubactivity.Event{Repo: it.Repo, Number: it.Number, Kind: it.Kind, Title: it.Title, Stage: it.Stage, At: now, Private: it.Private} + was, had := prev[activityEventKey(it.Repo, it.Number)] + switch { + case !had: + e.Type = hubactivity.EventOpened + case was.Stage != it.Stage && it.Stage != "": + e.Type = hubactivity.EventStage + e.From = was.Stage + default: + continue + } + p.pushEventLocked(e) + } + for _, m := range snap.Merged { + key := activityEventKey(m.Repo, m.Number) + if prevMerges[key] || (m.Private && !sharePrivate) { + continue + } + e := hubactivity.Event{Type: hubactivity.EventMerged, Repo: m.Repo, Number: m.Number, Kind: "pr", At: m.MergedAt, Private: m.Private} + if e.At.IsZero() { + e.At = now + } + if was, ok := prev[key]; ok { + e.Title, e.From = was.Title, was.Stage + } else if it, ok := cur[key]; ok { + e.Title, e.From = it.Title, it.Stage + } + p.pushEventLocked(e) + } +} + +// pushEventLocked numbers and buffers one event, dropping the oldest past +// hubactivity.MaxEvents and counting the drop. +func (p *activityPublisher) pushEventLocked(e hubactivity.Event) { + e.Seq = p.nextSeq + p.nextSeq++ + n, ok := hubactivity.NormalizeEvent(e) + if !ok { + return + } + p.events = append(p.events, n) + if over := len(p.events) - hubactivity.MaxEvents; over > 0 { + p.events = slices.Delete(p.events, 0, over) + p.eventsDropped += over + } +} + +func activityEventKey(repo string, n int) string { + return strings.ToLower(repo) + "#" + strconv.Itoa(n) +} + // notify signals a tracker publish without blocking: a publish arriving // while one is already pending merges into it. func (p *activityPublisher) notify() { @@ -206,18 +362,24 @@ func (p *activityPublisher) update(ctx context.Context) bool { return false } d, ok := p.build(ctx) - if !ok || d.ETag == p.acked { + if !ok || (d.ETag == p.acked && len(d.Events) == 0) { return false } + send := &d + if d.ETag != "" && d.ETag == p.acked { + // Only events are new: carry them on an unchanged marker. + send = unchangedMarker(d) + } p.lastUpdate = now - ack, status, sent := p.post(ctx, &d) + ack, status, sent := p.post(ctx, send) if !sent { if activityUpdateUnsupported(status) { p.backoff = now.Add(activityUpdateBackoff) } return false } - p.acknowledgeLocked(ack) + p.acknowledgeLocked(ack.ETag) + p.acknowledgeEventsLocked(ack.Events) return true } @@ -240,12 +402,19 @@ func redactActivity(d hubactivity.Digest, omit map[string]bool) hubactivity.Dige case omit[config.HeartbeatOmitRepos]: d.Items = nil d.Truncated = 0 + d.Events = nil + d.EventsDropped = 0 case omit[config.HeartbeatOmitTaskTitles]: items := slices.Clone(d.Items) for i := range items { items[i].Title = "" } d.Items = items + events := slices.Clone(d.Events) + for i := range events { + events[i].Title = "" + } + d.Events = events default: return d } @@ -256,29 +425,36 @@ func redactActivity(d hubactivity.Digest, omit map[string]bool) hubactivity.Dige // activityItemsWithTimeout runs items with an upper bound, recovering a // panic, so the tracker can never stall or crash the heartbeat loop. func activityItemsWithTimeout(ctx context.Context, items func(context.Context) ([]hubactivity.Item, bool), timeout time.Duration, logger *slog.Logger) ([]hubactivity.Item, bool) { - type result struct { - items []hubactivity.Item - ok bool - } + return callWithTimeout(ctx, items, timeout, logger) +} + +type timedRead[T any] struct { + items T + ok bool +} + +// callWithTimeout is activityItemsWithTimeout for any tracker read. +func callWithTimeout[T any](ctx context.Context, read func(context.Context) (T, bool), timeout time.Duration, logger *slog.Logger) (T, bool) { cctx, cancel := context.WithTimeout(ctx, timeout) defer cancel() - done := make(chan result, 1) + done := make(chan timedRead[T], 1) go func() { defer func() { if r := recover(); r != nil { logger.Warn("hub activity digest: tracker read panicked; sending no activity block", "recover", r) - done <- result{} + done <- timedRead[T]{} } }() - its, ok := items(cctx) - done <- result{items: its, ok: ok} + its, ok := read(cctx) + done <- timedRead[T]{items: its, ok: ok} }() select { case r := <-done: return r.items, r.ok case <-cctx.Done(): logger.Warn("hub activity digest: tracker read timed out; sending no activity block", "timeout", timeout.String()) - return nil, false + var zero T + return zero, false } } @@ -296,18 +472,25 @@ func heartbeatHiveID() string { // bearer and verifies the hub's signed acknowledgement the same way as a // heartbeat response. func postActivityUpdate(ctx context.Context, hubURL, hiveID string, d *hubactivity.Digest, logger *slog.Logger) (string, int, bool) { + ack, status, ok := postActivityUpdateAck(ctx, hubURL, hiveID, d, logger) + return ack.ETag, status, ok +} + +// postActivityUpdateAck is postActivityUpdate also returning the event +// acknowledgement. +func postActivityUpdateAck(ctx context.Context, hubURL, hiveID string, d *hubactivity.Digest, logger *slog.Logger) (activityAck, int, bool) { if hubURL == "" || hiveID == "" || d == nil { - return "", 0, false + return activityAck{}, 0, false } body, err := json.Marshal(ActivityUpdatePayload{Update: &ActivityUpdate{HiveID: hiveID, Activity: d}}) if err != nil { - return "", 0, false + return activityAck{}, 0, false } reqCtx, cancel := context.WithTimeout(ctx, heartbeatTimeout) defer cancel() req, err := http.NewRequestWithContext(reqCtx, http.MethodPost, hubURL+"/api/heartbeat", bytes.NewReader(body)) if err != nil { - return "", 0, false + return activityAck{}, 0, false } req.Header.Set("Content-Type", "application/json") req.Header.Set(ActivityUpdateHeader, "1") @@ -317,25 +500,25 @@ func postActivityUpdate(ctx context.Context, hubURL, hiveID string, d *hubactivi resp, err := http.DefaultClient.Do(req) if err != nil { logger.Debug("hub activity update unreachable", "error", err) - return "", 0, false + return activityAck{}, 0, false } defer func() { _ = resp.Body.Close() }() if resp.StatusCode >= 300 { logger.Debug("hub activity update rejected", "status", resp.StatusCode) - return "", resp.StatusCode, false + return activityAck{}, resp.StatusCode, false } raw, err := io.ReadAll(io.LimitReader(resp.Body, maxPayloadBytes)) if err != nil { - return "", resp.StatusCode, false + return activityAck{}, resp.StatusCode, false } result := spokeHeartbeatVerifier(logger).Verify(SpokeSSOPublicKeys(), hiveID, raw, resp.Header.Get(spokesig.SigHeader), time.Now()) if !result.Accepted { logger.Warn("hub activity update response REJECTED by signature verification", "reason", result.Reason, "hive_id", hiveID) - return "", resp.StatusCode, false + return activityAck{}, resp.StatusCode, false } var hbResp HeartbeatResponse if err := json.Unmarshal(raw, &hbResp); err != nil { - return "", resp.StatusCode, false + return activityAck{}, resp.StatusCode, false } - return hbResp.ActivityETag, resp.StatusCode, true + return activityAck{ETag: hbResp.ActivityETag, Events: hbResp.ActivityEventsAck}, resp.StatusCode, true } diff --git a/src/pkg/hub/spoke/activity_publisher_test.go b/src/pkg/hub/spoke/activity_publisher_test.go index 6da771926f..70a67ed6ba 100644 --- a/src/pkg/hub/spoke/activity_publisher_test.go +++ b/src/pkg/hub/spoke/activity_publisher_test.go @@ -264,15 +264,19 @@ type recordingPost struct { ok bool } -func (r *recordingPost) post(_ context.Context, d *hubactivity.Digest) (string, int, bool) { +func (r *recordingPost) post(_ context.Context, d *hubactivity.Digest) (activityAck, int, bool) { r.sent <- d if r.gate != nil { <-r.gate } if !r.ok { - return "", r.status, false + return activityAck{}, r.status, false } - return d.ETag, r.status, true + ack := activityAck{ETag: d.ETag} + if n := len(d.Events); n > 0 { + ack.Events = d.Events[n-1].Seq + } + return ack, r.status, true } func TestActivityUpdatesCoalesceToOnePer15sWithBackpressure(t *testing.T) { diff --git a/src/pkg/hub/spoke/heartbeat.go b/src/pkg/hub/spoke/heartbeat.go index 858be8e59e..00bf3f9225 100644 --- a/src/pkg/hub/spoke/heartbeat.go +++ b/src/pkg/hub/spoke/heartbeat.go @@ -1875,6 +1875,7 @@ func postHeartbeatToHub(ctx context.Context, hubURL string, payload *HeartbeatPa } if pub != nil { pub.acknowledgeLocked(hbResp.ActivityETag) + pub.acknowledgeEventsLocked(hbResp.ActivityEventsAck) } return &hbResp } @@ -2890,6 +2891,9 @@ type HeartbeatResponse struct { // ActivityETag is the activity digest etag the hub acknowledged ("" = // it holds none), decoded from hub.HeartbeatResponse.ActivityETag. ActivityETag string `json:"activity_etag,omitempty"` + // ActivityEventsAck is the highest activity event sequence number the + // hub acknowledged, decoded from hub.HeartbeatResponse.ActivityEventsAck. + ActivityEventsAck uint64 `json:"activity_events_ack,omitempty"` } // HubBanner is a message from the hub admin displayed on spoke dashboards. diff --git a/src/pkg/hubactivity/events.go b/src/pkg/hubactivity/events.go new file mode 100644 index 0000000000..598fb486c4 --- /dev/null +++ b/src/pkg/hubactivity/events.go @@ -0,0 +1,116 @@ +package hubactivity + +import ( + "slices" + "strings" + "time" +) + +// Activity events (RFC hivecommons/hive#15861 §A/§B, child #15964). The +// digest says where every item is now; events say how items moved since the +// hub last acknowledged them — stage hops, opens and merges — so the hub +// board can animate movement and show merge toasts. Both ends bound them: +// the spoke buffers at most MaxEvents, dropping the oldest and disclosing +// the count, and the hub keeps a fleet-wide in-memory ring. + +const ( + // MaxEvents caps the events one activity block carries (and the spoke + // buffers between acknowledgements). The oldest are dropped first. + MaxEvents = 200 +) + +// Event types. +const ( + EventOpened = "opened" + EventStage = "stage" + EventMerged = "merged" +) + +var allowedEventTypes = []string{EventOpened, EventStage, EventMerged} + +// Event is one stage hop, open or merge. Seq is the spoke's monotonic +// sequence number; the hub acknowledges the highest Seq it holds and the +// spoke trims its buffer up to it. +type Event struct { + Seq uint64 `json:"seq"` + Type string `json:"type"` + Repo string `json:"repo"` + Number int `json:"n"` + Kind string `json:"kind,omitempty"` + Title string `json:"title,omitempty"` + From string `json:"from,omitempty"` + Stage string `json:"stage,omitempty"` + At time.Time `json:"at"` + // Private marks a private-repository item; such events exist only + // when the spoke shares private items. + Private bool `json:"private,omitempty"` +} + +// Merge is one merged pull request the spoke's tracker reports. +type Merge struct { + Repo string + Number int + MergedAt time.Time + Private bool +} + +// Snapshot is what a spoke's tracker gives the activity publisher: the +// open items and the recently merged pull requests. +type Snapshot struct { + Items []Item + Merged []Merge +} + +// NormalizeEvent bounds and validates one event; ok is false for an event +// that must be dropped (unknown type, repo not owner/name, no number or no +// sequence number). +func NormalizeEvent(e Event) (Event, bool) { + out := e + out.Type = oneOf(strings.ToLower(cleanToken(e.Type)), allowedEventTypes) + out.Repo = cleanToken(e.Repo) + if out.Type == "" || e.Seq == 0 || !repoPattern.MatchString(out.Repo) || e.Number <= 0 || e.Number > maxCount { + return Event{}, false + } + out.Kind = oneOf(strings.ToLower(cleanToken(e.Kind)), allowedKinds) + out.Title = cleanText(e.Title, MaxTitleRunes) + out.From = slug(e.From) + out.Stage = slug(e.Stage) + out.At = e.At.UTC() + return out, true +} + +// SanitizeEvents normalizes events received from a spoke, drops malformed +// and duplicate (by Seq) rows, orders them by Seq and keeps at most +// MaxEvents, dropping the oldest. dropped is how many the cap removed. The +// input is not modified. +func SanitizeEvents(events []Event) (out []Event, dropped int) { + seen := map[uint64]bool{} + for _, e := range events { + n, ok := NormalizeEvent(e) + if !ok || seen[n.Seq] { + continue + } + seen[n.Seq] = true + out = append(out, n) + } + slices.SortStableFunc(out, func(a, b Event) int { + switch { + case a.Seq < b.Seq: + return -1 + case a.Seq > b.Seq: + return 1 + } + return 0 + }) + if len(out) > MaxEvents { + dropped = len(out) - MaxEvents + out = slices.Clone(out[dropped:]) + } + return out, dropped +} + +// ClampCount bounds a spoke-claimed count the way Sanitize bounds every +// integer in the digest. +func ClampCount(n int) int { + return clampCount(n) +} diff --git a/src/pkg/hubactivity/events_test.go b/src/pkg/hubactivity/events_test.go new file mode 100644 index 0000000000..2d7ad9ac90 --- /dev/null +++ b/src/pkg/hubactivity/events_test.go @@ -0,0 +1,161 @@ +package hubactivity + +import ( + "slices" + "testing" + "time" +) + +func TestNormalizeEvent_BoundsAndValidates(t *testing.T) { + loc := time.FixedZone("x", 3600) + in := Event{ + Seq: 7, + Type: " STAGE\t", + Repo: " o/r ", + Number: 42, + Kind: "PR", + Title: "hello\u2028world", + From: " needs-review ", + Stage: "actionable\n", + At: t0.In(loc), + Private: true, + } + got, ok := NormalizeEvent(in) + if !ok { + t.Fatal("expected event to be accepted") + } + if got.Type != EventStage || got.Repo != "o/r" || got.Kind != "pr" { + t.Errorf("tokens not normalized: %+v", got) + } + if got.Title == in.Title || got.Title == "" { + t.Errorf("title not cleaned: %q", got.Title) + } + if got.From != "needs-review" || got.Stage != "actionable" { + t.Errorf("from/stage not slugged: %q %q", got.From, got.Stage) + } + if got.At.Location() != time.UTC || !got.At.Equal(t0) { + t.Errorf("At not UTC-normalized: %v", got.At) + } + if !got.Private || got.Seq != 7 || got.Number != 42 { + t.Errorf("fields not carried: %+v", got) + } + + bad := in + bad.From, bad.Stage = "Needs Review!", "Actionable" + got, ok = NormalizeEvent(bad) + if !ok { + t.Fatal("invalid slugs must blank the field, not drop the event") + } + if got.From != "" || got.Stage != "" { + t.Errorf("invalid from/stage not blanked: %q %q", got.From, got.Stage) + } +} + +func TestNormalizeEvent_RejectsMalformed(t *testing.T) { + valid := Event{Seq: 1, Type: EventOpened, Repo: "o/r", Number: 1} + if _, ok := NormalizeEvent(valid); !ok { + t.Fatal("baseline event must be accepted") + } + cases := map[string]func(e *Event){ + "unknown type": func(e *Event) { e.Type = "closed" }, + "empty type": func(e *Event) { e.Type = "" }, + "zero seq": func(e *Event) { e.Seq = 0 }, + "bad repo": func(e *Event) { e.Repo = "not-a-repo" }, + "empty repo": func(e *Event) { e.Repo = "" }, + "zero number": func(e *Event) { e.Number = 0 }, + "neg number": func(e *Event) { e.Number = -3 }, + "huge number": func(e *Event) { e.Number = maxCount + 1 }, + } + for name, mutate := range cases { + e := valid + mutate(&e) + got, ok := NormalizeEvent(e) + if ok { + t.Errorf("%s: expected rejection, got %+v", name, got) + } + if got != (Event{}) { + t.Errorf("%s: rejected event must be zero, got %+v", name, got) + } + } +} + +func TestNormalizeEvent_UnknownKindDropped(t *testing.T) { + got, ok := NormalizeEvent(Event{Seq: 1, Type: EventMerged, Repo: "o/r", Number: 1, Kind: "discussion"}) + if !ok { + t.Fatal("expected acceptance") + } + if got.Kind != "" { + t.Errorf("unknown kind should be blanked, got %q", got.Kind) + } +} + +func TestSanitizeEvents_DropsMalformedDedupesAndOrders(t *testing.T) { + in := []Event{ + {Seq: 3, Type: EventStage, Repo: "o/r", Number: 3}, + {Seq: 0, Type: EventStage, Repo: "o/r", Number: 9}, // no seq + {Seq: 1, Type: EventOpened, Repo: "o/r", Number: 1}, + {Seq: 3, Type: EventMerged, Repo: "o/r", Number: 33}, // duplicate seq + {Seq: 2, Type: "bogus", Repo: "o/r", Number: 2}, // bad type + {Seq: 2, Type: EventMerged, Repo: "o/r", Number: 2}, + } + snapshot := slices.Clone(in) + + out, dropped := SanitizeEvents(in) + if dropped != 0 { + t.Errorf("dropped = %d, want 0 (cap not hit)", dropped) + } + var seqs []uint64 + var nums []int + for _, e := range out { + seqs = append(seqs, e.Seq) + nums = append(nums, e.Number) + } + if !slices.Equal(seqs, []uint64{1, 2, 3}) { + t.Errorf("seqs = %v, want [1 2 3]", seqs) + } + if !slices.Equal(nums, []int{1, 2, 3}) { + t.Errorf("first occurrence must win: numbers = %v, want [1 2 3]", nums) + } + if !slices.Equal(in, snapshot) { + t.Error("SanitizeEvents must not modify its input") + } +} + +func TestSanitizeEvents_CapsAtMaxEventsDroppingOldest(t *testing.T) { + total := MaxEvents + 25 + in := make([]Event, 0, total) + for i := total; i >= 1; i-- { // reverse order to exercise sorting too + in = append(in, Event{Seq: uint64(i), Type: EventStage, Repo: "o/r", Number: i}) + } + + out, dropped := SanitizeEvents(in) + if dropped != 25 { + t.Errorf("dropped = %d, want 25", dropped) + } + if len(out) != MaxEvents { + t.Fatalf("len = %d, want %d", len(out), MaxEvents) + } + if out[0].Seq != 26 || out[len(out)-1].Seq != uint64(total) { + t.Errorf("kept range = [%d, %d], want [26, %d]", out[0].Seq, out[len(out)-1].Seq, total) + } + for i := 1; i < len(out); i++ { + if out[i-1].Seq >= out[i].Seq { + t.Fatalf("not strictly ascending at %d: %d >= %d", i, out[i-1].Seq, out[i].Seq) + } + } +} + +func TestSanitizeEvents_Empty(t *testing.T) { + out, dropped := SanitizeEvents(nil) + if len(out) != 0 || dropped != 0 { + t.Errorf("got %v, %d; want empty, 0", out, dropped) + } +} + +func TestClampCount(t *testing.T) { + for in, want := range map[int]int{-5: 0, 0: 0, 7: 7, maxCount: maxCount, maxCount + 1: maxCount} { + if got := ClampCount(in); got != want { + t.Errorf("ClampCount(%d) = %d, want %d", in, got, want) + } + } +} diff --git a/src/pkg/hubactivity/hubactivity.go b/src/pkg/hubactivity/hubactivity.go index 97bb2f7acd..408a8e4c13 100644 --- a/src/pkg/hubactivity/hubactivity.go +++ b/src/pkg/hubactivity/hubactivity.go @@ -97,10 +97,16 @@ type Digest struct { Items []Item `json:"items,omitempty"` // Truncated is how many eligible items the cap left out. Truncated int `json:"truncated,omitempty"` + // Events are the stage hops, opens and merges since the hub last + // acknowledged (#15964), oldest first and at most MaxEvents. + // EventsDropped discloses how many the cap dropped since then. Neither + // is part of the content etag, and an unchanged marker may carry them. + Events []Event `json:"events,omitempty"` + EventsDropped int `json:"events_dropped,omitempty"` } // IsUnchangedMarker reports whether d only re-asserts a previously sent -// etag. +// etag. Events do not count: a marker may carry them. func (d Digest) IsUnchangedMarker() bool { return d.ETag != "" && d.Totals == nil && len(d.Items) == 0 && d.Truncated == 0 }