From 9da07a3eead7b4d3351cea4aad494e41f755a8be Mon Sep 17 00:00:00 2001 From: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Sat, 31 Oct 2026 10:01:49 -0400 Subject: [PATCH 1/6] =?UTF-8?q?=E2=9C=A8=20feat(hub):=20carry=20bounded=20?= =?UTF-8?q?activity=20events=20through=20the=20heartbeat=20and=20hub=20rin?= =?UTF-8?q?g?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- .../added-15964-hub-activity-events.md | 1 + src/cmd/hive/main.go | 8 + src/docs/api-reference/hub-server.md | 4 +- src/docs/telemetry.md | 2 +- src/pkg/dashboard/hub_activity.go | 34 ++- src/pkg/hub/activity.go | 19 +- src/pkg/hub/activity_events.go | 205 ++++++++++++++ src/pkg/hub/activity_events_test.go | 214 ++++++++++++++ src/pkg/hub/activity_update.go | 1 + src/pkg/hub/heartbeat.go | 4 + src/pkg/hub/server.go | 1 + src/pkg/hub/spoke/activity_events_test.go | 265 ++++++++++++++++++ src/pkg/hub/spoke/activity_publisher.go | 243 ++++++++++++++-- src/pkg/hub/spoke/activity_publisher_test.go | 10 +- src/pkg/hub/spoke/heartbeat.go | 4 + src/pkg/hubactivity/events.go | 115 ++++++++ src/pkg/hubactivity/hubactivity.go | 8 +- 17 files changed, 1093 insertions(+), 45 deletions(-) create mode 100644 changelog.d/added-15964-hub-activity-events.md create mode 100644 src/pkg/hub/activity_events.go create mode 100644 src/pkg/hub/activity_events_test.go create mode 100644 src/pkg/hub/spoke/activity_events_test.go create mode 100644 src/pkg/hubactivity/events.go diff --git a/changelog.d/added-15964-hub-activity-events.md b/changelog.d/added-15964-hub-activity-events.md new file mode 100644 index 0000000000..feae12d5c6 --- /dev/null +++ b/changelog.d/added-15964-hub-activity-events.md @@ -0,0 +1 @@ +- The heartbeat `activity` block now carries the stage hops, opens and merges a spoke saw since the hub last acknowledged them (at most 200, oldest dropped first, drops disclosed as `events_dropped`), and `GET /api/hub/activity` serves them as `events` from a fleet-wide, memory-only ring of the last 1,000 events, filtered per user and per hive like items, so the hub board can animate movement and show merge toasts ([#15964](https://github.com/hivecommons/hive/issues/15964), RFC [#15861](https://github.com/hivecommons/hive/issues/15861)). diff --git a/src/cmd/hive/main.go b/src/cmd/hive/main.go index 207bf617db..e304f38c60 100644 --- a/src/cmd/hive/main.go +++ b/src/cmd/hive/main.go @@ -5686,6 +5686,14 @@ func (b *boot) bootHeartbeatWith(deps bootHeartbeatDeps) { } return b.dashSrv.HubActivityItems(ctx) }, + // Snapshot supersedes Items and adds recent merges, from + // which the publisher derives merge events (#15964). + Snapshot: func(ctx context.Context) (hubactivity.Snapshot, bool) { + if b.dashSrv == nil { + return hubactivity.Snapshot{}, false + } + return b.dashSrv.HubActivitySnapshot(ctx) + }, Subscribe: func(notify func()) { if b.dashSrv != nil { b.dashSrv.OnStageTrackerPublish(notify) diff --git a/src/docs/api-reference/hub-server.md b/src/docs/api-reference/hub-server.md index ab623d6bee..3fba7b9f13 100644 --- a/src/docs/api-reference/hub-server.md +++ b/src/docs/api-reference/hub-server.md @@ -38,7 +38,7 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `POST` | `/api/shared-agent-pool/leases` | Hub handler-specific | Acquire or renew a shared CLI session lease | `pkg/hub/server.go:1708` | | `PUT` | `/api/shared-agent-pool/leases` | Hub handler-specific | Renew an active shared CLI session lease | `pkg/hub/server.go:1709` | | `DELETE` | `/api/shared-agent-pool/leases` | Hub handler-specific | Release a shared CLI session lease | `pkg/hub/server.go:1710` | -| `POST` | `/api/heartbeat` | Hub handler-specific | Heartbeat. A request carrying `X-Hive-Activity-Update` is an activity-only update ([#15963](https://github.com/hivecommons/hive/issues/15963)): body `{"activity_update":{"hive_id","activity"}}`, the same per-hive bearer, accepted only for a hive already in the registry (`404` otherwise); it replaces the in-memory activity digest and nothing else, and the signed response carries `ok` and `activity_etag` | `pkg/hub/server.go:1711` | +| `POST` | `/api/heartbeat` | Hub handler-specific | Heartbeat. A request carrying `X-Hive-Activity-Update` is an activity-only update ([#15963](https://github.com/hivecommons/hive/issues/15963)): body `{"activity_update":{"hive_id","activity"}}`, the same per-hive bearer, accepted only for a hive already in the registry (`404` otherwise); it replaces the in-memory activity digest and nothing else, and the signed response carries `ok`, `activity_etag` and `activity_events_ack` (the highest activity event sequence number the hub holds for the hive, [#15964](https://github.com/hivecommons/hive/issues/15964); full heartbeat responses carry it too) | `pkg/hub/server.go:1711` | | `POST` | `/api/task-status` | Hub handler-specific | Task Status | `pkg/hub/server.go:1712` | | `GET` | `/api/registry` | Hub handler-specific | Registry | `pkg/hub/server.go:1720` | | `GET` | `/api/hub/leaderboard` | Hub handler-specific | Leaderboard | `pkg/hub/server.go:1721` | @@ -46,7 +46,7 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/fleet-stats` | Hub handler-specific | Fleet Stats | `pkg/hub/server.go:1723` | | `GET` | `/api/hub/evidence` | Hub admin | Read-only cross-hive evidence drill-down from each spoke's stored heartbeat `evidence` summary ([#12116](https://github.com/hivecommons/hive/issues/12116)); non-admins get `403`. Returns `spokes`, `reporting`, pooled `calibration` (bucket 0–5 → `{n, bad, rate, lo, hi}`; the hub view folds the six buckets into four reviewer-confidence ranges), `findings[]` (the plain-language *What this says* sentences, generated from the same fields; [#12456](https://github.com/hivecommons/hive/issues/12456)), `by_acmm_level` (`level`, `hives`, `merged_90d`, `revert_rate`, and `time_to_merge` — `median_of_hive_medians_s`, `hives`, `n` — the median of the level's per-hive median time-to-merge, omitted when no hive at the level reports one; [#12464](https://github.com/hivecommons/hive/issues/12464)), `family_false_approve_rate` (`same`/`cross`/`mixed`, hives classed by their reviewer vs author family sets), fleet `record_coverage` (merged PRs with an evidence record over `prs_merged_90d`), fleet panels ([#12370](https://github.com/hivecommons/hive/issues/12370)) `outcomes_by_author_kind[]` (`kind`, `hives`, `n`, `merged`, `reverted`, `merge_rate`, `revert_rate`), `write_counters[]` (`tier`, `hives`, `attempted`, `blocked`, `block_rate`) and `shadow_gate` (`review_gate` / `intent` → `held` / `passed` outcomes with `revert_rate`), each omitted while no spoke reports it and repeated per `by_acmm_level[]` row, and per hive `hives[]` with `acmm_level`, `prs_merged_90d`, `record_coverage`, `family_mix`, false-approve / false-block / revert rates, families, a calibration series the raw `counts` behind them (including `by_author_kind`) so the drill-down can re-pool a hive / ACMM-level selection client-side, and `criteria[]`: the per-axis criteria table derived hub-side from the quadrant criteria (`axis`, `label`, `scored`, `criteria[]` with `name`, `label`, `status` `met`/`unmet`/`not_reported`, `display`/`score` when measured, `reason` otherwise; offline, evidence-less or unscored hives are `not_reported`, never `unmet`; [#12463](https://github.com/hivecommons/hive/issues/12463)). Every rate is `{n, rate, lo, hi}` with a 95% Wilson interval; `rate`/`lo`/`hi` are null when `n` < 10. Window selection ([RFC #12062](https://github.com/hivecommons/hive/issues/12062)): `?hours=24|168|720` pools each hive's matching heartbeat `windows` block instead of its 90-day block (any other or absent `hours` → 90 days); `?from=YYYY-MM-DD|RFC3339[&to=…]` (date-only `to` is inclusive, `to` defaults to now, `to` ≥ `from`, span ≤ 366 days, else `400`) is served from the smallest reported window that reaches back to `from` and flagged `window.approximate`. The response carries `window` (`hours`, `key`, `label`, `from`, `to`, `approximate`, `coverage_available` — `record_coverage` is only computed for the 90-day window because `prs_merged_90d` is a 90-day count), `window_unreported` and per-hive `window_unreported` for hives whose spoke predates per-window reporting. `?hive=[,…]` (repeatable) and `?level=` restrict the registry server-side to those hives / that ACMM level before pooling. Hub admins open it from the Fleet quadrant card or its **🔬 Evidence** button ([#12117](https://github.com/hivecommons/hive/issues/12117)), pick a window (24h / 7d / 30d / 90d / custom), filter it by hive and ACMM level and read each section's ⓘ explainer; the per-hive author-kind revert counts behind the *Revert rate by author kind* panel are `hives[].counts.by_author_kind` ([#12326](https://github.com/hivecommons/hive/issues/12326)). Never consulted by merge decisions | `pkg/hub/evidence.go` | | `GET` | `/api/hub/evidence/report` | Hub admin | Plain-language report of the hub evidence view ([#12456](https://github.com/hivecommons/hive/issues/12456)), rendered server-side from the same `EvidenceView` as `/api/hub/evidence` so every figure reconciles with the modal; non-admins get `403`. Same `hours=` / `from=` / `to=` window and `hive=` / `level=` filter parameters. `format=md` (default, `text/markdown`) or `format=html` (standalone page, inline CSS, no scripts or external assets, hive names escaped); any other value → `400`. `download=1` sets `Content-Disposition: attachment; filename="hive-evidence---.md|html"`, otherwise inline. Sections: Summary (the generated findings), What we measured and why, The numbers, How to read these, What is not reported yet and why, Grade per section, footer link to `review-evidence.md`. `Cache-Control: no-store`. Read-only; never consulted by merge decisions | `pkg/hub/evidence_report.go` | -| `GET` | `/api/hub/activity` | Signed-in user | Read-only hub-wide activity view ([#15951](https://github.com/hivecommons/hive/issues/15951), RFC [#15861](https://github.com/hivecommons/hive/issues/15861)) merged from each spoke's heartbeat `activity` digest, held in hub memory only. Authorization is per user and per hive before aggregation: hub admins see every registered hive; everyone else only hives they own or hold an unexpired grant for; anonymous → `401`. Returns `hives[]` (`hive_id`, `name`, `status` `ok`/`stale`/`not_reporting`/`missing`, `reporting`, `stale` once the block is older than two heartbeats, `last_heartbeat`, `received_at`, `generated_at`, `totals` — `items`, `by_stage`, `by_modifier`, counting private and truncated items too — and `truncated`), `items[]` (`hive_id`, `repo`, `n`, `kind`, `title`, `author_kind`, `stage`, `stage_n`, `since`, `mods`, `private`) ordered needs-human/needs-decision, stalled, furthest stage, oldest, and a response-level `truncated` (cap 2000). Each digest carries at most 100 items by default and never more than the 250-item hub ceiling; private-repository items appear only when the spoke opts in. Filters: `hive`, `repo`, `stage`, `mod`, `author_kind`, `q` (title substring or `#`), each ≤ 200 characters (else `400`). `ETag` + `If-None-Match` → `304`; `Cache-Control: private, no-cache`, `Vary: Cookie, Authorization`. No actions | `pkg/hub/activity.go` | +| `GET` | `/api/hub/activity` | Signed-in user | Read-only hub-wide activity view ([#15951](https://github.com/hivecommons/hive/issues/15951), RFC [#15861](https://github.com/hivecommons/hive/issues/15861)) merged from each spoke's heartbeat `activity` digest, held in hub memory only. Authorization is per user and per hive before aggregation: hub admins see every registered hive; everyone else only hives they own or hold an unexpired grant for; anonymous → `401`. Returns `hives[]` (`hive_id`, `name`, `status` `ok`/`stale`/`not_reporting`/`missing`, `reporting`, `stale` once the block is older than two heartbeats, `last_heartbeat`, `received_at`, `generated_at`, `totals` — `items`, `by_stage`, `by_modifier`, counting private and truncated items too — `truncated` and `events_dropped`), `items[]` (`hive_id`, `repo`, `n`, `kind`, `title`, `author_kind`, `stage`, `stage_n`, `since`, `mods`, `private`) ordered needs-human/needs-decision, stalled, furthest stage, oldest, a response-level `truncated` (cap 2000), and `events[]` ([#15964](https://github.com/hivecommons/hive/issues/15964): `hive_id`, `received_at`, `seq`, `type` `opened`/`stage`/`merged`, `repo`, `n`, `kind`, `title`, `from`, `stage`, `at`, `private`), oldest first, from a fleet-wide in-memory ring of the last 1,000 events. Events are filtered per user and per hive exactly like items, before aggregation, so an event for a hive the viewer cannot see is never delivered; `mod`/`author_kind` filters keep only events whose item is still on the hive's board. A spoke block carries at most 200 events; drops are disclosed per hive as `events_dropped`. New events change the `ETag`. Each digest carries at most 100 items by default and never more than the 250-item hub ceiling; private-repository items appear only when the spoke opts in. Filters: `hive`, `repo`, `stage`, `mod`, `author_kind`, `q` (title substring or `#`), each ≤ 200 characters (else `400`). `ETag` + `If-None-Match` → `304`; `Cache-Control: private, no-cache`, `Vary: Cookie, Authorization`. No actions | `pkg/hub/activity.go` | | `GET` | `/api/hub/version` | Hub handler-specific | Hub Version | `pkg/hub/server.go:1731` | | `DELETE` | `/api/hub/registry/{id}` | Hub handler-specific | Registry Delete | `pkg/hub/server.go:1741` | | `POST` | `/api/contribute/register` | Hub handler-specific | Contribute Proxy | `pkg/hub/server.go:1742` | diff --git a/src/docs/telemetry.md b/src/docs/telemetry.md index ce3713ab55..9f5a56a4cb 100644 --- a/src/docs/telemetry.md +++ b/src/docs/telemetry.md @@ -72,7 +72,7 @@ hub: share_private: false # private-repository items are dropped (still counted in totals) unless true ``` -When the cap applies, items needing a human or a decision are kept first, then stalled ones, then the furthest stage, and the number left out is reported as `truncated`. A repository counts as public only when the hub lists the hive as public (`hub.is_public`, its project repos) or it is on `HIVE_PUBLIC_REPOS`. Once the hub acknowledges a digest's etag, later beats send only `etag` and `generated_at` until the tracker changes. A tracker change also sends an activity-only update over the heartbeat endpoint and bearer, at most one every 15 seconds; changes arriving meanwhile are merged into it. `hub.heartbeat_omit` applies: `repos` withholds every item (totals remain) and `task_titles` blanks titles. +When the cap applies, items needing a human or a decision are kept first, then stalled ones, then the furthest stage, and the number left out is reported as `truncated`. A repository counts as public only when the hub lists the hive as public (`hub.is_public`, its project repos) or it is on `HIVE_PUBLIC_REPOS`. Once the hub acknowledges a digest's etag, later beats send only `etag` and `generated_at` until the tracker changes. A tracker change also sends an activity-only update over the heartbeat endpoint and bearer, at most one every 15 seconds; changes arriving meanwhile are merged into it. The block also carries `events` ([#15964](https://github.com/hivecommons/hive/issues/15964)): the stage hops, opens and merges the spoke saw since the hub last acknowledged them, derived by diffing successive tracker snapshots (the first one only seeds the baseline), each with a sequence number. At most 200 are buffered; the oldest are dropped first and `events_dropped` discloses how many. Events stay buffered, and are resent, until a heartbeat or update response's `activity_events_ack` covers them. Private-repository items yield no events unless `share_private` is on. The hub keeps the last 1,000 events across the fleet in memory only. `hub.heartbeat_omit` applies: `repos` withholds every item and event (totals remain) and `task_titles` blanks titles. ## Telemetry agent diff --git a/src/pkg/dashboard/hub_activity.go b/src/pkg/dashboard/hub_activity.go index ed575d880a..a278242a58 100644 --- a/src/pkg/dashboard/hub_activity.go +++ b/src/pkg/dashboard/hub_activity.go @@ -46,26 +46,48 @@ func (s *Server) notifyStageTrackerPublish() { // while the spoke has no status or the tracker cannot be built. Reading // does not count as a tracker client, so it adds no SSE nudges. func (s *Server) HubActivityItems(ctx context.Context) ([]hubactivity.Item, bool) { + snap, ok := s.HubActivitySnapshot(ctx) + return snap.Items, ok +} + +// HubActivitySnapshot is HubActivityItems plus the tracker's recently +// merged pull requests (RecentlyMerged), which the heartbeat publisher +// turns into merge events (#15964). Merges carry Private the same way. +func (s *Server) HubActivitySnapshot(ctx context.Context) (hubactivity.Snapshot, bool) { if s == nil { - return nil, false + return hubactivity.Snapshot{}, false } s.statusMu.RLock() status := s.status s.statusMu.RUnlock() if status == nil { - return nil, false + return hubactivity.Snapshot{}, false } body, _, _, _ := s.stageTrackerPayloadFrom(ctx, status, time.Now().UTC(), false) if body == nil { - return nil, false + return hubactivity.Snapshot{}, false } var tracker struct { - Items []StageTrackerItem `json:"items"` + Items []StageTrackerItem `json:"items"` + RecentlyMerged []StageTrackerMerged `json:"recently_merged"` } if err := json.Unmarshal(body, &tracker); err != nil { - return nil, false + return hubactivity.Snapshot{}, false + } + public := s.publicRepos() + return hubactivity.Snapshot{ + Items: hubActivityItems(tracker.Items, public), + Merged: hubActivityMerges(tracker.RecentlyMerged, public), + }, true +} + +// hubActivityMerges maps the tracker's recently merged list to merge rows. +func hubActivityMerges(merged []StageTrackerMerged, public publicRepoSet) []hubactivity.Merge { + out := make([]hubactivity.Merge, 0, len(merged)) + for _, m := range merged { + out = append(out, hubactivity.Merge{Repo: m.Repo, Number: m.Number, MergedAt: m.MergedAt, Private: !public.allows(m.Repo)}) } - return hubActivityItems(tracker.Items, s.publicRepos()), true + return out } // hubActivityItems maps tracker rows to digest rows. A row the tracker diff --git a/src/pkg/hub/activity.go b/src/pkg/hub/activity.go index abe58319e5..415e5ecbaf 100644 --- a/src/pkg/hub/activity.go +++ b/src/pkg/hub/activity.go @@ -64,6 +64,8 @@ type storedActivity struct { type activityStore struct { mu sync.Mutex byHive map[string]storedActivity + // events is the fleet-wide ring of recent activity events (#15964). + events activityEventRing } // record applies a heartbeat's activity block and returns the etag the hub @@ -85,6 +87,9 @@ func (st *activityStore) record(hiveID string, block *hubactivity.Digest, now ti } return "" } + // Events ride on markers and full digests alike, and are taken even + // when the marker's etag misses. + st.events.ingest(hiveID, block.Events, block.EventsDropped, now) if block.IsUnchangedMarker() { // Compare in the stored (cleaned) form, or a marker whose etag // changes under cleaning would never match and wedge the handshake. @@ -124,6 +129,7 @@ func (st *activityStore) forget(hiveID string) { st.mu.Lock() defer st.mu.Unlock() delete(st.byHive, hiveID) + st.events.forget(hiveID) } func (st *activityStore) forgetOldLocked(now time.Time) { @@ -132,6 +138,7 @@ func (st *activityStore) forgetOldLocked(now time.Time) { delete(st.byHive, id) } } + st.events.forgetOld(now, activityForgetAfter) } // snapshot copies the blocks of the given hives only, so data of hives the @@ -262,6 +269,9 @@ type ActivityHiveView struct { Totals *hubactivity.Totals `json:"totals,omitempty"` // Truncated is how many of this hive's items its digest left out. Truncated int `json:"truncated,omitempty"` + // EventsDropped is how many of this hive's events the spoke's or the + // hub's per-block cap dropped before they reached the ring (#15964). + EventsDropped int `json:"events_dropped,omitempty"` } // ActivityViewItem is one merged item tagged with its hive. @@ -274,6 +284,9 @@ type ActivityViewItem struct { type ActivityView struct { Hives []ActivityHiveView `json:"hives"` Items []ActivityViewItem `json:"items"` + // Events are the visible hives' recent stage hops, opens and merges + // from the fleet-wide ring, oldest first (#15964). + Events []ActivityViewEvent `json:"events"` // Truncated is how many matching items the response cap left out, on // top of what each hive's own digest already left out. Truncated int `json:"truncated,omitempty"` @@ -282,7 +295,7 @@ type ActivityView struct { // buildActivityView merges the blocks of the visible hives only. Hives must // already be authorized for the viewer. func buildActivityView(hives []activityHive, blocks map[string]storedActivity, f activityFilter, now time.Time) ActivityView { - view := ActivityView{Hives: []ActivityHiveView{}, Items: []ActivityViewItem{}} + view := ActivityView{Hives: []ActivityHiveView{}, Items: []ActivityViewItem{}, Events: []ActivityViewEvent{}} for _, h := range hives { hv := ActivityHiveView{HiveID: h.ID, Name: h.Name, LastHeartbeat: h.LastHeartbeat} block, ok := blocks[h.ID] @@ -359,7 +372,9 @@ func (s *HubServer) handleHubActivity(w http.ResponseWriter, r *http.Request) { for i, h := range hives { ids[i] = h.ID } - view := buildActivityView(hives, s.activity.snapshot(ids), f, now) + blocks := s.activity.snapshot(ids) + view := buildActivityView(hives, blocks, f, now) + addActivityEvents(&view, hives, blocks, s.activity.eventsSnapshot(ids), f) body, err := json.Marshal(view) if err != nil { s.logger.Error("hub activity: encode view", "error", err) diff --git a/src/pkg/hub/activity_events.go b/src/pkg/hub/activity_events.go new file mode 100644 index 0000000000..0a34e41381 --- /dev/null +++ b/src/pkg/hub/activity_events.go @@ -0,0 +1,205 @@ +package hub + +import ( + "slices" + "strconv" + "strings" + "time" + + "github.com/hivecommons/hive/pkg/hubactivity" +) + +// Fleet-wide activity event ring (RFC hivecommons/hive#15861 §A/§B, child +// #15964). Spokes attach the stage hops, opens and merges they saw since +// the hub last acknowledged to their activity block; the hub keeps the most +// recent activityEventRingSize of them across the fleet IN MEMORY ONLY and +// serves them on GET /api/hub/activity, so the hub board can animate +// movement and show merge toasts. Events are copied out per hive only for +// hives the viewer is authorized for, exactly like digests, and they are +// part of the response body, so its ETag changes when new events arrive. + +// activityEventRingSize bounds the fleet-wide ring; the oldest event is +// dropped first. +const activityEventRingSize = 1000 + +// activityEvent is one stored event tagged with its hive. +type activityEvent struct { + hiveID string + event hubactivity.Event + receivedAt time.Time +} + +// activityEventHive is the per-hive acknowledgement state. +type activityEventHive struct { + // lastSeq is the highest event sequence number accepted (the ack). + lastSeq uint64 + // dropped sums the drops the spoke disclosed and the hub's own cap + // applied to its batches. + dropped int + lastSeen time.Time +} + +// activityEventRing is guarded by activityStore.mu. The zero value is +// ready to use. +type activityEventRing struct { + entries []activityEvent + byHive map[string]*activityEventHive +} + +// ingest takes a spoke's events. Only events newer than the hive's +// acknowledged sequence number are kept, so a batch resent because an +// acknowledgement was lost is not stored (or its drops counted) twice. +func (r *activityEventRing) ingest(hiveID string, events []hubactivity.Event, dropped int, now time.Time) { + clean, capped := hubactivity.SanitizeEvents(events) + h := r.byHive[hiveID] + if h == nil { + if len(clean) == 0 { + return + } + if r.byHive == nil { + r.byHive = map[string]*activityEventHive{} + } + h = &activityEventHive{} + r.byHive[hiveID] = h + } + h.lastSeen = now + fresh := 0 + for _, e := range clean { + if e.Seq <= h.lastSeq { + continue + } + h.lastSeq = e.Seq + r.entries = append(r.entries, activityEvent{hiveID: hiveID, event: e, receivedAt: now}) + fresh++ + } + if fresh == 0 { + return + } + h.dropped = hubactivity.ClampCount(h.dropped + hubactivity.ClampCount(dropped) + capped) + if over := len(r.entries) - activityEventRingSize; over > 0 { + r.entries = slices.Delete(r.entries, 0, over) + } +} + +// ack is the highest sequence number held for the hive (0 = none). +func (r *activityEventRing) ack(hiveID string) uint64 { + if h := r.byHive[hiveID]; h != nil { + return h.lastSeq + } + return 0 +} + +// forget drops a hive's events and acknowledgement state. +func (r *activityEventRing) forget(hiveID string) { + if _, ok := r.byHive[hiveID]; !ok { + return + } + delete(r.byHive, hiveID) + r.entries = slices.DeleteFunc(r.entries, func(e activityEvent) bool { return e.hiveID == hiveID }) +} + +// forgetOld forgets hives that sent nothing for maxAge. +func (r *activityEventRing) forgetOld(now time.Time, maxAge time.Duration) { + for id, h := range r.byHive { + if now.Sub(h.lastSeen) > maxAge { + r.forget(id) + } + } +} + +// activityEventsSnapshot is the events and drop counts of a set of hives. +type activityEventsSnapshot struct { + events []activityEvent + dropped map[string]int +} + +// eventsAck is the highest event sequence number the hub holds for hiveID, +// returned to the spoke so it can trim its buffer. +func (st *activityStore) eventsAck(hiveID string) uint64 { + if st == nil { + return 0 + } + st.mu.Lock() + defer st.mu.Unlock() + return st.events.ack(hiveID) +} + +// eventsSnapshot copies the events of the given hives only, oldest first, +// so another hive's events are never copied out of the store. +func (st *activityStore) eventsSnapshot(hiveIDs []string) activityEventsSnapshot { + st.mu.Lock() + defer st.mu.Unlock() + want := make(map[string]bool, len(hiveIDs)) + for _, id := range hiveIDs { + want[id] = true + } + out := activityEventsSnapshot{dropped: map[string]int{}} + for _, e := range st.events.entries { + if want[e.hiveID] { + out.events = append(out.events, e) + } + } + for id := range want { + if h := st.events.byHive[id]; h != nil && h.dropped > 0 { + out.dropped[id] = h.dropped + } + } + return out +} + +// ActivityViewEvent is one event tagged with its hive. +type ActivityViewEvent struct { + HiveID string `json:"hive_id"` + ReceivedAt time.Time `json:"received_at"` + hubactivity.Event +} + +// matchesEvent applies the view filter to an event. Filters the event does +// not carry (modifier, author kind) are checked against the item's current +// row in the hive's digest; an event whose item is not there is left out. +func (f activityFilter) matchesEvent(hiveID string, e hubactivity.Event, current map[string]hubactivity.Item) bool { + it := hubactivity.Item{Repo: e.Repo, Number: e.Number, Kind: e.Kind, Title: e.Title, Stage: e.Stage} + if f.Mod != "" || f.AuthorKind != "" { + cur, ok := current[activityItemKey(e.Repo, e.Number)] + if !ok { + return false + } + it.Mods, it.AuthorKind = cur.Mods, cur.AuthorKind + } + return f.matches(hiveID, it) +} + +func activityItemKey(repo string, n int) string { + return strings.ToLower(repo) + "#" + strconv.Itoa(n) +} + +// addActivityEvents adds the filtered events of the visible hives to view. +// Hives must already be authorized for the viewer, and the snapshot taken +// for them only. +func addActivityEvents(view *ActivityView, hives []activityHive, blocks map[string]storedActivity, snap activityEventsSnapshot, f activityFilter) { + visible := make(map[string]bool, len(hives)) + for _, h := range hives { + if !h.Missing { + visible[h.ID] = true + } + } + current := map[string]map[string]hubactivity.Item{} + if f.Mod != "" || f.AuthorKind != "" { + for id, b := range blocks { + m := make(map[string]hubactivity.Item, len(b.digest.Items)) + for _, it := range b.digest.Items { + m[activityItemKey(it.Repo, it.Number)] = it + } + current[id] = m + } + } + for _, e := range snap.events { + if !visible[e.hiveID] || !f.matchesEvent(e.hiveID, e.event, current[e.hiveID]) { + continue + } + view.Events = append(view.Events, ActivityViewEvent{HiveID: e.hiveID, ReceivedAt: e.receivedAt.UTC(), Event: e.event}) + } + for i := range view.Hives { + view.Hives[i].EventsDropped = snap.dropped[view.Hives[i].HiveID] + } +} diff --git a/src/pkg/hub/activity_events_test.go b/src/pkg/hub/activity_events_test.go new file mode 100644 index 0000000000..8caedc09ef --- /dev/null +++ b/src/pkg/hub/activity_events_test.go @@ -0,0 +1,214 @@ +package hub + +import ( + "fmt" + "net/http" + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/hubactivity" +) + +func activityEv(seq uint64, typ, repo string, n int) hubactivity.Event { + return hubactivity.Event{Seq: seq, Type: typ, Repo: repo, Number: n, Kind: "pr", Title: fmt.Sprintf("%s #%d", repo, n), Stage: "ci_review", At: activityT0} +} + +func withEvents(d *hubactivity.Digest, events ...hubactivity.Event) *hubactivity.Digest { + out := *d + out.Events = events + return &out +} + +func viewEventKeys(v ActivityView) []string { + out := make([]string, 0, len(v.Events)) + for _, e := range v.Events { + out = append(out, fmt.Sprintf("%s:%s#%d:%s", e.HiveID, e.Repo, e.Number, e.Type)) + } + return out +} + +// Acceptance (#15964): a merge a spoke reports on a heartbeat is served by +// /api/hub/activity right after that one heartbeat, and the ETag changes +// so a poller picks it up. +func TestHubActivityEvents_MergeAppearsWithinOneHeartbeat(t *testing.T) { + cleanup := helperSetupTempDirs(t) + defer cleanup() + t.Setenv(hubAdminsEnv, "github:root") + s := newActivityHub() + mkUser(t, "root") + + d := activityDigest("o/a", 1, 2) + postActivityHeartbeat(t, s, "hive-a", d) + first := getActivity(t, s, "root", "", "") + if v := decodeActivity(t, first); len(v.Events) != 0 { + t.Fatalf("events before any movement = %v", viewEventKeys(v)) + } + etag := first.Header().Get("ETag") + + marker := &hubactivity.Digest{ETag: d.ETag, GeneratedAt: activityT0.Add(time.Minute)} + merge := activityEv(7, hubactivity.EventMerged, "o/a", 2) + resp := postActivityHeartbeat(t, s, "hive-a", withEvents(marker, merge)) + if resp.ActivityEventsAck != 7 || resp.ActivityETag != d.ETag { + t.Fatalf("heartbeat ack = %q / %d, want the held etag and event seq 7", resp.ActivityETag, resp.ActivityEventsAck) + } + + rec := getActivity(t, s, "root", "", etag) + if rec.Code != http.StatusOK || rec.Header().Get("ETag") == etag { + t.Fatalf("new events must change the etag: status %d etag %q (was %q)", rec.Code, rec.Header().Get("ETag"), etag) + } + v := decodeActivity(t, rec) + if got := viewEventKeys(v); len(got) != 1 || got[0] != "hive-a:o/a#2:merged" { + t.Fatalf("events = %v, want the merge", got) + } + if len(v.Items) != 2 { + t.Fatalf("a marker carrying events must keep the held digest: items %d", len(v.Items)) + } +} + +// An event of a hive the user is not authorized for is never delivered; +// filtering happens before the response (and its etag) is built. +func TestHubActivityEvents_PerUserScoping(t *testing.T) { + cleanup := helperSetupTempDirs(t) + defer cleanup() + t.Setenv(hubAdminsEnv, "github:root") + s := newActivityHub() + + postActivityHeartbeat(t, s, "hive-a", withEvents(activityDigest("o/a", 1), activityEv(1, hubactivity.EventOpened, "o/a", 1))) + postActivityHeartbeat(t, s, "hive-b", withEvents(activityDigest("o/b", 3), activityEv(1, hubactivity.EventMerged, "o/b", 3))) + if err := saveSaaSUser(&SaaSUser{GitHubUsername: "alice", Hives: map[string]string{"hive-a": "viewer"}}); err != nil { + t.Fatal(err) + } + mkUser(t, "mallory") + mkUser(t, "root") + + rec := getActivity(t, s, "alice", "", "") + if got := viewEventKeys(decodeActivity(t, rec)); len(got) != 1 || got[0] != "hive-a:o/a#1:opened" { + t.Fatalf("alice events = %v, want only hive-a's", got) + } + if strings.Contains(rec.Body.String(), "o/b") { + t.Fatalf("hive-b leaked to alice: %s", rec.Body.String()) + } + aliceETag := rec.Header().Get("ETag") + + // Movement on a hive alice cannot see must not even change her etag. + postActivityHeartbeat(t, s, "hive-b", withEvents(activityDigest("o/b", 3), activityEv(2, hubactivity.EventStage, "o/b", 3))) + if again := getActivity(t, s, "alice", "", aliceETag); again.Code != http.StatusNotModified { + t.Fatalf("hive-b movement changed alice's view: status %d", again.Code) + } + + if v := decodeActivity(t, getActivity(t, s, "mallory", "", "")); len(v.Events) != 0 { + t.Fatalf("ungranted user saw events %v", viewEventKeys(v)) + } + if got := viewEventKeys(decodeActivity(t, getActivity(t, s, "root", "", ""))); len(got) != 3 { + t.Fatalf("admin events = %v, want all three", got) + } + + // Filters apply to events too. + if got := viewEventKeys(decodeActivity(t, getActivity(t, s, "root", "?hive=hive-b", ""))); len(got) != 2 { + t.Fatalf("hive filter events = %v, want hive-b's two", got) + } + if got := viewEventKeys(decodeActivity(t, getActivity(t, s, "root", "?mod=stalled", ""))); len(got) != 0 { + t.Fatalf("modifier filter must drop events of items without it: %v", got) + } +} + +// Private items the spoke does not share never become events (the spoke's +// exclusion, tested in pkg/hub/spoke); an event the hub receives for a hive +// it then forgets is purged with it. +func TestHubActivityEvents_ForgetPurgesEvents(t *testing.T) { + var st activityStore + st.record("hive-a", withEvents(activityDigest("o/a", 1), activityEv(1, hubactivity.EventOpened, "o/a", 1)), activityT0) + st.record("hive-b", withEvents(activityDigest("o/b", 1), activityEv(1, hubactivity.EventOpened, "o/b", 1)), activityT0) + + st.forget("hive-a") + + snap := st.eventsSnapshot([]string{"hive-a", "hive-b"}) + if len(snap.events) != 1 || snap.events[0].hiveID != "hive-b" { + t.Fatalf("events after forget = %+v", snap.events) + } + if st.eventsAck("hive-a") != 0 { + t.Fatal("a forgotten hive must lose its event acknowledgement too") + } +} + +func TestActivityEventRing_DedupesByAcknowledgedSeq(t *testing.T) { + var st activityStore + batch := withEvents(activityDigest("o/a", 1), activityEv(10, hubactivity.EventOpened, "o/a", 1), activityEv(11, hubactivity.EventStage, "o/a", 1)) + batch.EventsDropped = 3 + + st.record("hive-a", batch, activityT0) + // The spoke lost the ack and resends the same batch plus one new event. + resend := withEvents(batch, append(batch.Events, activityEv(12, hubactivity.EventMerged, "o/a", 1))...) + st.record("hive-a", resend, activityT0.Add(time.Minute)) + // A batch with nothing new does not count its drops again. + st.record("hive-a", batch, activityT0.Add(2*time.Minute)) + + snap := st.eventsSnapshot([]string{"hive-a"}) + if len(snap.events) != 3 { + t.Fatalf("stored %d events, want 3 (no duplicates)", len(snap.events)) + } + if got := st.eventsAck("hive-a"); got != 12 { + t.Fatalf("ack = %d, want 12", got) + } + if snap.dropped["hive-a"] != 6 { + t.Fatalf("disclosed drops = %d, want 3 per batch with new events (6)", snap.dropped["hive-a"]) + } +} + +// The fleet-wide ring keeps the last 1,000 events across all hives, oldest +// dropped first; each block is also capped at hubactivity.MaxEvents with +// the overflow disclosed. +func TestActivityEventRing_FleetCapAndPerBlockCap(t *testing.T) { + var st activityStore + seq := uint64(0) + batch := func(repo string, n int) []hubactivity.Event { + out := make([]hubactivity.Event, 0, n) + for range n { + seq++ + out = append(out, activityEv(seq, hubactivity.EventStage, repo, int(seq))) + } + return out + } + + // 250 events in one block: the hub keeps the newest 200 and discloses 50. + st.record("hive-a", withEvents(activityDigest("o/a", 1), batch("o/a", 250)...), activityT0) + snap := st.eventsSnapshot([]string{"hive-a"}) + if len(snap.events) != hubactivity.MaxEvents || snap.events[0].event.Seq != 51 || snap.dropped["hive-a"] != 50 { + t.Fatalf("per-block cap: %d events from seq %d, dropped %d; want 200 from 51, 50 dropped", len(snap.events), snap.events[0].event.Seq, snap.dropped["hive-a"]) + } + + // Five more full blocks across two hives overflow the fleet ring. + for i := range 5 { + hive, repo := "hive-a", "o/a" + if i%2 == 1 { + hive, repo = "hive-b", "o/b" + } + st.record(hive, withEvents(activityDigest(repo, 1), batch(repo, 200)...), activityT0.Add(time.Duration(i+1)*time.Minute)) + } + all := st.eventsSnapshot([]string{"hive-a", "hive-b"}) + if len(all.events) != activityEventRingSize { + t.Fatalf("ring holds %d events, want %d", len(all.events), activityEventRingSize) + } + if first, last := all.events[0].event.Seq, all.events[len(all.events)-1].event.Seq; first != seq-activityEventRingSize+1 || last != seq { + t.Fatalf("ring spans seq %d..%d, want the newest %d ending at %d", first, last, activityEventRingSize, seq) + } +} + +func TestActivityEventRing_DropsMalformedEvents(t *testing.T) { + var st activityStore + st.record("hive-a", withEvents(activityDigest("o/a", 1), + hubactivity.Event{Seq: 1, Type: "exploded", Repo: "o/a", Number: 1}, + hubactivity.Event{Seq: 2, Type: hubactivity.EventMerged, Repo: "not a repo", Number: 1}, + hubactivity.Event{Seq: 0, Type: hubactivity.EventMerged, Repo: "o/a", Number: 1}, + hubactivity.Event{Seq: 3, Type: hubactivity.EventMerged, Repo: "o/a", Number: 1, Title: strings.Repeat("x", 500)}, + ), activityT0) + + snap := st.eventsSnapshot([]string{"hive-a"}) + if len(snap.events) != 1 || snap.events[0].event.Seq != 3 { + t.Fatalf("events = %+v, want only the valid merge", snap.events) + } + if n := len([]rune(snap.events[0].event.Title)); n != hubactivity.MaxTitleRunes { + t.Fatalf("title runes = %d, want bounded to %d", n, hubactivity.MaxTitleRunes) + } +} diff --git a/src/pkg/hub/activity_update.go b/src/pkg/hub/activity_update.go index 11f5ed7e49..31da1e7761 100644 --- a/src/pkg/hub/activity_update.go +++ b/src/pkg/hub/activity_update.go @@ -76,6 +76,7 @@ func (s *HubServer) handleActivityUpdate(w http.ResponseWriter, r *http.Request) } resp := HeartbeatResponse{OK: true} resp.ActivityETag = s.activity.record(hiveID, payload.Update.Activity, time.Now().UTC()) + resp.ActivityEventsAck = s.activity.eventsAck(hiveID) w.Header().Set("Content-Type", "application/json") s.signHeartbeatResponse(w, &resp, hiveID) data, _ := json.Marshal(resp) diff --git a/src/pkg/hub/heartbeat.go b/src/pkg/hub/heartbeat.go index 9f6ce240af..1545db6c2d 100644 --- a/src/pkg/hub/heartbeat.go +++ b/src/pkg/hub/heartbeat.go @@ -476,6 +476,10 @@ type HeartbeatResponse struct { // this hive ("" = none). A spoke whose etag differs sends its full // digest instead of an unchanged marker on the next beat. ActivityETag string `json:"activity_etag,omitempty"` + // ActivityEventsAck is the highest activity event sequence number the + // hub holds for this hive (#15964); the spoke drops buffered events up + // to it. 0/absent = none. Decoded by spoke.HeartbeatResponse. + ActivityEventsAck uint64 `json:"activity_events_ack,omitempty"` } // HubBanner is a message from the hub admin displayed on spoke dashboards. diff --git a/src/pkg/hub/server.go b/src/pkg/hub/server.go index 0d2686b9ed..64eb5abe4c 100644 --- a/src/pkg/hub/server.go +++ b/src/pkg/hub/server.go @@ -2824,6 +2824,7 @@ func (s *HubServer) handleHeartbeat(w http.ResponseWriter, r *http.Request) { } resp.SelfExperimentFlag = s.selfExperimentFlagForHeartbeat(payload.HiveID, payload.SelfExperimentsOptIn) resp.ActivityETag = activityAck + resp.ActivityEventsAck = s.activity.eventsAck(payload.HiveID) if entry.IsPublic != payload.IsPublic { resp.IsPublic = &entry.IsPublic diff --git a/src/pkg/hub/spoke/activity_events_test.go b/src/pkg/hub/spoke/activity_events_test.go new file mode 100644 index 0000000000..ba501cb00a --- /dev/null +++ b/src/pkg/hub/spoke/activity_events_test.go @@ -0,0 +1,265 @@ +package spoke + +import ( + "context" + "fmt" + "sync" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/hubactivity" +) + +// activitySnapshotBox is a tracker snapshot (items and merges) a test can +// swap. +type activitySnapshotBox struct { + mu sync.Mutex + snap hubactivity.Snapshot +} + +func (b *activitySnapshotBox) Set(items []hubactivity.Item, merged ...hubactivity.Merge) { + b.mu.Lock() + b.snap = hubactivity.Snapshot{Items: items, Merged: merged} + b.mu.Unlock() +} + +func (b *activitySnapshotBox) Snapshot(context.Context) (hubactivity.Snapshot, bool) { + b.mu.Lock() + defer b.mu.Unlock() + return hubactivity.Snapshot{Items: append([]hubactivity.Item(nil), b.snap.Items...), Merged: append([]hubactivity.Merge(nil), b.snap.Merged...)}, true +} + +func testEventPublisher(opts ActivityOptions, box *activitySnapshotBox, clock *activityClock) *activityPublisher { + p := newActivityPublisher(ActivitySource{ + Options: func() ActivityOptions { return opts }, + Snapshot: box.Snapshot, + }, "", nil2Logger()) + p.now = clock.Now + return p +} + +func stagedItem(repo string, n int, stage string) hubactivity.Item { + return hubactivity.Item{Repo: repo, Number: n, Kind: "pr", Title: fmt.Sprintf("%s #%d", repo, n), Stage: stage, StageNumber: 10} +} + +func eventKeys(events []hubactivity.Event) []string { + out := make([]string, 0, len(events)) + for _, e := range events { + out = append(out, fmt.Sprintf("%s#%d:%s:%s>%s", e.Repo, e.Number, e.Type, e.From, e.Stage)) + } + return out +} + +func sameKeys(got, want []string) bool { + return fmt.Sprint(got) == fmt.Sprint(want) +} + +// Acceptance (#15964): a merge on the spoke rides on the very next +// heartbeat's activity block, alongside the stage hops and opens seen +// since the previous one. The first snapshot only seeds the baseline. +func TestActivityEventsStageHopsOpensAndMergeOnTheNextBeat(t *testing.T) { + box := &activitySnapshotBox{} + box.Set([]hubactivity.Item{stagedItem("o/r", 1, "ci_review"), stagedItem("o/r", 2, "ci_review")}) + clock := &activityClock{now: activityTestT0} + p := testEventPublisher(ActivityOptions{Enabled: true}, box, clock) + ctx := context.Background() + + if seed := p.heartbeatDigestLocked(ctx); len(seed.Events) != 0 { + t.Fatalf("first beat replayed the board as events: %v", eventKeys(seed.Events)) + } + + mergedAt := activityTestT0.Add(time.Minute) + box.Set([]hubactivity.Item{stagedItem("o/r", 1, "merge_queue"), stagedItem("o/r", 3, "actionable")}, + hubactivity.Merge{Repo: "o/r", Number: 2, MergedAt: mergedAt}) + clock.Advance(2 * time.Minute) + d := p.heartbeatDigestLocked(ctx) + + want := []string{"o/r#1:stage:ci_review>merge_queue", "o/r#3:opened:>actionable", "o/r#2:merged:ci_review>"} + if got := eventKeys(d.Events); !sameKeys(got, want) { + t.Fatalf("events = %v, want %v", got, want) + } + merge := d.Events[2] + if !merge.At.Equal(mergedAt) || merge.Title != "o/r #2" { + t.Fatalf("merge event = %+v, want the merge time and the item's last title", merge) + } + for i := 1; i < len(d.Events); i++ { + if d.Events[i].Seq <= d.Events[i-1].Seq { + t.Fatalf("sequence numbers not increasing: %v", d.Events) + } + } + + // The merge stays in the tracker's recent list: no second event. + clock.Advance(2 * time.Minute) + p.acknowledgeEventsLocked(d.Events[len(d.Events)-1].Seq) + if again := p.heartbeatDigestLocked(ctx); len(again.Events) != 0 { + t.Fatalf("an already reported merge was reported again: %v", eventKeys(again.Events)) + } +} + +// Events stay buffered (and are resent) until the hub acknowledges them; +// an acknowledgement drops exactly the events up to its sequence number. +func TestActivityEventsTrimmedOnlyByAcknowledgement(t *testing.T) { + box := &activitySnapshotBox{} + box.Set(nil) + p := testEventPublisher(ActivityOptions{Enabled: true}, box, &activityClock{now: activityTestT0}) + ctx := context.Background() + p.heartbeatDigestLocked(ctx) + + box.Set([]hubactivity.Item{stagedItem("o/r", 1, "a")}) + first := p.heartbeatDigestLocked(ctx) + box.Set([]hubactivity.Item{stagedItem("o/r", 1, "a"), stagedItem("o/r", 2, "a")}) + second := p.heartbeatDigestLocked(ctx) + if len(first.Events) != 1 || len(second.Events) != 2 { + t.Fatalf("unacknowledged events must be resent: %d then %d", len(first.Events), len(second.Events)) + } + + // An accepted response without an events ack (a hub before #15964, + // or one that restarted) keeps the buffer. + p.acknowledgeLocked(second.ETag) + p.acknowledgeEventsLocked(0) + marker := p.heartbeatDigestLocked(ctx) + if !marker.IsUnchangedMarker() || len(marker.Events) != 2 { + t.Fatalf("marker = %+v, want the etag only plus both pending events", marker) + } + + p.acknowledgeEventsLocked(first.Events[0].Seq) + if rest := p.heartbeatDigestLocked(ctx); len(rest.Events) != 1 || rest.Events[0].Number != 2 { + t.Fatalf("after a partial ack events = %v, want only #2", eventKeys(rest.Events)) + } + p.acknowledgeEventsLocked(second.Events[1].Seq) + if done := p.heartbeatDigestLocked(ctx); len(done.Events) != 0 { + t.Fatalf("after a full ack events = %v, want none", eventKeys(done.Events)) + } +} + +// The buffer holds at most hubactivity.MaxEvents: the oldest go first and +// the drop count is disclosed until an acknowledgement covers it. +func TestActivityEventsCapAt200DisclosesDrops(t *testing.T) { + box := &activitySnapshotBox{} + box.Set(nil) + p := testEventPublisher(ActivityOptions{Enabled: true, MaxItems: 1}, box, &activityClock{now: activityTestT0}) + ctx := context.Background() + p.heartbeatDigestLocked(ctx) + + var items []hubactivity.Item + for n := 1; n <= 250; n++ { + items = append(items, stagedItem("o/r", n, "a")) + } + box.Set(items) + d := p.heartbeatDigestLocked(ctx) + + if len(d.Events) != hubactivity.MaxEvents || d.EventsDropped != 50 { + t.Fatalf("events = %d dropped = %d, want %d and 50", len(d.Events), d.EventsDropped, hubactivity.MaxEvents) + } + if d.Events[0].Number != 51 || d.Events[len(d.Events)-1].Number != 250 { + t.Fatalf("kept #%d..#%d, want the newest (#51..#250)", d.Events[0].Number, d.Events[len(d.Events)-1].Number) + } + + p.acknowledgeEventsLocked(d.Events[len(d.Events)-1].Seq) + if after := p.heartbeatDigestLocked(ctx); len(after.Events) != 0 || after.EventsDropped != 0 { + t.Fatalf("after the ack: %d events, %d dropped; want both cleared", len(after.Events), after.EventsDropped) + } +} + +// Private-repository items yield no events unless the spoke shares them. +func TestActivityEventsExcludePrivateItemsUnlessShared(t *testing.T) { + for _, share := range []bool{false, true} { + box := &activitySnapshotBox{} + box.Set(nil) + p := testEventPublisher(ActivityOptions{Enabled: true, SharePrivate: share}, box, &activityClock{now: activityTestT0}) + ctx := context.Background() + p.heartbeatDigestLocked(ctx) + + secret := stagedItem("o/secret", 9, "a") + secret.Private = true + box.Set([]hubactivity.Item{secret, stagedItem("o/public", 1, "a")}, + hubactivity.Merge{Repo: "o/secret", Number: 8, MergedAt: activityTestT0, Private: true}) + d := p.heartbeatDigestLocked(ctx) + + var private int + for _, e := range d.Events { + if e.Repo == "o/secret" { + private++ + if !e.Private { + t.Fatalf("shared private event not marked private: %+v", e) + } + } + } + if want := map[bool]int{false: 0, true: 2}[share]; private != want { + t.Fatalf("share_private=%v: %d private events, want %d (%v)", share, private, want, eventKeys(d.Events)) + } + } +} + +// Turning publishing off forgets the buffer, so turning it back on starts +// from a fresh baseline instead of replaying stale movement. +func TestActivityEventsResetWhenDisabled(t *testing.T) { + opts := ActivityOptions{Enabled: true} + box := &activitySnapshotBox{} + box.Set(nil) + p := newActivityPublisher(ActivitySource{Options: func() ActivityOptions { return opts }, Snapshot: box.Snapshot}, "", nil2Logger()) + ctx := context.Background() + p.heartbeatDigestLocked(ctx) + box.Set([]hubactivity.Item{stagedItem("o/r", 1, "a")}) + p.heartbeatDigestLocked(ctx) + + opts.Enabled = false + if d := p.heartbeatDigestLocked(ctx); d != nil { + t.Fatalf("disabled publisher sent %+v", d) + } + opts.Enabled = true + box.Set([]hubactivity.Item{stagedItem("o/r", 1, "a"), stagedItem("o/r", 2, "a")}) + if d := p.heartbeatDigestLocked(ctx); len(d.Events) != 0 { + t.Fatalf("re-enabled publisher replayed %v", eventKeys(d.Events)) + } +} + +func TestRedactActivityAppliesToEvents(t *testing.T) { + d := hubactivity.Digest{Events: []hubactivity.Event{{Seq: 1, Type: hubactivity.EventMerged, Repo: "o/r", Number: 1, Title: "secret"}}, EventsDropped: 4} + + repos := redactActivity(d, map[string]bool{config.HeartbeatOmitRepos: true}) + if len(repos.Events) != 0 || repos.EventsDropped != 0 { + t.Fatalf("omit repos left events: %+v", repos) + } + titles := redactActivity(d, map[string]bool{config.HeartbeatOmitTaskTitles: true}) + if len(titles.Events) != 1 || titles.Events[0].Title != "" { + t.Fatalf("omit task titles left a title: %+v", titles.Events) + } + if d.Events[0].Title != "secret" { + t.Fatal("redaction modified the input") + } +} + +// An out-of-band update carries new events even when the digest itself is +// unchanged, as an unchanged marker, and applies the event ack. +func TestActivityUpdateSendsPendingEventsOnAMarker(t *testing.T) { + box := &activitySnapshotBox{} + box.Set([]hubactivity.Item{stagedItem("o/r", 1, "a")}) + clock := &activityClock{now: activityTestT0} + p := testEventPublisher(ActivityOptions{Enabled: true}, box, clock) + rec := &recordingPost{sent: make(chan *hubactivity.Digest, 4), status: 200, ok: true} + p.post = rec.post + ctx := context.Background() + + if !p.update(ctx) { + t.Fatal("first update not sent") + } + <-rec.sent + if p.update(ctx) { + t.Fatal("nothing changed: no update expected") + } + + // A merge of an item no longer tracked leaves the digest unchanged. + box.Set([]hubactivity.Item{stagedItem("o/r", 1, "a")}, hubactivity.Merge{Repo: "o/r", Number: 5, MergedAt: activityTestT0}) + if !p.update(ctx) { + t.Fatal("pending merge event not sent") + } + sent := <-rec.sent + if !sent.IsUnchangedMarker() || len(sent.Events) != 1 || sent.Events[0].Type != hubactivity.EventMerged { + t.Fatalf("update = %+v, want a marker carrying the merge", sent) + } + if len(p.events) != 0 { + t.Fatalf("acknowledged events still buffered: %v", eventKeys(p.events)) + } +} diff --git a/src/pkg/hub/spoke/activity_publisher.go b/src/pkg/hub/spoke/activity_publisher.go index 4e2ed4ec6e..40fdf1023c 100644 --- a/src/pkg/hub/spoke/activity_publisher.go +++ b/src/pkg/hub/spoke/activity_publisher.go @@ -8,6 +8,8 @@ import ( "log/slog" "net/http" "slices" + "strconv" + "strings" "sync" "sync/atomic" "time" @@ -29,6 +31,11 @@ import ( // activityUpdateMinInterval. Publishes that arrive while one is waiting // or in flight merge into it; the rest are dropped (the next heartbeat // carries the digest anyway). +// +// Both also carry the activity events (#15964): stage hops, opens and +// merges the publisher derives by diffing successive tracker snapshots, +// buffered (at most hubactivity.MaxEvents, oldest dropped first, the drop +// count disclosed) until the hub acknowledges them by sequence number. // ActivityUpdateHeader marks an activity-only update on POST /api/heartbeat. const ActivityUpdateHeader = "X-Hive-Activity-Update" @@ -57,11 +64,14 @@ type ActivityOptions struct { // ActivitySource is the StartHeartbeat callback that feeds the activity // digest. Options is read on every build so a config reload applies to the // next beat. Items returns the tracked items with Private set for -// private-repository rows (ok=false: no tracker yet). Subscribe, optional, -// registers a non-blocking hook run on every tracker publish. +// private-repository rows (ok=false: no tracker yet). Snapshot, optional, +// supersedes Items and also reports recently merged pull requests, which +// the publisher turns into merge events. Subscribe, optional, registers a +// non-blocking hook run on every tracker publish. type ActivitySource struct { Options func() ActivityOptions Items func(ctx context.Context) (items []hubactivity.Item, ok bool) + Snapshot func(ctx context.Context) (snap hubactivity.Snapshot, ok bool) Subscribe func(notify func()) } @@ -79,9 +89,16 @@ type ActivityUpdate struct { Activity *hubactivity.Digest `json:"activity"` } +// activityAck is what an accepted response acknowledged: the digest etag +// the hub holds and the highest event sequence number it holds. +type activityAck struct { + ETag string + Events uint64 +} + // activityPostFunc sends one activity-only update. ok is true only for an // accepted, verified response; status is the HTTP status (0 = no response). -type activityPostFunc func(ctx context.Context, d *hubactivity.Digest) (ack string, status int, ok bool) +type activityPostFunc func(ctx context.Context, d *hubactivity.Digest) (ack activityAck, status int, ok bool) type activityPublisher struct { src ActivitySource @@ -99,6 +116,18 @@ type activityPublisher struct { acked string lastUpdate time.Time backoff time.Time + + // Event state (#15964), also guarded by mu. prevItems is nil until the + // first snapshot seeds it, so startup never replays the board as opens. + events []hubactivity.Event + eventsDropped int + // sentDropped and sentMaxSeq are what the latest build carried, so an + // acknowledgement covering it clears the disclosed drops. + sentDropped int + sentMaxSeq uint64 + nextSeq uint64 + prevItems map[string]hubactivity.Item + prevMerges map[string]bool } // activityPub is the running heartbeat loop's publisher (nil when no @@ -115,31 +144,56 @@ func newActivityPublisher(src ActivitySource, hubURL string, logger *slog.Logger now: time.Now, after: time.After, kick: make(chan struct{}, 1), + // Sequence numbers start from the wall clock so a restarted spoke + // numbers past what the hub acknowledged from its previous run. + nextSeq: uint64(max(time.Now().UnixMilli(), 1)) * 1000, } - p.post = func(ctx context.Context, d *hubactivity.Digest) (string, int, bool) { - return postActivityUpdate(ctx, hubURL, heartbeatHiveID(), d, logger) + p.post = func(ctx context.Context, d *hubactivity.Digest) (activityAck, int, bool) { + return postActivityUpdateAck(ctx, hubURL, heartbeatHiveID(), d, logger) } return p } // build returns the current digest, redacted for hub.heartbeat_omit. // ok is false when publishing is off or the tracker has nothing to give. +// It also records the events since the previous build and attaches the +// unacknowledged ones. Callers hold p.mu. func (p *activityPublisher) build(ctx context.Context) (hubactivity.Digest, bool) { - if p.src.Options == nil || p.src.Items == nil { + if p.src.Options == nil || (p.src.Items == nil && p.src.Snapshot == nil) { return hubactivity.Digest{}, false } opts := p.src.Options() if !opts.Enabled { + p.resetEventsLocked() return hubactivity.Digest{}, false } - items, ok := activityItemsWithTimeout(ctx, p.src.Items, activityItemsTimeout, p.logger) + snap, ok := p.readSnapshot(ctx) if !ok { return hubactivity.Digest{}, false } - d := hubactivity.Build(items, hubactivity.BuildOptions{MaxItems: opts.MaxItems, SharePrivate: opts.SharePrivate, Now: p.now()}) + now := p.now() + p.observeLocked(snap, opts.SharePrivate, now) + d := hubactivity.Build(snap.Items, hubactivity.BuildOptions{MaxItems: opts.MaxItems, SharePrivate: opts.SharePrivate, Now: now}) + d.Events = slices.Clone(p.events) + d.EventsDropped = p.eventsDropped + p.sentDropped = p.eventsDropped + p.sentMaxSeq = 0 + if n := len(p.events); n > 0 { + p.sentMaxSeq = p.events[n-1].Seq + } return redactActivity(d, omitSet()), true } +// readSnapshot reads the tracker through Snapshot when set, else Items, +// bounded by activityItemsTimeout. +func (p *activityPublisher) readSnapshot(ctx context.Context) (hubactivity.Snapshot, bool) { + if p.src.Snapshot != nil { + return callWithTimeout(ctx, p.src.Snapshot, activityItemsTimeout, p.logger) + } + items, ok := activityItemsWithTimeout(ctx, p.src.Items, activityItemsTimeout, p.logger) + return hubactivity.Snapshot{Items: items}, ok +} + // heartbeatDigestLocked is the activity block for a heartbeat: nil when // there is nothing to publish, an unchanged marker when the hub already // holds this etag, else the full digest. Callers hold p.mu. @@ -149,17 +203,119 @@ func (p *activityPublisher) heartbeatDigestLocked(ctx context.Context) *hubactiv return nil } if d.ETag != "" && d.ETag == p.acked { - return &hubactivity.Digest{ETag: d.ETag, GeneratedAt: d.GeneratedAt} + return unchangedMarker(d) } return &d } +// unchangedMarker is d reduced to its etag and generated_at, keeping the +// events it carries. +func unchangedMarker(d hubactivity.Digest) *hubactivity.Digest { + return &hubactivity.Digest{ETag: d.ETag, GeneratedAt: d.GeneratedAt, Events: d.Events, EventsDropped: d.EventsDropped} +} + // acknowledgeLocked records the etag the hub reported holding. Callers hold // p.mu and call it only for an accepted, verified response. func (p *activityPublisher) acknowledgeLocked(etag string) { p.acked = etag } +// acknowledgeEventsLocked drops the buffered events up to the sequence +// number the hub reported holding (0 = none, e.g. a hub before #15964 +// or one that restarted: the buffer is kept and resent). Once the ack +// covers the latest build, the drops it disclosed are cleared. Callers +// hold p.mu and call it only for an accepted, verified response. +func (p *activityPublisher) acknowledgeEventsLocked(seq uint64) { + if seq == 0 { + return + } + p.events = slices.DeleteFunc(p.events, func(e hubactivity.Event) bool { return e.Seq <= seq }) + if seq >= p.sentMaxSeq { + p.eventsDropped = max(p.eventsDropped-p.sentDropped, 0) + p.sentDropped = 0 + } +} + +// resetEventsLocked forgets all event state, so re-enabling publishing +// starts from a fresh baseline instead of replaying stale movement. +func (p *activityPublisher) resetEventsLocked() { + p.events, p.eventsDropped, p.sentDropped, p.sentMaxSeq = nil, 0, 0, 0 + p.prevItems, p.prevMerges = nil, nil +} + +// observeLocked diffs snap against the previous snapshot and buffers the +// stage hops, opens and merges it finds. Private-repository items yield no +// events unless sharePrivate. The first snapshot only seeds the baseline. +func (p *activityPublisher) observeLocked(snap hubactivity.Snapshot, sharePrivate bool, now time.Time) { + cur := make(map[string]hubactivity.Item, len(snap.Items)) + for _, it := range snap.Items { + cur[activityEventKey(it.Repo, it.Number)] = it + } + merges := make(map[string]bool, len(snap.Merged)) + for _, m := range snap.Merged { + merges[activityEventKey(m.Repo, m.Number)] = true + } + seeded := p.prevItems != nil + prev, prevMerges := p.prevItems, p.prevMerges + p.prevItems, p.prevMerges = cur, merges + if !seeded { + return + } + for _, it := range snap.Items { + if it.Private && !sharePrivate { + continue + } + e := hubactivity.Event{Repo: it.Repo, Number: it.Number, Kind: it.Kind, Title: it.Title, Stage: it.Stage, At: now, Private: it.Private} + was, had := prev[activityEventKey(it.Repo, it.Number)] + switch { + case !had: + e.Type = hubactivity.EventOpened + case was.Stage != it.Stage && it.Stage != "": + e.Type = hubactivity.EventStage + e.From = was.Stage + default: + continue + } + p.pushEventLocked(e) + } + for _, m := range snap.Merged { + key := activityEventKey(m.Repo, m.Number) + if prevMerges[key] || (m.Private && !sharePrivate) { + continue + } + e := hubactivity.Event{Type: hubactivity.EventMerged, Repo: m.Repo, Number: m.Number, Kind: "pr", At: m.MergedAt, Private: m.Private} + if e.At.IsZero() { + e.At = now + } + if was, ok := prev[key]; ok { + e.Title, e.From = was.Title, was.Stage + } else if it, ok := cur[key]; ok { + e.Title, e.From = it.Title, it.Stage + } + p.pushEventLocked(e) + } +} + +// pushEventLocked numbers and buffers one event, dropping the oldest past +// hubactivity.MaxEvents and counting the drop. +func (p *activityPublisher) pushEventLocked(e hubactivity.Event) { + e.Seq = p.nextSeq + p.nextSeq++ + n, ok := hubactivity.NormalizeEvent(e) + if !ok { + return + } + p.events = append(p.events, n) + if over := len(p.events) - hubactivity.MaxEvents; over > 0 { + p.events = slices.Delete(p.events, 0, over) + p.eventsDropped += over + } +} + +func activityEventKey(repo string, n int) string { + return strings.ToLower(repo) + "#" + strconv.Itoa(n) +} + // notify signals a tracker publish without blocking: a publish arriving // while one is already pending merges into it. func (p *activityPublisher) notify() { @@ -206,18 +362,24 @@ func (p *activityPublisher) update(ctx context.Context) bool { return false } d, ok := p.build(ctx) - if !ok || d.ETag == p.acked { + if !ok || (d.ETag == p.acked && len(d.Events) == 0) { return false } + send := &d + if d.ETag != "" && d.ETag == p.acked { + // Only events are new: carry them on an unchanged marker. + send = unchangedMarker(d) + } p.lastUpdate = now - ack, status, sent := p.post(ctx, &d) + ack, status, sent := p.post(ctx, send) if !sent { if activityUpdateUnsupported(status) { p.backoff = now.Add(activityUpdateBackoff) } return false } - p.acknowledgeLocked(ack) + p.acknowledgeLocked(ack.ETag) + p.acknowledgeEventsLocked(ack.Events) return true } @@ -240,12 +402,19 @@ func redactActivity(d hubactivity.Digest, omit map[string]bool) hubactivity.Dige case omit[config.HeartbeatOmitRepos]: d.Items = nil d.Truncated = 0 + d.Events = nil + d.EventsDropped = 0 case omit[config.HeartbeatOmitTaskTitles]: items := slices.Clone(d.Items) for i := range items { items[i].Title = "" } d.Items = items + events := slices.Clone(d.Events) + for i := range events { + events[i].Title = "" + } + d.Events = events default: return d } @@ -256,29 +425,36 @@ func redactActivity(d hubactivity.Digest, omit map[string]bool) hubactivity.Dige // activityItemsWithTimeout runs items with an upper bound, recovering a // panic, so the tracker can never stall or crash the heartbeat loop. func activityItemsWithTimeout(ctx context.Context, items func(context.Context) ([]hubactivity.Item, bool), timeout time.Duration, logger *slog.Logger) ([]hubactivity.Item, bool) { - type result struct { - items []hubactivity.Item - ok bool - } + return callWithTimeout(ctx, items, timeout, logger) +} + +type timedRead[T any] struct { + items T + ok bool +} + +// callWithTimeout is activityItemsWithTimeout for any tracker read. +func callWithTimeout[T any](ctx context.Context, read func(context.Context) (T, bool), timeout time.Duration, logger *slog.Logger) (T, bool) { cctx, cancel := context.WithTimeout(ctx, timeout) defer cancel() - done := make(chan result, 1) + done := make(chan timedRead[T], 1) go func() { defer func() { if r := recover(); r != nil { logger.Warn("hub activity digest: tracker read panicked; sending no activity block", "recover", r) - done <- result{} + done <- timedRead[T]{} } }() - its, ok := items(cctx) - done <- result{items: its, ok: ok} + its, ok := read(cctx) + done <- timedRead[T]{items: its, ok: ok} }() select { case r := <-done: return r.items, r.ok case <-cctx.Done(): logger.Warn("hub activity digest: tracker read timed out; sending no activity block", "timeout", timeout.String()) - return nil, false + var zero T + return zero, false } } @@ -296,18 +472,25 @@ func heartbeatHiveID() string { // bearer and verifies the hub's signed acknowledgement the same way as a // heartbeat response. func postActivityUpdate(ctx context.Context, hubURL, hiveID string, d *hubactivity.Digest, logger *slog.Logger) (string, int, bool) { + ack, status, ok := postActivityUpdateAck(ctx, hubURL, hiveID, d, logger) + return ack.ETag, status, ok +} + +// postActivityUpdateAck is postActivityUpdate also returning the event +// acknowledgement. +func postActivityUpdateAck(ctx context.Context, hubURL, hiveID string, d *hubactivity.Digest, logger *slog.Logger) (activityAck, int, bool) { if hubURL == "" || hiveID == "" || d == nil { - return "", 0, false + return activityAck{}, 0, false } body, err := json.Marshal(ActivityUpdatePayload{Update: &ActivityUpdate{HiveID: hiveID, Activity: d}}) if err != nil { - return "", 0, false + return activityAck{}, 0, false } reqCtx, cancel := context.WithTimeout(ctx, heartbeatTimeout) defer cancel() req, err := http.NewRequestWithContext(reqCtx, http.MethodPost, hubURL+"/api/heartbeat", bytes.NewReader(body)) if err != nil { - return "", 0, false + return activityAck{}, 0, false } req.Header.Set("Content-Type", "application/json") req.Header.Set(ActivityUpdateHeader, "1") @@ -317,25 +500,25 @@ func postActivityUpdate(ctx context.Context, hubURL, hiveID string, d *hubactivi resp, err := http.DefaultClient.Do(req) if err != nil { logger.Debug("hub activity update unreachable", "error", err) - return "", 0, false + return activityAck{}, 0, false } defer func() { _ = resp.Body.Close() }() if resp.StatusCode >= 300 { logger.Debug("hub activity update rejected", "status", resp.StatusCode) - return "", resp.StatusCode, false + return activityAck{}, resp.StatusCode, false } raw, err := io.ReadAll(io.LimitReader(resp.Body, maxPayloadBytes)) if err != nil { - return "", resp.StatusCode, false + return activityAck{}, resp.StatusCode, false } result := spokeHeartbeatVerifier(logger).Verify(SpokeSSOPublicKeys(), hiveID, raw, resp.Header.Get(spokesig.SigHeader), time.Now()) if !result.Accepted { logger.Warn("hub activity update response REJECTED by signature verification", "reason", result.Reason, "hive_id", hiveID) - return "", resp.StatusCode, false + return activityAck{}, resp.StatusCode, false } var hbResp HeartbeatResponse if err := json.Unmarshal(raw, &hbResp); err != nil { - return "", resp.StatusCode, false + return activityAck{}, resp.StatusCode, false } - return hbResp.ActivityETag, resp.StatusCode, true + return activityAck{ETag: hbResp.ActivityETag, Events: hbResp.ActivityEventsAck}, resp.StatusCode, true } diff --git a/src/pkg/hub/spoke/activity_publisher_test.go b/src/pkg/hub/spoke/activity_publisher_test.go index 6da771926f..70a67ed6ba 100644 --- a/src/pkg/hub/spoke/activity_publisher_test.go +++ b/src/pkg/hub/spoke/activity_publisher_test.go @@ -264,15 +264,19 @@ type recordingPost struct { ok bool } -func (r *recordingPost) post(_ context.Context, d *hubactivity.Digest) (string, int, bool) { +func (r *recordingPost) post(_ context.Context, d *hubactivity.Digest) (activityAck, int, bool) { r.sent <- d if r.gate != nil { <-r.gate } if !r.ok { - return "", r.status, false + return activityAck{}, r.status, false } - return d.ETag, r.status, true + ack := activityAck{ETag: d.ETag} + if n := len(d.Events); n > 0 { + ack.Events = d.Events[n-1].Seq + } + return ack, r.status, true } func TestActivityUpdatesCoalesceToOnePer15sWithBackpressure(t *testing.T) { diff --git a/src/pkg/hub/spoke/heartbeat.go b/src/pkg/hub/spoke/heartbeat.go index 858be8e59e..00bf3f9225 100644 --- a/src/pkg/hub/spoke/heartbeat.go +++ b/src/pkg/hub/spoke/heartbeat.go @@ -1875,6 +1875,7 @@ func postHeartbeatToHub(ctx context.Context, hubURL string, payload *HeartbeatPa } if pub != nil { pub.acknowledgeLocked(hbResp.ActivityETag) + pub.acknowledgeEventsLocked(hbResp.ActivityEventsAck) } return &hbResp } @@ -2890,6 +2891,9 @@ type HeartbeatResponse struct { // ActivityETag is the activity digest etag the hub acknowledged ("" = // it holds none), decoded from hub.HeartbeatResponse.ActivityETag. ActivityETag string `json:"activity_etag,omitempty"` + // ActivityEventsAck is the highest activity event sequence number the + // hub acknowledged, decoded from hub.HeartbeatResponse.ActivityEventsAck. + ActivityEventsAck uint64 `json:"activity_events_ack,omitempty"` } // HubBanner is a message from the hub admin displayed on spoke dashboards. diff --git a/src/pkg/hubactivity/events.go b/src/pkg/hubactivity/events.go new file mode 100644 index 0000000000..f2400a59d2 --- /dev/null +++ b/src/pkg/hubactivity/events.go @@ -0,0 +1,115 @@ +package hubactivity + +import ( + "slices" + "time" +) + +// Activity events (RFC hivecommons/hive#15861 §A/§B, child #15964). The +// digest says where every item is now; events say how items moved since the +// hub last acknowledged them — stage hops, opens and merges — so the hub +// board can animate movement and show merge toasts. Both ends bound them: +// the spoke buffers at most MaxEvents, dropping the oldest and disclosing +// the count, and the hub keeps a fleet-wide in-memory ring. + +const ( + // MaxEvents caps the events one activity block carries (and the spoke + // buffers between acknowledgements). The oldest are dropped first. + MaxEvents = 200 +) + +// Event types. +const ( + EventOpened = "opened" + EventStage = "stage" + EventMerged = "merged" +) + +var allowedEventTypes = []string{EventOpened, EventStage, EventMerged} + +// Event is one stage hop, open or merge. Seq is the spoke's monotonic +// sequence number; the hub acknowledges the highest Seq it holds and the +// spoke trims its buffer up to it. +type Event struct { + Seq uint64 `json:"seq"` + Type string `json:"type"` + Repo string `json:"repo"` + Number int `json:"n"` + Kind string `json:"kind,omitempty"` + Title string `json:"title,omitempty"` + From string `json:"from,omitempty"` + Stage string `json:"stage,omitempty"` + At time.Time `json:"at"` + // Private marks a private-repository item; such events exist only + // when the spoke shares private items. + Private bool `json:"private,omitempty"` +} + +// Merge is one merged pull request the spoke's tracker reports. +type Merge struct { + Repo string + Number int + MergedAt time.Time + Private bool +} + +// Snapshot is what a spoke's tracker gives the activity publisher: the +// open items and the recently merged pull requests. +type Snapshot struct { + Items []Item + Merged []Merge +} + +// NormalizeEvent bounds and validates one event; ok is false for an event +// that must be dropped (unknown type, repo not owner/name, no number or no +// sequence number). +func NormalizeEvent(e Event) (Event, bool) { + out := e + out.Type = oneOf(cleanToken(e.Type), allowedEventTypes) + out.Repo = cleanToken(e.Repo) + if out.Type == "" || e.Seq == 0 || !repoPattern.MatchString(out.Repo) || e.Number <= 0 || e.Number > maxCount { + return Event{}, false + } + out.Kind = oneOf(cleanToken(e.Kind), allowedKinds) + out.Title = cleanText(e.Title, MaxTitleRunes) + out.From = slug(e.From) + out.Stage = slug(e.Stage) + out.At = e.At.UTC() + return out, true +} + +// SanitizeEvents normalizes events received from a spoke, drops malformed +// and duplicate (by Seq) rows, orders them by Seq and keeps at most +// MaxEvents, dropping the oldest. dropped is how many the cap removed. The +// input is not modified. +func SanitizeEvents(events []Event) (out []Event, dropped int) { + seen := map[uint64]bool{} + for _, e := range events { + n, ok := NormalizeEvent(e) + if !ok || seen[n.Seq] { + continue + } + seen[n.Seq] = true + out = append(out, n) + } + slices.SortStableFunc(out, func(a, b Event) int { + switch { + case a.Seq < b.Seq: + return -1 + case a.Seq > b.Seq: + return 1 + } + return 0 + }) + if len(out) > MaxEvents { + dropped = len(out) - MaxEvents + out = slices.Clone(out[dropped:]) + } + return out, dropped +} + +// ClampCount bounds a spoke-claimed count the way Sanitize bounds every +// integer in the digest. +func ClampCount(n int) int { + return clampCount(n) +} diff --git a/src/pkg/hubactivity/hubactivity.go b/src/pkg/hubactivity/hubactivity.go index 97bb2f7acd..408a8e4c13 100644 --- a/src/pkg/hubactivity/hubactivity.go +++ b/src/pkg/hubactivity/hubactivity.go @@ -97,10 +97,16 @@ type Digest struct { Items []Item `json:"items,omitempty"` // Truncated is how many eligible items the cap left out. Truncated int `json:"truncated,omitempty"` + // Events are the stage hops, opens and merges since the hub last + // acknowledged (#15964), oldest first and at most MaxEvents. + // EventsDropped discloses how many the cap dropped since then. Neither + // is part of the content etag, and an unchanged marker may carry them. + Events []Event `json:"events,omitempty"` + EventsDropped int `json:"events_dropped,omitempty"` } // IsUnchangedMarker reports whether d only re-asserts a previously sent -// etag. +// etag. Events do not count: a marker may carry them. func (d Digest) IsUnchangedMarker() bool { return d.ETag != "" && d.Totals == nil && len(d.Items) == 0 && d.Truncated == 0 } From 53d578842f1882eaf11238689a8d274492e6a4ef Mon Sep 17 00:00:00 2001 From: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Sat, 31 Oct 2026 10:08:44 -0400 Subject: [PATCH 2/6] test(hubactivity): cover activity event normalization and sanitizing The coverage gate failed with hubactivity at 76.3% (floor 90%) because events.go landed without tests. Cover NormalizeEvent (bounding, UTC normalization, every rejection branch), SanitizeEvents (dedupe by Seq, ordering, MaxEvents cap with oldest-dropped disclosure, input not modified) and ClampCount. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- src/pkg/hubactivity/events_test.go | 151 +++++++++++++++++++++++++++++ 1 file changed, 151 insertions(+) create mode 100644 src/pkg/hubactivity/events_test.go diff --git a/src/pkg/hubactivity/events_test.go b/src/pkg/hubactivity/events_test.go new file mode 100644 index 0000000000..4788d281c3 --- /dev/null +++ b/src/pkg/hubactivity/events_test.go @@ -0,0 +1,151 @@ +package hubactivity + +import ( + "slices" + "testing" + "time" +) + +func TestNormalizeEvent_BoundsAndValidates(t *testing.T) { + loc := time.FixedZone("x", 3600) + in := Event{ + Seq: 7, + Type: " STAGE\t", + Repo: " o/r ", + Number: 42, + Kind: "PR", + Title: "hello\u2028world", + From: "Needs Review!", + Stage: "Actionable", + At: t0.In(loc), + Private: true, + } + got, ok := NormalizeEvent(in) + if !ok { + t.Fatal("expected event to be accepted") + } + if got.Type != EventStage || got.Repo != "o/r" || got.Kind != "pr" { + t.Errorf("tokens not normalized: %+v", got) + } + if got.Title == in.Title || got.Title == "" { + t.Errorf("title not cleaned: %q", got.Title) + } + if got.From != slug(in.From) || got.Stage != slug(in.Stage) || got.From == "" || got.Stage == "" { + t.Errorf("from/stage not slugged: %q %q", got.From, got.Stage) + } + if got.At.Location() != time.UTC || !got.At.Equal(t0) { + t.Errorf("At not UTC-normalized: %v", got.At) + } + if !got.Private || got.Seq != 7 || got.Number != 42 { + t.Errorf("fields not carried: %+v", got) + } +} + +func TestNormalizeEvent_RejectsMalformed(t *testing.T) { + valid := Event{Seq: 1, Type: EventOpened, Repo: "o/r", Number: 1} + if _, ok := NormalizeEvent(valid); !ok { + t.Fatal("baseline event must be accepted") + } + cases := map[string]func(e *Event){ + "unknown type": func(e *Event) { e.Type = "closed" }, + "empty type": func(e *Event) { e.Type = "" }, + "zero seq": func(e *Event) { e.Seq = 0 }, + "bad repo": func(e *Event) { e.Repo = "not-a-repo" }, + "empty repo": func(e *Event) { e.Repo = "" }, + "zero number": func(e *Event) { e.Number = 0 }, + "neg number": func(e *Event) { e.Number = -3 }, + "huge number": func(e *Event) { e.Number = maxCount + 1 }, + } + for name, mutate := range cases { + e := valid + mutate(&e) + got, ok := NormalizeEvent(e) + if ok { + t.Errorf("%s: expected rejection, got %+v", name, got) + } + if got != (Event{}) { + t.Errorf("%s: rejected event must be zero, got %+v", name, got) + } + } +} + +func TestNormalizeEvent_UnknownKindDropped(t *testing.T) { + got, ok := NormalizeEvent(Event{Seq: 1, Type: EventMerged, Repo: "o/r", Number: 1, Kind: "discussion"}) + if !ok { + t.Fatal("expected acceptance") + } + if got.Kind != "" { + t.Errorf("unknown kind should be blanked, got %q", got.Kind) + } +} + +func TestSanitizeEvents_DropsMalformedDedupesAndOrders(t *testing.T) { + in := []Event{ + {Seq: 3, Type: EventStage, Repo: "o/r", Number: 3}, + {Seq: 0, Type: EventStage, Repo: "o/r", Number: 9}, // no seq + {Seq: 1, Type: EventOpened, Repo: "o/r", Number: 1}, + {Seq: 3, Type: EventMerged, Repo: "o/r", Number: 33}, // duplicate seq + {Seq: 2, Type: "bogus", Repo: "o/r", Number: 2}, // bad type + {Seq: 2, Type: EventMerged, Repo: "o/r", Number: 2}, + } + snapshot := slices.Clone(in) + + out, dropped := SanitizeEvents(in) + if dropped != 0 { + t.Errorf("dropped = %d, want 0 (cap not hit)", dropped) + } + var seqs []uint64 + var nums []int + for _, e := range out { + seqs = append(seqs, e.Seq) + nums = append(nums, e.Number) + } + if !slices.Equal(seqs, []uint64{1, 2, 3}) { + t.Errorf("seqs = %v, want [1 2 3]", seqs) + } + if !slices.Equal(nums, []int{1, 2, 3}) { + t.Errorf("first occurrence must win: numbers = %v, want [1 2 3]", nums) + } + if !slices.Equal(in, snapshot) { + t.Error("SanitizeEvents must not modify its input") + } +} + +func TestSanitizeEvents_CapsAtMaxEventsDroppingOldest(t *testing.T) { + total := MaxEvents + 25 + in := make([]Event, 0, total) + for i := total; i >= 1; i-- { // reverse order to exercise sorting too + in = append(in, Event{Seq: uint64(i), Type: EventStage, Repo: "o/r", Number: i}) + } + + out, dropped := SanitizeEvents(in) + if dropped != 25 { + t.Errorf("dropped = %d, want 25", dropped) + } + if len(out) != MaxEvents { + t.Fatalf("len = %d, want %d", len(out), MaxEvents) + } + if out[0].Seq != 26 || out[len(out)-1].Seq != uint64(total) { + t.Errorf("kept range = [%d, %d], want [26, %d]", out[0].Seq, out[len(out)-1].Seq, total) + } + for i := 1; i < len(out); i++ { + if out[i-1].Seq >= out[i].Seq { + t.Fatalf("not strictly ascending at %d: %d >= %d", i, out[i-1].Seq, out[i].Seq) + } + } +} + +func TestSanitizeEvents_Empty(t *testing.T) { + out, dropped := SanitizeEvents(nil) + if len(out) != 0 || dropped != 0 { + t.Errorf("got %v, %d; want empty, 0", out, dropped) + } +} + +func TestClampCount(t *testing.T) { + for in, want := range map[int]int{-5: 0, 0: 0, 7: 7, maxCount: maxCount, maxCount + 1: maxCount} { + if got := ClampCount(in); got != want { + t.Errorf("ClampCount(%d) = %d, want %d", in, got, want) + } + } +} From f0a524d7b39945cfebebbb314b261a2dcf168035 Mon Sep 17 00:00:00 2001 From: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Sat, 31 Oct 2026 10:22:24 -0400 Subject: [PATCH 3/6] fix(hubactivity): fold event type and kind tokens to lower case before validation Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- src/pkg/hubactivity/events.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/pkg/hubactivity/events.go b/src/pkg/hubactivity/events.go index f2400a59d2..598fb486c4 100644 --- a/src/pkg/hubactivity/events.go +++ b/src/pkg/hubactivity/events.go @@ -2,6 +2,7 @@ package hubactivity import ( "slices" + "strings" "time" ) @@ -65,12 +66,12 @@ type Snapshot struct { // sequence number). func NormalizeEvent(e Event) (Event, bool) { out := e - out.Type = oneOf(cleanToken(e.Type), allowedEventTypes) + out.Type = oneOf(strings.ToLower(cleanToken(e.Type)), allowedEventTypes) out.Repo = cleanToken(e.Repo) if out.Type == "" || e.Seq == 0 || !repoPattern.MatchString(out.Repo) || e.Number <= 0 || e.Number > maxCount { return Event{}, false } - out.Kind = oneOf(cleanToken(e.Kind), allowedKinds) + out.Kind = oneOf(strings.ToLower(cleanToken(e.Kind)), allowedKinds) out.Title = cleanText(e.Title, MaxTitleRunes) out.From = slug(e.From) out.Stage = slug(e.Stage) From e3dbf248ebdd652f7c9e3a9d5266a70e21dd08a9 Mon Sep 17 00:00:00 2001 From: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Sat, 31 Oct 2026 10:34:03 -0400 Subject: [PATCH 4/6] fix(hub): count a resent batch's drop disclosure once The spoke's events_dropped is cumulative until an ack covers its latest build, so a batch resent after a lost ack restates the drops the hub already counted. The ring now records the count each counted batch claimed per hive and, when the batch holds an already-acknowledged event (proof the spoke never saw the ack), adds only the increase over it; a batch of only new events marks a spoke that trimmed and reset, so its disclosure counts in full. The dedup test now asserts 3 drops for 3 real drops across the resend, and covers a grown resend and a post-ack batch. Hive-Fix-Attempt: 1/6 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- src/pkg/hub/activity_events.go | 24 ++++++++++++++++++++---- src/pkg/hub/activity_events_test.go | 24 +++++++++++++++++++++--- 2 files changed, 41 insertions(+), 7 deletions(-) diff --git a/src/pkg/hub/activity_events.go b/src/pkg/hub/activity_events.go index 0a34e41381..226d386ed7 100644 --- a/src/pkg/hub/activity_events.go +++ b/src/pkg/hub/activity_events.go @@ -35,8 +35,14 @@ type activityEventHive struct { lastSeq uint64 // dropped sums the drops the spoke disclosed and the hub's own cap // applied to its batches. - dropped int - lastSeen time.Time + dropped int + // disclosed is the drop count the latest counted batch claimed (the + // spoke's disclosure plus the hub's cap). The spoke's disclosure is + // cumulative until an ack covers its latest build, so a batch resent + // after a lost ack restates the drops already counted; only the + // increase over disclosed is added then. + disclosed int + lastSeen time.Time } // activityEventRing is guarded by activityStore.mu. The zero value is @@ -63,9 +69,13 @@ func (r *activityEventRing) ingest(hiveID string, events []hubactivity.Event, dr r.byHive[hiveID] = h } h.lastSeen = now - fresh := 0 + fresh, restated := 0, false for _, e := range clean { if e.Seq <= h.lastSeq { + // The spoke trims acknowledged events, so holding one proves + // it never saw the ack and its drop disclosure is cumulative + // over the one already counted. + restated = true continue } h.lastSeq = e.Seq @@ -75,7 +85,13 @@ func (r *activityEventRing) ingest(hiveID string, events []hubactivity.Event, dr if fresh == 0 { return } - h.dropped = hubactivity.ClampCount(h.dropped + hubactivity.ClampCount(dropped) + capped) + claimed := hubactivity.ClampCount(hubactivity.ClampCount(dropped) + capped) + add := claimed + if restated { + add = max(claimed-h.disclosed, 0) + } + h.disclosed = claimed + h.dropped = hubactivity.ClampCount(h.dropped + add) if over := len(r.entries) - activityEventRingSize; over > 0 { r.entries = slices.Delete(r.entries, 0, over) } diff --git a/src/pkg/hub/activity_events_test.go b/src/pkg/hub/activity_events_test.go index 8caedc09ef..1e6a85e476 100644 --- a/src/pkg/hub/activity_events_test.go +++ b/src/pkg/hub/activity_events_test.go @@ -138,7 +138,8 @@ func TestActivityEventRing_DedupesByAcknowledgedSeq(t *testing.T) { batch.EventsDropped = 3 st.record("hive-a", batch, activityT0) - // The spoke lost the ack and resends the same batch plus one new event. + // The spoke lost the ack and resends the same batch plus one new event; + // its cumulative disclosure restates the 3 drops already counted. resend := withEvents(batch, append(batch.Events, activityEv(12, hubactivity.EventMerged, "o/a", 1))...) st.record("hive-a", resend, activityT0.Add(time.Minute)) // A batch with nothing new does not count its drops again. @@ -151,8 +152,25 @@ func TestActivityEventRing_DedupesByAcknowledgedSeq(t *testing.T) { if got := st.eventsAck("hive-a"); got != 12 { t.Fatalf("ack = %d, want 12", got) } - if snap.dropped["hive-a"] != 6 { - t.Fatalf("disclosed drops = %d, want 3 per batch with new events (6)", snap.dropped["hive-a"]) + if snap.dropped["hive-a"] != 3 { + t.Fatalf("disclosed drops = %d, want 3 (a resend restates, not repeats, them)", snap.dropped["hive-a"]) + } + + // A resend whose disclosure grew counts only the increase. + grown := withEvents(resend, append(resend.Events, activityEv(13, hubactivity.EventStage, "o/a", 1))...) + grown.EventsDropped = 5 + st.record("hive-a", grown, activityT0.Add(3*time.Minute)) + if got := st.eventsSnapshot([]string{"hive-a"}).dropped["hive-a"]; got != 5 { + t.Fatalf("drops after grown resend = %d, want 5", got) + } + + // After the ack lands the spoke trims and resets its disclosure, so a + // batch of only new events discloses only new drops, counted in full. + acked := withEvents(activityDigest("o/a", 1), activityEv(14, hubactivity.EventStage, "o/a", 1)) + acked.EventsDropped = 2 + st.record("hive-a", acked, activityT0.Add(4*time.Minute)) + if got := st.eventsSnapshot([]string{"hive-a"}).dropped["hive-a"]; got != 7 { + t.Fatalf("drops after acked batch = %d, want 7", got) } } From c8dca0e0353693f4d54e7d71f9e5264f2c106d8c Mon Sep 17 00:00:00 2001 From: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Sat, 31 Oct 2026 10:39:39 -0400 Subject: [PATCH 5/6] test(hubactivity): feed NormalizeEvent valid stage slugs slug() only accepts lowercase [a-z0-9][a-z0-9_.-]* so "Needs Review!" and "Actionable" correctly normalize to ""; the test asserted the opposite. Exercise trimming with padded valid slugs and assert that invalid slugs blank the field without dropping the event. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- src/pkg/hubactivity/events_test.go | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/pkg/hubactivity/events_test.go b/src/pkg/hubactivity/events_test.go index 4788d281c3..2d7ad9ac90 100644 --- a/src/pkg/hubactivity/events_test.go +++ b/src/pkg/hubactivity/events_test.go @@ -15,8 +15,8 @@ func TestNormalizeEvent_BoundsAndValidates(t *testing.T) { Number: 42, Kind: "PR", Title: "hello\u2028world", - From: "Needs Review!", - Stage: "Actionable", + From: " needs-review ", + Stage: "actionable\n", At: t0.In(loc), Private: true, } @@ -30,7 +30,7 @@ func TestNormalizeEvent_BoundsAndValidates(t *testing.T) { if got.Title == in.Title || got.Title == "" { t.Errorf("title not cleaned: %q", got.Title) } - if got.From != slug(in.From) || got.Stage != slug(in.Stage) || got.From == "" || got.Stage == "" { + if got.From != "needs-review" || got.Stage != "actionable" { t.Errorf("from/stage not slugged: %q %q", got.From, got.Stage) } if got.At.Location() != time.UTC || !got.At.Equal(t0) { @@ -39,6 +39,16 @@ func TestNormalizeEvent_BoundsAndValidates(t *testing.T) { if !got.Private || got.Seq != 7 || got.Number != 42 { t.Errorf("fields not carried: %+v", got) } + + bad := in + bad.From, bad.Stage = "Needs Review!", "Actionable" + got, ok = NormalizeEvent(bad) + if !ok { + t.Fatal("invalid slugs must blank the field, not drop the event") + } + if got.From != "" || got.Stage != "" { + t.Errorf("invalid from/stage not blanked: %q %q", got.From, got.Stage) + } } func TestNormalizeEvent_RejectsMalformed(t *testing.T) { From a597761d38aa702de500bff5cc20607f9f404bc5 Mon Sep 17 00:00:00 2001 From: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Sat, 31 Oct 2026 10:43:11 -0400 Subject: [PATCH 6/6] docs(hub): pin the drop-count limit after a lost ack and full buffer turnover A resend whose earlier events were already trimmed for room carries no repeated Seq, so the hub cannot tell the spoke's cumulative disclosure restates drops already counted and adds them once more. Document the limit at the detection site and pin the bounded over-count in a test. Hive-Fix-Attempt: 2/6 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- src/pkg/hub/activity_events.go | 8 ++++- src/pkg/hub/activity_events_test.go | 45 +++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/src/pkg/hub/activity_events.go b/src/pkg/hub/activity_events.go index 226d386ed7..e5aa5ffa83 100644 --- a/src/pkg/hub/activity_events.go +++ b/src/pkg/hub/activity_events.go @@ -74,7 +74,13 @@ func (r *activityEventRing) ingest(hiveID string, events []hubactivity.Event, dr if e.Seq <= h.lastSeq { // The spoke trims acknowledged events, so holding one proves // it never saw the ack and its drop disclosure is cumulative - // over the one already counted. + // over the one already counted. Known limit: if the spoke + // buffered more than hubactivity.MaxEvents new events before + // the resend, the earlier events were trimmed for room and + // nothing here repeats a Seq, so the restatement goes + // undetected and the previous batch's drops are counted once + // more. The over-count is bounded by one batch's disclosure + // and affects only the drop counter, never stored events. restated = true continue } diff --git a/src/pkg/hub/activity_events_test.go b/src/pkg/hub/activity_events_test.go index 1e6a85e476..a83b1a5220 100644 --- a/src/pkg/hub/activity_events_test.go +++ b/src/pkg/hub/activity_events_test.go @@ -174,6 +174,51 @@ func TestActivityEventRing_DedupesByAcknowledgedSeq(t *testing.T) { } } +// Known limit, pinned so a change is deliberate: a lost ack followed by a +// full buffer turnover (more than hubactivity.MaxEvents new events) leaves +// no repeated Seq in the resend, so the hub cannot see that the spoke's +// cumulative disclosure restates drops already counted and counts the +// earlier batch's drops once more. The over-count is bounded by that one +// batch's disclosure; stored events are never duplicated. +func TestActivityEventRing_RestatedDropsAfterFullTurnoverOverCountOnce(t *testing.T) { + var st activityStore + first := withEvents(activityDigest("o/a", 1), activityEv(1, hubactivity.EventOpened, "o/a", 1), activityEv(2, hubactivity.EventStage, "o/a", 1)) + first.EventsDropped = 3 + st.record("hive-a", first, activityT0) + + // The ack is lost and the spoke buffers MaxEvents newer events, so the + // resend holds only Seq 3.. and its disclosure 3 (old) + 2 (new) = 5. + turned := make([]hubactivity.Event, 0, hubactivity.MaxEvents) + for i := range hubactivity.MaxEvents { + turned = append(turned, activityEv(uint64(3+i), hubactivity.EventStage, "o/a", 1)) + } + resend := withEvents(first, turned...) + resend.EventsDropped = 5 + st.record("hive-a", resend, activityT0.Add(time.Minute)) + + snap := st.eventsSnapshot([]string{"hive-a"}) + if len(snap.events) != 2+hubactivity.MaxEvents { + t.Fatalf("stored %d events, want %d (no duplicates)", len(snap.events), 2+hubactivity.MaxEvents) + } + if got := st.eventsAck("hive-a"); got != uint64(2+hubactivity.MaxEvents) { + t.Fatalf("ack = %d, want %d", got, 2+hubactivity.MaxEvents) + } + // Exact accounting would be 5; the undetectable restatement adds the + // first batch's 3 once more, and never more than that. + if snap.dropped["hive-a"] != 8 { + t.Fatalf("disclosed drops = %d, want 8 (5 exact + the 3 restated once, undetectable without a repeated Seq)", snap.dropped["hive-a"]) + } + + // A further resend that does repeat a Seq is detected again and only + // the increase over the last counted disclosure is added. + again := withEvents(resend, turned[len(turned)-1], activityEv(uint64(3+hubactivity.MaxEvents), hubactivity.EventMerged, "o/a", 1)) + again.EventsDropped = 6 + st.record("hive-a", again, activityT0.Add(2*time.Minute)) + if got := st.eventsSnapshot([]string{"hive-a"}).dropped["hive-a"]; got != 9 { + t.Fatalf("drops after detected resend = %d, want 9", got) + } +} + // The fleet-wide ring keeps the last 1,000 events across all hives, oldest // dropped first; each block is also capped at hubactivity.MaxEvents with // the overflow disclosed.