Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/added-15963-spoke-activity-digest.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- Spokes now publish the bounded activity digest to the hub: each heartbeat carries it, built in process from the stage tracker, with up to `hub.activity.max_items` items (default 100, hub ceiling 250). When the cap applies, the most urgent items are kept and `truncated` reports how many were left out. Private-repository items are left out unless `hub.activity.share_private` is set, but totals still count them. Once the hub acknowledges an etag, later beats send only that etag. A tracker change sends an activity-only update on the heartbeat endpoint at most once every 15 seconds, and changes that arrive meanwhile are merged into it. `hub.activity.enabled` defaults on for hosted spokes and stays opt-in for self-hosted ones ([#15963](https://github.com/hivecommons/hive/issues/15963), RFC [#15861](https://github.com/hivecommons/hive/issues/15861)).
23 changes: 20 additions & 3 deletions src/cmd/hive/boot_heartbeat_deps_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -181,9 +181,26 @@ func TestBootHeartbeatWith_PublishesIdentityBeforeTheLoopAndWiresEveryCallback(t
heartbeatCallback[spoke.ClusterIDCallback](t, f)
heartbeatCallback[spoke.FreshStatusCollector](t, f)
heartbeatCallback[spoke.UpgradeAnnounceCallback](t, f)
if len(f.callbacks) != 15 {
t.Fatalf("registered %d callbacks, want 15 (one of each)", len(f.callbacks))
}
activity := heartbeatCallback[spoke.ActivitySource](t, f)
if len(f.callbacks) != 16 {
t.Fatalf("registered %d callbacks, want 16 (one of each)", len(f.callbacks))
}
// hub.activity is self-hosted opt-in: this config is not hosted, so the
// digest stays off until the operator enables it, and the options are
// read per build so a reload applies without restarting the loop.
if opts := activity.Options(); opts.Enabled {
t.Fatalf("self-hosted activity options = %+v, want disabled by default", opts)
}
on := true
cfg.Hub.Activity = config.HubActivityConfig{Enabled: &on, MaxItems: 40, SharePrivate: true}
if opts := activity.Options(); !opts.Enabled || opts.MaxItems != 40 || !opts.SharePrivate {
t.Fatalf("opted-in activity options = %+v, want enabled with the configured cap and share_private", opts)
}
if items, ok := activity.Items(context.Background()); ok || items != nil {
t.Fatalf("activity items before any status = %v, %v; want none", items, ok)
}
activity.Subscribe(func() {})
activity.Subscribe(nil)

// The collector honours a runtime hub.enabled=false without being torn
// down: it returns nil so StartHeartbeat skips the beat.
Expand Down
26 changes: 25 additions & 1 deletion src/cmd/hive/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ import (
"github.com/hivecommons/hive/pkg/hooks"
"github.com/hivecommons/hive/pkg/hub"
spoke "github.com/hivecommons/hive/pkg/hub/spoke"
"github.com/hivecommons/hive/pkg/hubactivity"
"github.com/hivecommons/hive/pkg/intent"
"github.com/hivecommons/hive/pkg/ioscan"
"github.com/hivecommons/hive/pkg/jev"
Expand Down Expand Up @@ -5667,7 +5668,30 @@ func (b *boot) bootHeartbeatWith(deps bootHeartbeatDeps) {
RepoTargetMisconfigured: b.repoTargetMisconfigured(),
RepoTargetIssue: b.repoTargetIssueMessage(),
}
}))
}),
// Hub-wide activity digest (#15963): built from the dashboard's
// memoised stage tracker; tracker publishes trigger coalesced
// activity-only updates. hub.activity is read per build.
spoke.ActivitySource{
Options: func() spoke.ActivityOptions {
return spoke.ActivityOptions{
Enabled: b.cfg.Hub.Enabled && b.cfg.Hub.ActivityEnabled(),
MaxItems: b.cfg.Hub.Activity.MaxItems,
SharePrivate: b.cfg.Hub.Activity.SharePrivate,
}
},
Items: func(ctx context.Context) ([]hubactivity.Item, bool) {
if b.dashSrv == nil {
return nil, false
}
return b.dashSrv.HubActivityItems(ctx)
},
Subscribe: func(notify func()) {
if b.dashSrv != nil {
b.dashSrv.OnStageTrackerPublish(notify)
}
},
})

if b.cfg.Hub.TaskStatusPushEnabled() {
deps.startTaskStatusPush(b.ctx, b.hubURL, func() *spoke.TaskStatusPayload {
Expand Down
2 changes: 1 addition & 1 deletion src/docs/api-reference/hub-server.md
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f
| `POST` | `/api/shared-agent-pool/leases` | Hub handler-specific | Acquire or renew a shared CLI session lease | `pkg/hub/server.go:1708` |
| `PUT` | `/api/shared-agent-pool/leases` | Hub handler-specific | Renew an active shared CLI session lease | `pkg/hub/server.go:1709` |
| `DELETE` | `/api/shared-agent-pool/leases` | Hub handler-specific | Release a shared CLI session lease | `pkg/hub/server.go:1710` |
| `POST` | `/api/heartbeat` | Hub handler-specific | Heartbeat | `pkg/hub/server.go:1711` |
| `POST` | `/api/heartbeat` | Hub handler-specific | Heartbeat. A request carrying `X-Hive-Activity-Update` is an activity-only update ([#15963](https://github.com/hivecommons/hive/issues/15963)): body `{"activity_update":{"hive_id","activity"}}`, the same per-hive bearer, accepted only for a hive already in the registry (`404` otherwise); it replaces the in-memory activity digest and nothing else, and the signed response carries `ok` and `activity_etag` | `pkg/hub/server.go:1711` |
| `POST` | `/api/task-status` | Hub handler-specific | Task Status | `pkg/hub/server.go:1712` |
| `GET` | `/api/registry` | Hub handler-specific | Registry | `pkg/hub/server.go:1720` |
| `GET` | `/api/hub/leaderboard` | Hub handler-specific | Leaderboard | `pkg/hub/server.go:1721` |
Expand Down
14 changes: 14 additions & 0 deletions src/docs/telemetry.md
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,20 @@ hub:
| `task_titles` | Leaderboard `current_task` | Hub leaderboard shows who is active but not what they are working on. |
| `dashboard_urls` | `dashboard_url`, `snapshot_url`, public-URL self-check | Hub cannot link to the hive's dashboard or snapshot, and cannot probe URL reachability. |

## Activity digest

`hub.activity` controls the bounded, read-only activity digest the heartbeat carries for the hub-wide Activity Monitor ([#15963](https://github.com/hivecommons/hive/issues/15963), RFC [#15861](https://github.com/hivecommons/hive/issues/15861)). It is built in process from the dashboard's stage tracker: repo, number, kind, title, author kind, stage and modifier pills of open items, plus totals by stage and modifier.

```yaml
hub:
activity:
enabled: true # default: on for hosted spokes (hub.hive_type: hosted) not withholding repos; off for self-hosted
max_items: 100 # default 100; clamped to the hub's 250-item ceiling
share_private: false # private-repository items are dropped (still counted in totals) unless true
```

When the cap applies, items needing a human or a decision are kept first, then stalled ones, then the furthest stage, and the number left out is reported as `truncated`. A repository counts as public only when the hub lists the hive as public (`hub.is_public`, its project repos) or it is on `HIVE_PUBLIC_REPOS`. Once the hub acknowledges a digest's etag, later beats send only `etag` and `generated_at` until the tracker changes. A tracker change also sends an activity-only update over the heartbeat endpoint and bearer, at most one every 15 seconds; changes arriving meanwhile are merged into it. `hub.heartbeat_omit` applies: `repos` withholds every item (totals remain) and `task_titles` blanks titles.

## Telemetry agent

The telemetry agent audits and, once opted in, improves the **observability of the managed project** — the repositories the hive watches, not the hive's own internal metrics/tracing (see [Config.OTel/Tracing](operator-reference.md) for that, a deliberately separate concern). It is an L5/L6-only agent: absent from the roster below L5 and paused in every governor mode at L5/L6 until an operator explicitly opts in.
Expand Down
1 change: 1 addition & 0 deletions src/internal/testutil/dashboard_import_allowlist.txt
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ github.com/hivecommons/hive/pkg/hivestate
github.com/hivecommons/hive/pkg/hooks
github.com/hivecommons/hive/pkg/housestyle
github.com/hivecommons/hive/pkg/hub
github.com/hivecommons/hive/pkg/hubactivity
github.com/hivecommons/hive/pkg/hubbackup
github.com/hivecommons/hive/pkg/inferencehealth
github.com/hivecommons/hive/pkg/ioscan
Expand Down
3 changes: 3 additions & 0 deletions src/pkg/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -4431,6 +4431,9 @@ type HubConfig struct {
// dashboard_urls. Upgrade/config delivery is unaffected. See
// docs/telemetry.md and HeartbeatOmitClasses.
HeartbeatOmit []string `yaml:"heartbeat_omit,omitempty" json:"heartbeat_omit,omitempty"`
// Activity configures the bounded activity digest published to the hub
// with the heartbeat (#15963). See HubActivityConfig and ActivityEnabled.
Activity HubActivityConfig `yaml:"activity,omitempty" json:"activity,omitempty"`
// ContributeWallEnabled opts a hive into the public contributor wall on
// /contribute. Default false so no deployment gets a public posting surface
// without an operator decision.
Expand Down
42 changes: 42 additions & 0 deletions src/pkg/config/hub_activity.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
package config

import (
"slices"
"strings"
)

// HubActivityConfig is the hub.activity block: the bounded, read-only
// activity digest a spoke attaches to its hub heartbeat for the hub-wide
// Activity Monitor (RFC hivecommons/hive#15861, #15963). The wire contract
// and its ceilings live in pkg/hubactivity.
type HubActivityConfig struct {
// Enabled publishes the digest. A POINTER so an absent value means "use
// the default": ON for a hosted spoke (hub.hive_type: hosted) whose
// hub.heartbeat_omit does not withhold repos, OFF for every self-hosted
// or standalone install. Resolve with HubConfig.ActivityEnabled.
Enabled *bool `yaml:"enabled,omitempty" json:"enabled,omitempty"`
// MaxItems caps the items in one digest. 0/unset means 100; values above
// the hub's finite ceiling (hubactivity.MaxItemsCeiling, 250) are clamped.
MaxItems int `yaml:"max_items,omitempty" json:"max_items,omitempty"`
// SharePrivate keeps items from private repositories in the digest.
// Default false: they are dropped but still counted in totals.
SharePrivate bool `yaml:"share_private,omitempty" json:"share_private,omitempty"`
}

// ActivityEnabled reports whether this spoke publishes its activity digest.
//
// An explicit hub.activity.enabled always wins. Otherwise the digest is on
// only for a hosted spoke, whose hub already operates the hive, and only
// when hub.heartbeat_omit does not withhold repos: a digest names
// repositories, so an operator who withheld them from the heartbeat has not
// agreed to share them here either. Self-hosted spokes stay opt-in.
func (h HubConfig) ActivityEnabled() bool {
if h.Activity.Enabled != nil {
return *h.Activity.Enabled
}
if !strings.EqualFold(strings.TrimSpace(h.HiveType), HiveTypeHosted) {
return false
}
omit, err := HeartbeatOmitClasses(h.HeartbeatOmit)
return err == nil && !slices.Contains(omit, HeartbeatOmitRepos)
}
53 changes: 53 additions & 0 deletions src/pkg/config/hub_activity_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
package config

import (
"testing"

"gopkg.in/yaml.v3"
)

func TestHubActivityEnabledDefaults(t *testing.T) {
on, off := true, false
for _, tc := range []struct {
name string
hub HubConfig
want bool
}{
{"self-hosted defaults off", HubConfig{}, false},
{"local hive type defaults off", HubConfig{HiveType: "local"}, false},
{"hosted spoke defaults on", HubConfig{HiveType: HiveTypeHosted}, true},
{"hosted match is case and space insensitive", HubConfig{HiveType: " Hosted "}, true},
{"hosted withholding repos defaults off", HubConfig{HiveType: HiveTypeHosted, HeartbeatOmit: []string{"users", "repos"}}, false},
{"hosted withholding other classes stays on", HubConfig{HiveType: HiveTypeHosted, HeartbeatOmit: []string{"users", "task_titles"}}, true},
{"hosted with an invalid omit list fails closed", HubConfig{HiveType: HiveTypeHosted, HeartbeatOmit: []string{"bogus"}}, false},
{"self-hosted explicit opt-in", HubConfig{Activity: HubActivityConfig{Enabled: &on}}, true},
{"hosted explicit opt-out", HubConfig{HiveType: HiveTypeHosted, Activity: HubActivityConfig{Enabled: &off}}, false},
{"explicit opt-in wins over withheld repos", HubConfig{HeartbeatOmit: []string{"repos"}, Activity: HubActivityConfig{Enabled: &on}}, true},
} {
t.Run(tc.name, func(t *testing.T) {
if got := tc.hub.ActivityEnabled(); got != tc.want {
t.Fatalf("ActivityEnabled() = %v, want %v", got, tc.want)
}
})
}
}

func TestHubActivityConfigYAMLKeys(t *testing.T) {
var cfg Config
src := "hub:\n activity:\n enabled: true\n max_items: 40\n share_private: true\n"
if err := yaml.Unmarshal([]byte(src), &cfg); err != nil {
t.Fatal(err)
}
a := cfg.Hub.Activity
if a.Enabled == nil || !*a.Enabled || a.MaxItems != 40 || !a.SharePrivate {
t.Fatalf("hub.activity decoded as %+v", a)
}

var empty Config
if err := yaml.Unmarshal([]byte("hub:\n url: https://hub.example.test\n"), &empty); err != nil {
t.Fatal(err)
}
if empty.Hub.Activity.Enabled != nil || empty.Hub.Activity.MaxItems != 0 || empty.Hub.Activity.SharePrivate {
t.Fatalf("absent hub.activity must stay unset, got %+v", empty.Hub.Activity)
}
}
1 change: 1 addition & 0 deletions src/pkg/config/testdata/agent_runtime_golden.json
Original file line number Diff line number Diff line change
Expand Up @@ -557,6 +557,7 @@
"AutoUpgrade": false,
"AutoUpgradeMode": "",
"ContributeSuspended": false,
"activity": {},
"ContributeWallEnabled": false,
"ContributeWallRetentionDays": 0,
"NPSEnabled": null,
Expand Down
96 changes: 96 additions & 0 deletions src/pkg/dashboard/hub_activity.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
package dashboard

import (
"context"
"encoding/json"
"slices"
"sync"
"time"

"github.com/hivecommons/hive/pkg/hubactivity"
)

// Hub activity digest source (RFC hivecommons/hive#15861, #15963). The
// spoke's heartbeat builds its bounded activity digest from the stage
// tracker this server already memoises: no HTTP self-fetch and no GitHub
// call of its own.

// stageTrackerPublishHooks maps a *Server to the hook run on every tracker
// invalidation (status publish or webhook re-read). Kept beside the Server
// rather than in it so the hook stays a pure addition.
var stageTrackerPublishHooks sync.Map

// OnStageTrackerPublish registers fn to run on every tracker publish,
// replacing any earlier hook; nil removes it. fn runs on the publish and
// webhook paths and must not block.
func (s *Server) OnStageTrackerPublish(fn func()) {
if s == nil {
return
}
if fn == nil {
stageTrackerPublishHooks.Delete(s)
return
}
stageTrackerPublishHooks.Store(s, fn)
}

func (s *Server) notifyStageTrackerPublish() {
if v, ok := stageTrackerPublishHooks.Load(s); ok {
v.(func())()
}
}

// HubActivityItems returns every tracked item as a hub activity row, with
// Private set unless the repo is known public (hub.is_public project repos
// or HIVE_PUBLIC_REPOS, the same set the Battle Log trusts). ok is false
// while the spoke has no status or the tracker cannot be built. Reading
// does not count as a tracker client, so it adds no SSE nudges.
func (s *Server) HubActivityItems(ctx context.Context) ([]hubactivity.Item, bool) {
if s == nil {
return nil, false
}
s.statusMu.RLock()
status := s.status
s.statusMu.RUnlock()
if status == nil {
return nil, false
}
body, _, _, _ := s.stageTrackerPayloadFrom(ctx, status, time.Now().UTC(), false)
if body == nil {
return nil, false
}
var tracker struct {
Items []StageTrackerItem `json:"items"`
}
if err := json.Unmarshal(body, &tracker); err != nil {
return nil, false
}
return hubActivityItems(tracker.Items, s.publicRepos()), true
}

// hubActivityItems maps tracker rows to digest rows. A row the tracker
// flags NeedsHuman (a stage-only human wait such as ready-for-human-merge
// included) carries the needs-human modifier so the digest's urgency order
// matches the tracker's "Needs attention".
func hubActivityItems(items []StageTrackerItem, public publicRepoSet) []hubactivity.Item {
out := make([]hubactivity.Item, 0, len(items))
for _, it := range items {
mods := slices.Clone(it.Modifiers)
if it.NeedsHuman && !slices.Contains(mods, hubactivity.ModNeedsHuman) && !slices.Contains(mods, hubactivity.ModNeedsDecision) {
mods = append([]string{hubactivity.ModNeedsHuman}, mods...)
}
out = append(out, hubactivity.Item{
Repo: it.Repo,
Number: it.Number,
Kind: it.Kind,
Title: it.Title,
AuthorKind: it.AuthorKind,
Stage: it.StageSlug,
StageNumber: it.Stage,
Since: it.StageSince,
Mods: mods,
Private: !public.allows(it.Repo),
})
}
return out
}
Loading
Loading