From ac2debe3406f5a839739e77be2de347966f7703d Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Mon, 16 Nov 2026 16:02:17 -0500 Subject: [PATCH 1/2] =?UTF-8?q?=E2=9C=A8=20governor:=20release-feed=20watc?= =?UTF-8?q?her=20kicks=20agents=20when=20a=20tracked=20upstream=20dependen?= =?UTF-8?q?cy=20releases?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add governor.release_feeds: per agent, GitHub release/tag, PyPI or generic index feeds. The watcher polls them with If-None-Match/If-Modified-Since through the spoke's upstream-proxy rules, persists last-seen versions in /data/release-feeds.json, and on a new version queues a kick through the kick queue with a ${NEW_RELEASES} block, recording the reason on the governor kick-history entry. Kicks pass the CI-failure kick gates and are bounded by a per-agent cooldown. Private indexes authenticate with auth_var, an existing per-agent variable that is never logged. Fixes #18494 Signed-off-by: Andy Anderson --- .../added-18494-release-feed-watcher.md | 1 + src/cmd/hive/boot.go | 4 + src/cmd/hive/main.go | 5 +- src/cmd/hive/release_feed_watcher.go | 196 ++++++++ src/cmd/hive/release_feed_watcher_test.go | 227 ++++++++++ src/docs/agent-configuration.md | 2 +- src/docs/documentation-map.md | 1 + src/docs/env-vars.md | 2 + .../operator-reference/runtime-and-quota.md | 9 + src/docs/release-feeds.md | 127 ++++++ src/pkg/config/config.go | 5 + src/pkg/config/release_feeds.go | 202 +++++++++ src/pkg/config/release_feeds_test.go | 140 ++++++ src/pkg/config/validate.go | 3 + .../dashboard/settings_parity_allowlist.txt | 11 + src/pkg/governor/governor.go | 17 +- src/pkg/governor/kick_reason_test.go | 27 ++ src/pkg/releasefeed/egress.go | 41 ++ src/pkg/releasefeed/fetch.go | 365 +++++++++++++++ src/pkg/releasefeed/fetch_test.go | 309 +++++++++++++ src/pkg/releasefeed/store.go | 134 ++++++ src/pkg/releasefeed/store_test.go | 92 ++++ src/pkg/releasefeed/watcher.go | 420 ++++++++++++++++++ src/pkg/releasefeed/watcher_test.go | 359 +++++++++++++++ src/pkg/scheduler/new_releases.go | 33 ++ src/pkg/scheduler/new_releases_test.go | 41 ++ src/pkg/scheduler/scheduler.go | 29 +- src/pkg/scheduler/templates.go | 3 + src/pkg/snapshot/state.go | 4 +- 29 files changed, 2791 insertions(+), 18 deletions(-) create mode 100644 changelog.d/added-18494-release-feed-watcher.md create mode 100644 src/cmd/hive/release_feed_watcher.go create mode 100644 src/cmd/hive/release_feed_watcher_test.go create mode 100644 src/docs/release-feeds.md create mode 100644 src/pkg/config/release_feeds.go create mode 100644 src/pkg/config/release_feeds_test.go create mode 100644 src/pkg/governor/kick_reason_test.go create mode 100644 src/pkg/releasefeed/egress.go create mode 100644 src/pkg/releasefeed/fetch.go create mode 100644 src/pkg/releasefeed/fetch_test.go create mode 100644 src/pkg/releasefeed/store.go create mode 100644 src/pkg/releasefeed/store_test.go create mode 100644 src/pkg/releasefeed/watcher.go create mode 100644 src/pkg/releasefeed/watcher_test.go create mode 100644 src/pkg/scheduler/new_releases.go create mode 100644 src/pkg/scheduler/new_releases_test.go diff --git a/changelog.d/added-18494-release-feed-watcher.md b/changelog.d/added-18494-release-feed-watcher.md new file mode 100644 index 0000000000..71a1a62317 --- /dev/null +++ b/changelog.d/added-18494-release-feed-watcher.md @@ -0,0 +1 @@ +- **Release-feed watcher: kick an agent when a tracked upstream dependency releases.** `governor.release_feeds.agents..sources` lists GitHub releases or tags (`owner/repo`), PyPI packages, or a generic index URL. The governor polls them every `interval_s` (default `HIVE_RELEASE_FEED_INTERVAL`, else 15 minutes) with `If-None-Match` / `If-Modified-Since`, through the spoke's upstream-proxy rules, and keeps last-seen versions in `/data/release-feeds.json`. A new version queues a kick for the subscribed agent through the kick queue with a `${NEW_RELEASES}` block (package, old, new, URL) and records the reason on the governor kick-history entry; the first answer is a baseline and an unchanged feed kicks nothing. Kicks pass the same gates as the webhook CI-failure kick and are bounded by a per-agent cooldown (default `HIVE_RELEASE_FEED_COOLDOWN`, else 1 hour). Private indexes authenticate with `auth_var`, an existing per-agent or hive-level variable whose value is never logged. See `src/docs/release-feeds.md` ([#18494](https://github.com/hivecommons/hive/issues/18494)). diff --git a/src/cmd/hive/boot.go b/src/cmd/hive/boot.go index 0250ccc02e..7aa8b8edbe 100644 --- a/src/cmd/hive/boot.go +++ b/src/cmd/hive/boot.go @@ -39,6 +39,7 @@ import ( "github.com/hivecommons/hive/pkg/priorart" "github.com/hivecommons/hive/pkg/promptsrc" "github.com/hivecommons/hive/pkg/questionclose" + "github.com/hivecommons/hive/pkg/releasefeed" "github.com/hivecommons/hive/pkg/repoagents" "github.com/hivecommons/hive/pkg/retro" "github.com/hivecommons/hive/pkg/review/eventdispatch" @@ -288,6 +289,9 @@ type boot struct { // owned by its independent ticker goroutine. matrixIngestSt *matrixIngestState matrixIngestStarted bool + // releaseFeed is the release-feed watcher (#18494), started once by + // startReleaseFeedWatcher. + releaseFeed *releasefeed.Watcher // gfi is good-first-issue curation state (good_first_issue.*). gfi *gfiState // firstTimer is first-time contributor welcome state diff --git a/src/cmd/hive/main.go b/src/cmd/hive/main.go index 8a7967ecf3..d398563b0f 100644 --- a/src/cmd/hive/main.go +++ b/src/cmd/hive/main.go @@ -2502,7 +2502,7 @@ func (b *boot) bootStateWith(deps bootStateDeps) { if len(b.saved.KickHistory) > 0 { records := make([]governor.KickRecord, len(b.saved.KickHistory)) for i, ke := range b.saved.KickHistory { - records[i] = governor.KickRecord{Timestamp: ke.Timestamp, Agent: ke.Agent, Outcome: ke.Outcome, OutcomeReason: ke.OutcomeReason} + records[i] = governor.KickRecord{Timestamp: ke.Timestamp, Agent: ke.Agent, Outcome: ke.Outcome, OutcomeReason: ke.OutcomeReason, Reason: ke.Reason} } b.gov.SeedKickHistory(records) b.logger.Info("kick history restored", "entries", len(records)) @@ -6354,6 +6354,7 @@ func (b *boot) runLoopWith(deps runLoopDeps) { } b.startReviewEvents() b.startMatrixIngest() + b.startReleaseFeedWatcher() // Webhook health re-check: in webhook mode the base tick (900s) is longer // than the health window (2 × eval_interval_s), so without this a lapse @@ -6830,7 +6831,7 @@ func persistStateWithPaths(agentMgr crux.Runtime, gov *governor.Governor, cfg *c govKickHistory := gov.KickHistory() kickEntries := make([]snapshot.GovKickEntry, len(govKickHistory)) for i, kr := range govKickHistory { - kickEntries[i] = snapshot.GovKickEntry{Timestamp: kr.Timestamp, Agent: kr.Agent, Outcome: kr.Outcome, OutcomeReason: kr.OutcomeReason} + kickEntries[i] = snapshot.GovKickEntry{Timestamp: kr.Timestamp, Agent: kr.Agent, Outcome: kr.Outcome, OutcomeReason: kr.OutcomeReason, Reason: kr.Reason} } state := &snapshot.PersistedState{ diff --git a/src/cmd/hive/release_feed_watcher.go b/src/cmd/hive/release_feed_watcher.go new file mode 100644 index 0000000000..8fbb2011c9 --- /dev/null +++ b/src/cmd/hive/release_feed_watcher.go @@ -0,0 +1,196 @@ +package main + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/pkg/advisory" + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/governor" + "github.com/hivecommons/hive/pkg/releasefeed" + "github.com/hivecommons/hive/pkg/resolve" +) + +// releaseFeedStatePath is where the watcher keeps last-seen versions and +// validators across restarts. A var so tests can point it at a temp dir. +var releaseFeedStatePath = "/data/release-feeds.json" + +// releaseFeedKickHeader opens a targeted release-feed kick when the agent's +// own template does not place ${NEW_RELEASES} itself. +const releaseFeedKickHeader = "Upstream dependencies you track published new versions. Act on them now (for example, open the dependency bump); detection was done by the hive's release-feed watcher, so do not poll the feeds yourself.\n\n" + +// releaseFeedKicker gates and delivers a release-feed kick. Gating mirrors the +// webhook CI-failure kicker: agent known, enabled and not paused in config or +// at runtime, and governor eligibility (cadence pause, on-demand, budget). A +// refused kick returns releasefeed.ErrGated so the releases stay pending. +type releaseFeedKicker struct { + cfg func() *config.Config + gov func() *governor.Governor + isPaused func(agent string) bool + // render returns the agent's full kick prompt, whose ${NEW_RELEASES} + // the watcher's Block fills; nil or "" falls back to a targeted kick. + render func(agent string) string + // enqueue hands the message to the runtime's kick queue. + enqueue func(agent, message string) error +} + +func (k *releaseFeedKicker) Kick(_ context.Context, agent string, releases []releasefeed.Release) error { + if reason := k.gate(agent); reason != "" { + return fmt.Errorf("%w: %s", releasefeed.ErrGated, reason) + } + if k.enqueue == nil { + return fmt.Errorf("%w: no kick sender", releasefeed.ErrGated) + } + block := releasefeed.FormatBlock(releases) + message := "" + if k.render != nil { + message = k.render(agent) + } + if !strings.Contains(message, releasefeed.BlockHeader) { + message = releaseFeedKickHeader + block + } + // Release names and URLs are upstream text an agent may quote into an + // issue or PR, so they go through the same mention sanitizer as every + // other hive-authored kick. + message = advisory.NeutralizeMentions(message) + if err := k.enqueue(agent, message); err != nil { + return err + } + if gov := k.governor(); gov != nil { + gov.RecordKickWithReason(agent, releasefeed.Reason(releases)) + } + return nil +} + +func (k *releaseFeedKicker) gate(agent string) string { + cfg := k.config() + if cfg == nil { + return "no config" + } + ac, ok := cfg.Agents[agent] + if !ok { + return "unknown agent" + } + if !ac.Enabled || ac.Paused || (k.isPaused != nil && k.isPaused(agent)) { + return "agent paused" + } + if gov := k.governor(); gov != nil && !gov.AgentEligibleForCELKick(agent) { + return "governor gated" + } + return "" +} + +func (k *releaseFeedKicker) config() *config.Config { + if k.cfg != nil { + return k.cfg() + } + return nil +} + +func (k *releaseFeedKicker) governor() *governor.Governor { + if k.gov != nil { + return k.gov() + } + return nil +} + +// releaseFeedSubscriptions maps governor.release_feeds onto watcher sources. +func releaseFeedSubscriptions(cfg *config.Config) map[string][]releasefeed.Source { + out := map[string][]releasefeed.Source{} + if cfg == nil || cfg.Governor.ReleaseFeeds == nil { + return out + } + for agent, a := range cfg.Governor.ReleaseFeeds.Agents { + if len(a.Sources) == 0 { + continue + } + srcs := make([]releasefeed.Source, 0, len(a.Sources)) + for _, s := range a.Sources { + srcs = append(srcs, releasefeed.Source{ + Kind: s.Kind, Repo: s.Repo, Package: s.Package, URL: s.URL, + VersionField: s.VersionField, Name: s.Name, AuthVar: s.AuthVar, + }) + } + out[agent] = srcs + } + return out +} + +// releaseFeedToken resolves a source's auth variable through the agent's +// effective variables (per-agent over hive-level), the same registry that +// fills its kick prompt. An unresolved variable yields "". The value is +// returned to the fetcher only and never logged. +func releaseFeedToken(cfg *config.Config, agent, name string) string { + if cfg == nil || !config.ValidVariableName(name) { + return "" + } + raw := "${" + name + "}" + v := cfg.ResolveRegistryForAgent(agent, nil).Expand(context.Background(), raw, resolve.ScopeTemplate, nil) + if v == raw { + return "" + } + return strings.TrimSpace(v) +} + +// newReleaseFeedWatcher builds the watcher from boot state. Config, governor, +// scheduler and runtime are resolved per call so a reload or late wiring is +// picked up without restarting it. +func (b *boot) newReleaseFeedWatcher() *releasefeed.Watcher { + return releasefeed.New(releasefeed.Options{ + Enabled: func() bool { return b.cfg.ReleaseFeedsEnabled() }, + Subscriptions: func() map[string][]releasefeed.Source { return releaseFeedSubscriptions(b.cfg) }, + Interval: func() time.Duration { return b.cfg.ReleaseFeedInterval() }, + Cooldown: func(agent string) time.Duration { return b.cfg.ReleaseFeedCooldown(agent) }, + Token: func(agent, name string) string { return releaseFeedToken(b.cfg, agent, name) }, + Store: releasefeed.NewFileStore(releaseFeedStatePath), + Kicker: b.newReleaseFeedKicker(), + Logger: b.logger, + }) +} + +// newReleaseFeedKicker wires the kicker to boot state. +func (b *boot) newReleaseFeedKicker() *releaseFeedKicker { + return &releaseFeedKicker{ + cfg: func() *config.Config { return b.cfg }, + gov: func() *governor.Governor { return b.gov }, + isPaused: func(agent string) bool { return b.agentMgr != nil && b.agentMgr.IsPaused(agent) }, + render: func(agent string) string { + if b.sched == nil { + return "" + } + return b.sched.BuildAgentMessageFromLastActionable(agent) + }, + enqueue: func(agent, message string) error { + if b.agentMgr == nil { + return errNoAgentRuntime + } + // The kick queue (#18285) delivers once the agent is idle, so a + // release never interrupts a running turn. A runtime without it + // (CRUX_MODE=sidecar) takes the plain kick. + if q, ok := b.agentMgr.(crux.KickQueuer); ok { + _, err := q.EnqueueKick(agent, "", message) + return err + } + return b.agentMgr.SendKick(agent, message) + }, + } +} + +// startReleaseFeedWatcher starts the watcher goroutine once and installs its +// ${NEW_RELEASES} source on the scheduler. The goroutine idles (one cheap +// config check per interval) while governor.release_feeds lists no source, +// so enabling it by a config reload needs no restart. +func (b *boot) startReleaseFeedWatcher() { + if b.releaseFeed != nil || b.ctx == nil || b.cfg == nil { + return + } + w := b.newReleaseFeedWatcher() + b.releaseFeed = w + if b.sched != nil { + b.sched.SetNewReleasesSource(w.Block) + } + go w.Run(b.ctx) +} diff --git a/src/cmd/hive/release_feed_watcher_test.go b/src/cmd/hive/release_feed_watcher_test.go new file mode 100644 index 0000000000..571affbd55 --- /dev/null +++ b/src/cmd/hive/release_feed_watcher_test.go @@ -0,0 +1,227 @@ +package main + +import ( + "context" + "errors" + "io" + "log/slog" + "path/filepath" + "strings" + "testing" + + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/cruxtest" + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/governor" + "github.com/hivecommons/hive/pkg/releasefeed" + "github.com/hivecommons/hive/pkg/scheduler" +) + +type releaseFeedSent struct{ agent, message string } + +func newReleaseFeedKickerForTest(t *testing.T, cfg *config.Config) (*releaseFeedKicker, *governor.Governor, *[]releaseFeedSent) { + t.Helper() + var sent []releaseFeedSent + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + gov := governor.New(cfg.Governor, cfg.EnabledAgents(), logger) + k := &releaseFeedKicker{ + cfg: func() *config.Config { return cfg }, + gov: func() *governor.Governor { return gov }, + enqueue: func(agent, message string) error { + sent = append(sent, releaseFeedSent{agent: agent, message: message}) + return nil + }, + } + return k, gov, &sent +} + +func releaseFeedTestReleases() []releasefeed.Release { + return []releasefeed.Release{{Source: "pypi:requests", Package: "requests", Old: "2.32.0", New: "2.33.0", URL: "https://pypi.org/project/requests/2.33.0/"}} +} + +// A release-feed kick goes through the gates the webhook CI-failure kick +// uses, carries the release block, and leaves its reason in kick history. +func TestReleaseFeedKicker_GatesKicksAndRecordsReason(t *testing.T) { + cfg := ciKickTestConfig() + k, gov, sent := newReleaseFeedKickerForTest(t, cfg) + rels := releaseFeedTestReleases() + ctx := context.Background() + + if err := k.Kick(ctx, "quality", rels); err != nil { + t.Fatalf("kick: %v", err) + } + if len(*sent) != 1 || (*sent)[0].agent != "quality" { + t.Fatalf("sent = %+v", *sent) + } + msg := (*sent)[0].message + if !strings.HasPrefix(msg, releaseFeedKickHeader) || !strings.Contains(msg, "- requests: 2.32.0 -> 2.33.0") { + t.Fatalf("targeted kick = %q", msg) + } + hist := gov.KickHistory() + if len(hist) != 1 || hist[0].Reason != "new release: requests 2.32.0 -> 2.33.0" { + t.Fatalf("kick history = %+v", hist) + } + + // The agent's own template places ${NEW_RELEASES}: its full prompt is sent. + k.render = func(agent string) string { + return "full prompt for " + agent + "\n" + releasefeed.FormatBlock(rels) + } + if err := k.Kick(ctx, "quality", rels); err != nil { + t.Fatal(err) + } + if got := (*sent)[1].message; !strings.HasPrefix(got, "full prompt for quality") || strings.Contains(got, releaseFeedKickHeader) { + t.Fatalf("templated kick = %q", got) + } + // A template without the variable still gets the targeted kick. + k.render = func(string) string { return "full prompt" } + if err := k.Kick(ctx, "quality", rels); err != nil { + t.Fatal(err) + } + if got := (*sent)[2].message; !strings.HasPrefix(got, releaseFeedKickHeader) { + t.Fatalf("fallback kick = %q", got) + } + + gated := func(name string) { + t.Helper() + err := k.Kick(ctx, "quality", rels) + if !errors.Is(err, releasefeed.ErrGated) || !strings.Contains(err.Error(), name) { + t.Fatalf("%s: err = %v", name, err) + } + } + ac := cfg.Agents["quality"] + ac.Paused = true + cfg.Agents["quality"] = ac + gated("agent paused") + ac.Paused = false + cfg.Agents["quality"] = ac + + k.isPaused = func(string) bool { return true } + gated("agent paused") + k.isPaused = nil + + if err := k.Kick(ctx, "ghost", rels); !errors.Is(err, releasefeed.ErrGated) { + t.Fatalf("unknown agent: %v", err) + } + + onDemand := governor.New(cfg.Governor, map[string]config.AgentConfig{"quality": {Enabled: true, OnDemand: true}}, slog.New(slog.NewTextHandler(io.Discard, nil))) + k.gov = func() *governor.Governor { return onDemand } + gated("governor gated") + k.gov = func() *governor.Governor { return gov } + + k.cfg = func() *config.Config { return nil } + gated("no config") + k.cfg = func() *config.Config { return cfg } + + k.gov = nil + k.enqueue = nil + gated("no kick sender") + + k.enqueue = func(string, string) error { return errNoAgentRuntime } + if err := k.Kick(ctx, "quality", rels); !errors.Is(err, errNoAgentRuntime) { + t.Fatalf("delivery failure: %v", err) + } + k.cfg = nil + if got := k.gate("quality"); got != "no config" { + t.Fatalf("nil cfg func gate = %q", got) + } +} + +func TestReleaseFeedSubscriptions(t *testing.T) { + if got := releaseFeedSubscriptions(nil); len(got) != 0 { + t.Fatalf("nil config: %v", got) + } + if got := releaseFeedSubscriptions(&config.Config{}); len(got) != 0 { + t.Fatalf("no block: %v", got) + } + cfg := &config.Config{Governor: config.GovernorConfig{ReleaseFeeds: &config.ReleaseFeedsConfig{ + Agents: map[string]config.ReleaseFeedAgentConfig{ + "deps": {Sources: []config.ReleaseFeedSource{{Kind: "pypi", Package: "requests", AuthVar: "TOK", Name: "Requests"}}}, + "idle": {}, + }, + }}} + got := releaseFeedSubscriptions(cfg) + if len(got) != 1 || len(got["deps"]) != 1 { + t.Fatalf("subscriptions = %+v", got) + } + want := releasefeed.Source{Kind: "pypi", Package: "requests", AuthVar: "TOK", Name: "Requests"} + if got["deps"][0] != want { + t.Fatalf("source = %+v", got["deps"][0]) + } +} + +// The auth token comes from the agent's effective variables; an unset or +// invalid name yields "". +func TestReleaseFeedToken(t *testing.T) { + cfg := ciKickTestConfig() + ac := cfg.Agents["quality"] + ac.Variables = map[string]config.VarDef{"INDEX_TOKEN": {Type: "static", Value: "s3cret"}} + cfg.Agents["quality"] = ac + if got := releaseFeedToken(cfg, "quality", "INDEX_TOKEN"); got != "s3cret" { + t.Fatalf("per-agent token = %q", got) + } + if got := releaseFeedToken(cfg, "scanner", "INDEX_TOKEN"); got != "" { + t.Fatalf("another agent's variable leaked: %q", got) + } + if got := releaseFeedToken(cfg, "quality", "bad-name"); got != "" { + t.Fatalf("invalid name = %q", got) + } + if got := releaseFeedToken(nil, "quality", "INDEX_TOKEN"); got != "" { + t.Fatalf("nil config = %q", got) + } +} + +// startReleaseFeedWatcher builds the watcher once and installs it; without a +// context or config it does nothing. +func TestStartReleaseFeedWatcher(t *testing.T) { + old := releaseFeedStatePath + releaseFeedStatePath = filepath.Join(t.TempDir(), "release-feeds.json") + t.Cleanup(func() { releaseFeedStatePath = old }) + + (&boot{}).startReleaseFeedWatcher() + + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + cfg := ciKickTestConfig() + b := &boot{ctx: ctx, cfg: cfg, logger: slog.New(slog.NewTextHandler(io.Discard, nil))} + b.startReleaseFeedWatcher() + first := b.releaseFeed + if first == nil { + t.Fatal("watcher not started") + } + b.startReleaseFeedWatcher() + if b.releaseFeed != first { + t.Fatal("watcher started twice") + } + + if first.Block("quality") != "" { + t.Fatal("fresh watcher has pending releases") + } +} + +// The boot kicker resolves the scheduler and runtime per call, so it works +// before boot finishes wiring them and picks them up once it has. +func TestNewReleaseFeedKicker_ResolvesBootState(t *testing.T) { + cfg := ciKickTestConfig() + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + b := &boot{cfg: cfg, logger: logger} + k := b.newReleaseFeedKicker() + if k.config() != cfg || k.governor() != nil { + t.Fatal("kicker not bound to boot config/governor") + } + if k.isPaused("quality") { + t.Fatal("no runtime must read as not paused") + } + if got := k.render("quality"); got != "" { + t.Fatalf("no scheduler must render empty, got %q", got) + } + if err := k.enqueue("quality", "m"); !errors.Is(err, errNoAgentRuntime) { + t.Fatalf("no runtime: %v", err) + } + + b.sched = scheduler.New(cfg, logger) + b.agentMgr = cruxtest.NewRuntime(cfg.Agents, logger, crux.ProjectContext{}) + _ = k.isPaused("quality") + // Delivery is not exercised here: the queue would start a real pump for + // the fixture agent. The scheduler render is pure. + _ = k.render("quality") +} diff --git a/src/docs/agent-configuration.md b/src/docs/agent-configuration.md index eb864ed619..443033a889 100644 --- a/src/docs/agent-configuration.md +++ b/src/docs/agent-configuration.md @@ -1168,7 +1168,7 @@ Entries persist in `/data/repo-agents.json` (`HIVE_REPO_AGENTS_STATE` overrides ## Kick templates: what an agent is told to do -`kick_template` names a Markdown file resolved from the hive's policies checkout (`/data/policies/examples/kubestellar/agents/`, or the directory your `policies:` config points at), falling back to the defaults embedded in the binary (`src/pkg/policies/defaults/`). It is the agent's **periodic work prompt**: on every kick, the template is loaded, variables like `${ISSUE_LIST}`, `${PR_LIST}`, `${REVIEW_PERSPECTIVES}`, `${AGENT_NAME}`, `${PROJECT_ORG}`, and `${KNOWLEDGE}` are substituted, and the result is dispatched to the agent's session. The shared kick preamble always tells agents that the governor-supplied issue/PR lists are authoritative and that proxy-blocked `gh pr list`, `gh issue list`, and `gh search` discovery should not be used. +`kick_template` names a Markdown file resolved from the hive's policies checkout (`/data/policies/examples/kubestellar/agents/`, or the directory your `policies:` config points at), falling back to the defaults embedded in the binary (`src/pkg/policies/defaults/`). It is the agent's **periodic work prompt**: on every kick, the template is loaded, variables like `${ISSUE_LIST}`, `${PR_LIST}`, `${REVIEW_PERSPECTIVES}`, `${AGENT_NAME}`, `${PROJECT_ORG}`, and `${KNOWLEDGE}` are substituted, and the result is dispatched to the agent's session. `${NEW_RELEASES}` renders the upstream releases the [release-feed watcher](release-feeds.md) found for the agent and has not yet delivered (empty otherwise). The shared kick preamble always tells agents that the governor-supplied issue/PR lists are authoritative and that proxy-blocked `gh pr list`, `gh issue list`, and `gh search` discovery should not be used. Resolution order: the agent's explicit `kick_template` wins; otherwise the ACMM pack's template for that agent at the current level; otherwise convention — `/data/agents//CLAUDE.md`, then `.md` in the policies checkout, then the embedded default. Pack templates carry the level's policy in their names — `scanner-holdgated.md` is scanner-at-L5; the same scanner at L6 gets `scanner-automerge.md`. diff --git a/src/docs/documentation-map.md b/src/docs/documentation-map.md index e7971aeef8..fc3efbe9a8 100644 --- a/src/docs/documentation-map.md +++ b/src/docs/documentation-map.md @@ -50,6 +50,7 @@ Documentation for the current Hive line (branch `v5`; the code and docs live und - [dibs domain cutover](dibs-domain-cutover.md) — staged operator sequence for moving dibs to `dibs.hivecommons.dev`, including DNS, Let's Encrypt quota hold, Certificate/Ingress manifests, redirect verification, and rollback. - [Serving spokes from the fleet wildcard certificate](spoke-wildcard-tls.md) — how to point a cluster's provisioned spokes at the wildcard instead of one certificate per hive, the two cluster prerequisites that must hold first, and which hosts a wildcard cannot cover. - [Tagged releases](releases.md) — **Release maintainer:** trigger/version rules, manual recovery, channel relationship, and release SBOM artifacts. +- [Release-feed watcher](release-feeds.md) — **Off until an agent lists a source:** kicks an agent when a tracked GitHub release/tag, PyPI package or index publishes a new version, with `${NEW_RELEASES}` in the prompt. - [Release sentinel](release-sentinel.md) — **Opt-in; off by default:** bounded repair for failing release CI, human escalation, and separately opted-in retagging. - [Dashboard-triggered standalone upgrades](dashboard-standalone-upgrades.md) — the dashboard upgrade button for standalone (Compose/Podman-Quadlet) hives: the `HIVE_DEPLOYMENT_RUNTIME`/`HIVE_DEPLOYMENT_PODMAN_MODE` runtime contract, the closed host-side helper (`hive-dashboard-upgrade-helper.sh`), and why an unproven runtime deliberately hides the button. ### Dashboards and integration reference diff --git a/src/docs/env-vars.md b/src/docs/env-vars.md index 01ad1696c6..9306d94764 100644 --- a/src/docs/env-vars.md +++ b/src/docs/env-vars.md @@ -66,6 +66,8 @@ behavior. [Maintenance limitations](#keeping-this-reference-current) still apply | `HIVE_RELEASE_SENTINEL_ENABLED` | No | `release_sentinel.enabled` (default `false`) | Process-level override for the opt-in release sentinel ([#9585](https://github.com/hivecommons/hive/issues/9585)). `true`/`1`/`on`/`yes` turns it on and `false`/`0`/`off`/`no` turns it off regardless of config; any other value leaves the config in charge. See [release-sentinel.md](release-sentinel.md). | | `HIVE_RELEASE_SENTINEL_RETAG_ENABLED` | No | `release_sentinel.retag_enabled` (default `false`) | Process-level override for the release sentinel's separate retag opt-in ([#9585](https://github.com/hivecommons/hive/issues/9585)): when on, the hive moves the `v` tag to a merged, marked fix PR's merge commit with one leased, atomic tag push. Same values as `HIVE_RELEASE_SENTINEL_ENABLED`. It never turns retagging on while the sentinel itself is off. See [release-sentinel.md](release-sentinel.md#retag-after-merge). | | `HIVE_SCANNER_BUSY_BRANCH_MINUTES` | No | `60` | Busy-branch window for the repair queues ([#18488](https://github.com/hivecommons/hive/issues/18488)). A red or conflicted PR the hive App did not open is listed as "suggest, do not push" instead of as a push target while its head is younger than this many minutes or a claim by another worker covers the PR or a linked issue; the agent posts the fix as one suggestion comment for the worker to apply as its own signed commit. Whole minutes; unset, unparsable or non-positive values keep `60`. | +| `HIVE_RELEASE_FEED_INTERVAL` | No | `governor.release_feeds.interval_s`, else `15m` | Poll interval of the release-feed watcher ([#18494](https://github.com/hivecommons/hive/issues/18494)) as a Go duration. An explicit `interval_s` wins over it; never below `1m`. Invalid or non-positive values fall back to `15m`. See [release-feeds.md](release-feeds.md). | +| `HIVE_RELEASE_FEED_COOLDOWN` | No | `governor.release_feeds.cooldown_s` / per-agent `cooldown_s`, else `1h` | Default minimum gap between two release-feed kicks of one agent, as a Go duration. The YAML keys win over it. See [release-feeds.md](release-feeds.md#cooldown). | | `HIVE_ATTRIBUTED_CLOSED_PR_LOOKBACK` | No | `14d` | Bounds the per-repo closed-PR attribution scan that feeds dashboard outcome/rework metrics during PR enumeration. The scan lists closed PRs sorted by `updated_at` descending, stops when results are older than this window, and also stops after five pages so large repositories are never walked from history on every governor tick. Values accept Go durations such as `336h`, or day counts such as `14`/`14d`; invalid or non-positive values fall back to `14d`. | | `HIVE_LEVEL` | No | config/pack value | ACMM level bootstrap/override used by hosted flows and the entrypoint pack selection. | | `HIVE_ID` | No | config or generated id | Stable hive/spoke identifier override; passed through to launched agents. | diff --git a/src/docs/operator-reference/runtime-and-quota.md b/src/docs/operator-reference/runtime-and-quota.md index f51851ece5..472560f2f6 100644 --- a/src/docs/operator-reference/runtime-and-quota.md +++ b/src/docs/operator-reference/runtime-and-quota.md @@ -407,6 +407,15 @@ v5 and v6 spokes) tracking one repo all stay on webhook mode. records `ci_failure_kick` — "CI failed on #N (check) — kicked agent X" / "nudged contributor lane". **Agents never watch or poll CI themselves**: detection is out-of-band via this webhook path, or the sweep as fallback. +- **Upstream releases kick subscribed agents.** `governor.release_feeds` + lists, per agent, GitHub release/tag, PyPI or generic index feeds. The + release-feed watcher polls them with conditional requests every + `interval_s` (default `HIVE_RELEASE_FEED_INTERVAL`, else 15m) and, on a new + version, queues a kick for the agent through the same gates as the + CI-failure kick above, with the releases in `${NEW_RELEASES}`, the reason in + kick history, and at most one such kick per agent per cooldown + ([#18494](https://github.com/hivecommons/hive/issues/18494)). See + [release-feeds.md](../release-feeds.md). - **Gated workflow runs are approved for trusted heads.** GitHub holds workflow runs for dependabot and first-time/fork contributors behind "Approve and run workflows" (`action_required`); such runs create no check diff --git a/src/docs/release-feeds.md b/src/docs/release-feeds.md new file mode 100644 index 0000000000..3748199aa2 --- /dev/null +++ b/src/docs/release-feeds.md @@ -0,0 +1,127 @@ +# Release-feed watcher + +Kicks an agent as soon as an upstream dependency it tracks publishes a new +version, instead of leaving the agent to find the release on its own timer. +**Off until an agent lists a source.** Tracking issue: +[#18494](https://github.com/hivecommons/hive/issues/18494). + +A custom agent that bumps dependencies used to learn about a release only +when its cadence came round, which delayed the bump and spent kicks that found +nothing. The governor already raises signals for PRs, CI and issues; the +release-feed watcher adds one for upstream releases. + +## Turning it on + +```yaml +governor: + release_feeds: + # enabled: false # explicit off switch; absent = on when any agent lists a source + interval_s: 900 # default: HIVE_RELEASE_FEED_INTERVAL, else 15m (never below 1m) + cooldown_s: 3600 # default: HIVE_RELEASE_FEED_COOLDOWN, else 1h + agents: + deps-bot: + cooldown_s: 1800 # optional per-agent override + sources: + - kind: github_release # latest published release + repo: kubernetes/kubernetes + - kind: github_tag # newest tag as GitHub lists it + repo: psf/requests + - kind: pypi # PyPI package JSON API + package: requests + - kind: index # any URL; private indexes use auth_var + url: https://pkgs.example.com/widgets/latest + version_field: version # JSON key (default "version") + name: widgets # label in the kick (default: the URL) + auth_var: WIDGETS_INDEX_TOKEN +``` + +| Kind | Request | Version read from | +|---|---|---| +| `github_release` | `GET https://api.github.com/repos//releases/latest` | `tag_name` | +| `github_tag` | `GET https://api.github.com/repos//tags?per_page=1` | first tag's `name` | +| `pypi` | `GET https://pypi.org/pypi//json` | `info.version` | +| `index` | `GET ` | a JSON object's `version_field`, or the first non-empty line of a plain-text body | + +`hive.yaml` validation rejects an unknown `kind`, a GitHub `repo` that is not +`owner/repo`, a `pypi` source without `package`, an `index` URL that is not +http(s), a negative interval or cooldown, and an `auth_var` that is not a +valid variable name. The keys are YAML-only; Settings has no control for them +yet. + +## What happens on a poll + +Every interval the watcher fetches each subscribed feed **once** (a feed two +agents share is one request) and compares the version with what each +subscriber was last told about: + +1. **First answer is a baseline.** A newly added feed, or a fresh state file, + records the current version and kicks nothing. +2. **Nothing changed, no kick.** Requests are conditional: the stored `ETag` + goes out as `If-None-Match` and `Last-Modified` as `If-Modified-Since`, so + an unchanged feed usually costs a `304` with no body. +3. **New version: kick within the same poll.** The release joins the agent's + pending list and, unless the agent is in its [cooldown](#cooldown), the + agent is kicked at once through the runtime's + [kick queue](design/crux.md), which delivers when the agent is idle so a + release never interrupts a running turn. A runtime without the queue + (`CRUX_MODE=sidecar`) gets a plain kick. + +The kick passes the same gates as the webhook CI-failure kick +([runtime-and-quota.md](operator-reference/runtime-and-quota.md)): the agent +must be known, enabled and not paused in config or at runtime, and eligible +for an event-driven kick (no cadence pause, not on-demand, budget not +exhausted). A refused or failed kick keeps the releases pending and they go +out on a later poll once the gate opens. + +Kick history records the reason, for example +`new release: requests 2.32.0 -> 2.33.0`, on the governor kick entry +(`reason` in the persisted state and the kick-history API). + +State (last-seen versions, validators, pending releases, last kick time) is +kept in `/data/release-feeds.json`, so a release published while the hive was +down still kicks after the restart. An unreadable state file is logged and +replaced: the next answers become baselines. + +## The `${NEW_RELEASES}` variable + +`${NEW_RELEASES}` is a built-in kick-template variable. It renders the +agent's pending releases, empty when there are none: + +```text +NEW UPSTREAM RELEASES (release-feed watcher): +- requests: 2.32.0 -> 2.33.0 (https://pypi.org/project/requests/2.33.0/) +- kubernetes/kubernetes: v1.34.1 -> v1.35.0 (https://github.com/kubernetes/kubernetes/releases/tag/v1.35.0) +``` + +When the agent's kick template contains `${NEW_RELEASES}`, the release kick +is the agent's full rendered prompt with the block in place. Otherwise the +kick is a short targeted message: a header telling the agent to act on the +releases (and not to poll the feeds itself) followed by the block. Versions +are reduced to one printable line of at most 128 characters, links to http(s) +without credentials, and the message goes through the hive's mention +sanitizer. + +## Cooldown + +An agent gets at most one release-feed kick per cooldown. Releases that land +inside it wait in the pending list; a feed that moves twice before its kick +is folded into one line that keeps the original old version and the newest +new one (`1.1.0 -> 1.3.0`). Precedence: the agent's `cooldown_s`, then the +block's `cooldown_s`, then `HIVE_RELEASE_FEED_COOLDOWN`, then `1h`. + +## Egress and credentials + +Polls go through the spoke's upstream-proxy rules (`network.upstream_proxy`, +`HIVE_UPSTREAM_PROXY_*`, or `HTTPS_PROXY`), the same selector the control +plane's other outbound clients use; a locked-down spoke must allow the feed +hosts there. + +A private index authenticates with `auth_var`, the name of an existing +variable: the agent's own `variables:` entry wins over a hive-level +`variables.defs` entry, exactly as for its kick prompt. The resolved value is +sent as `Authorization: Bearer `. It is never logged, never written to +the state file, and dropped by Go's HTTP client on a redirect to another host. +If the variable does not resolve, the feed is skipped with a warning naming +the variable (not its value) rather than polled anonymously. An index URL that +carries a credential in its query string is redacted (query replaced by a +short digest) in logs, the state file and the kick block. diff --git a/src/pkg/config/config.go b/src/pkg/config/config.go index f4eafb9d67..bad39b735d 100644 --- a/src/pkg/config/config.go +++ b/src/pkg/config/config.go @@ -1794,6 +1794,11 @@ type GovernorConfig struct { // "Webhook-driven CI failure reaction". CIFailureWebhookKick *bool `yaml:"ci_failure_webhook_kick,omitempty"` CIFailureKickCooldownS int `yaml:"ci_failure_kick_cooldown_s,omitempty"` + // ReleaseFeeds is the release-feed watcher (hivecommons/hive#18494): per + // agent, the upstream feeds whose new versions kick that agent with a + // ${NEW_RELEASES} block. Nil (the default) means off. See + // docs/release-feeds.md. + ReleaseFeeds *ReleaseFeedsConfig `yaml:"release_feeds,omitempty" json:"ReleaseFeeds,omitempty"` // MergeTrainStallWarnS is how long, in seconds, a merge train's head // may sit without progressing before the Activity Monitor marks the // train stalled on the queued rows' chips and the dashboard writes one diff --git a/src/pkg/config/release_feeds.go b/src/pkg/config/release_feeds.go new file mode 100644 index 0000000000..3a1dc22a3a --- /dev/null +++ b/src/pkg/config/release_feeds.go @@ -0,0 +1,202 @@ +package config + +import ( + "fmt" + "os" + "strings" + "time" +) + +// Release-feed watcher (hivecommons/hive#18494). governor.release_feeds lists, +// per agent, the upstream feeds whose new versions should kick that agent at +// once instead of waiting for its timer: GitHub releases or tags of an +// owner/repo, a PyPI package, or a generic index URL. The watcher polls each +// feed on ReleaseFeedInterval with conditional requests, persists the last +// version it saw, and kicks the subscribed agent through the kick queue with a +// ${NEW_RELEASES} block. A kick per agent is bounded by its cooldown. + +// Release-feed source kinds. +const ( + ReleaseFeedKindGitHubRelease = "github_release" + ReleaseFeedKindGitHubTag = "github_tag" + ReleaseFeedKindPyPI = "pypi" + ReleaseFeedKindIndex = "index" +) + +// Release-feed timing defaults. Each is env-overridable (a Go duration such as +// "10m"); an explicit hive.yaml value wins over the env, and the env over the +// constant. The interval is never allowed below MinReleaseFeedInterval so a +// typo cannot turn the watcher into a hammer on a public index. +const ( + ReleaseFeedIntervalEnv = "HIVE_RELEASE_FEED_INTERVAL" + DefaultReleaseFeedInterval = 15 * time.Minute + MinReleaseFeedInterval = time.Minute + + ReleaseFeedCooldownEnv = "HIVE_RELEASE_FEED_COOLDOWN" + DefaultReleaseFeedCooldown = time.Hour +) + +// ReleaseFeedsConfig is governor.release_feeds. +type ReleaseFeedsConfig struct { + // Enabled turns the watcher off when explicitly false. Absent means on + // whenever at least one agent lists a source. + Enabled *bool `yaml:"enabled,omitempty"` + // IntervalS is the poll interval in seconds (0 = env or default). + IntervalS int `yaml:"interval_s,omitempty"` + // CooldownS is the default per-agent minimum gap between two + // release-feed kicks, in seconds (0 = env or default). + CooldownS int `yaml:"cooldown_s,omitempty"` + // Agents maps an agent name to the feeds it subscribes to. + Agents map[string]ReleaseFeedAgentConfig `yaml:"agents,omitempty"` +} + +// ReleaseFeedAgentConfig is one agent's subscription. +type ReleaseFeedAgentConfig struct { + Sources []ReleaseFeedSource `yaml:"sources,omitempty"` + // CooldownS overrides the block-level cooldown for this agent. + CooldownS int `yaml:"cooldown_s,omitempty"` +} + +// ReleaseFeedSource is one upstream feed. +type ReleaseFeedSource struct { + // Kind is github_release, github_tag, pypi or index. + Kind string `yaml:"kind"` + // Repo is owner/repo for the GitHub kinds. + Repo string `yaml:"repo,omitempty"` + // Package is the PyPI project name for kind pypi. + Package string `yaml:"package,omitempty"` + // URL is the index endpoint for kind index. It returns either JSON + // carrying the version under VersionField, or plain text whose first + // non-empty line is the version. + URL string `yaml:"url,omitempty"` + // VersionField is the top-level JSON key holding the version for kind + // index (default "version"). + VersionField string `yaml:"version_field,omitempty"` + // Name labels the feed in the kick block (default: repo, package or URL). + Name string `yaml:"name,omitempty"` + // AuthVar names an existing per-agent (or hive-level) variable whose + // resolved value is sent as the bearer token. The value is never logged. + AuthVar string `yaml:"auth_var,omitempty"` +} + +// ReleaseFeedsEnabled reports whether the watcher should run: not explicitly +// disabled and at least one agent lists a source. +func (c *Config) ReleaseFeedsEnabled() bool { + if c == nil || c.Governor.ReleaseFeeds == nil { + return false + } + rf := c.Governor.ReleaseFeeds + if rf.Enabled != nil && !*rf.Enabled { + return false + } + for _, a := range rf.Agents { + if len(a.Sources) > 0 { + return true + } + } + return false +} + +// ReleaseFeedInterval is the effective poll interval. +func (c *Config) ReleaseFeedInterval() time.Duration { + return releaseFeedInterval(c, os.Getenv) +} + +func releaseFeedInterval(c *Config, getenv func(string) string) time.Duration { + d := DefaultReleaseFeedInterval + if v, ok := envDuration(getenv, ReleaseFeedIntervalEnv); ok { + d = v + } + if c != nil && c.Governor.ReleaseFeeds != nil && c.Governor.ReleaseFeeds.IntervalS > 0 { + d = time.Duration(c.Governor.ReleaseFeeds.IntervalS) * time.Second + } + return max(d, MinReleaseFeedInterval) +} + +// ReleaseFeedCooldown is the effective cooldown for agent. +func (c *Config) ReleaseFeedCooldown(agent string) time.Duration { + return releaseFeedCooldown(c, agent, os.Getenv) +} + +func releaseFeedCooldown(c *Config, agent string, getenv func(string) string) time.Duration { + d := DefaultReleaseFeedCooldown + if v, ok := envDuration(getenv, ReleaseFeedCooldownEnv); ok { + d = v + } + if c == nil || c.Governor.ReleaseFeeds == nil { + return d + } + rf := c.Governor.ReleaseFeeds + if rf.CooldownS > 0 { + d = time.Duration(rf.CooldownS) * time.Second + } + if a, ok := rf.Agents[agent]; ok && a.CooldownS > 0 { + d = time.Duration(a.CooldownS) * time.Second + } + return d +} + +func envDuration(getenv func(string) string, name string) (time.Duration, bool) { + if getenv == nil { + return 0, false + } + raw := strings.TrimSpace(getenv(name)) + if raw == "" { + return 0, false + } + d, err := time.ParseDuration(raw) + if err != nil || d <= 0 { + return 0, false + } + return d, true +} + +// validateReleaseFeeds checks governor.release_feeds. +func validateReleaseFeeds(rf *ReleaseFeedsConfig) error { + if rf == nil { + return nil + } + if rf.IntervalS < 0 { + return fmt.Errorf("governor.release_feeds.interval_s must be at least 0") + } + if rf.CooldownS < 0 { + return fmt.Errorf("governor.release_feeds.cooldown_s must be at least 0") + } + for agent, a := range rf.Agents { + if a.CooldownS < 0 { + return fmt.Errorf("governor.release_feeds.agents.%s.cooldown_s must be at least 0", agent) + } + for i, src := range a.Sources { + if err := validateReleaseFeedSource(src); err != nil { + return fmt.Errorf("governor.release_feeds.agents.%s.sources[%d]: %w", agent, i, err) + } + } + } + return nil +} + +func validateReleaseFeedSource(src ReleaseFeedSource) error { + switch src.Kind { + case ReleaseFeedKindGitHubRelease, ReleaseFeedKindGitHubTag: + owner, repo, ok := strings.Cut(strings.TrimSpace(src.Repo), "/") + if !ok || owner == "" || repo == "" || strings.Contains(repo, "/") { + return fmt.Errorf("kind %s needs repo as owner/repo, got %q", src.Kind, src.Repo) + } + case ReleaseFeedKindPyPI: + if strings.TrimSpace(src.Package) == "" { + return fmt.Errorf("kind pypi needs package") + } + case ReleaseFeedKindIndex: + u := strings.TrimSpace(src.URL) + if !strings.HasPrefix(u, "https://") && !strings.HasPrefix(u, "http://") { + return fmt.Errorf("kind index needs an http(s) url, got %q", src.URL) + } + default: + return fmt.Errorf("kind must be one of %s, %s, %s, %s; got %q", + ReleaseFeedKindGitHubRelease, ReleaseFeedKindGitHubTag, ReleaseFeedKindPyPI, ReleaseFeedKindIndex, src.Kind) + } + if src.AuthVar != "" && !ValidVariableName(src.AuthVar) { + return fmt.Errorf("auth_var %q is not a valid variable name", src.AuthVar) + } + return nil +} diff --git a/src/pkg/config/release_feeds_test.go b/src/pkg/config/release_feeds_test.go new file mode 100644 index 0000000000..1d2773b17e --- /dev/null +++ b/src/pkg/config/release_feeds_test.go @@ -0,0 +1,140 @@ +package config + +import ( + "strings" + "testing" + "time" +) + +func releaseFeedTestConfig() *Config { + return &Config{Governor: GovernorConfig{ReleaseFeeds: &ReleaseFeedsConfig{ + Agents: map[string]ReleaseFeedAgentConfig{ + "deps": {Sources: []ReleaseFeedSource{{Kind: ReleaseFeedKindPyPI, Package: "requests"}}}, + }, + }}} +} + +func TestReleaseFeedsEnabled(t *testing.T) { + var nilCfg *Config + if nilCfg.ReleaseFeedsEnabled() || (&Config{}).ReleaseFeedsEnabled() { + t.Fatal("no release_feeds block must mean off") + } + cfg := releaseFeedTestConfig() + if !cfg.ReleaseFeedsEnabled() { + t.Fatal("an agent with a source must turn the watcher on") + } + off := false + cfg.Governor.ReleaseFeeds.Enabled = &off + if cfg.ReleaseFeedsEnabled() { + t.Fatal("enabled: false must win") + } + empty := &Config{Governor: GovernorConfig{ReleaseFeeds: &ReleaseFeedsConfig{ + Agents: map[string]ReleaseFeedAgentConfig{"deps": {}}, + }}} + if empty.ReleaseFeedsEnabled() { + t.Fatal("agents without sources must leave the watcher off") + } +} + +func TestReleaseFeedInterval_Precedence(t *testing.T) { + env := map[string]string{} + getenv := func(k string) string { return env[k] } + if got := releaseFeedInterval(nil, getenv); got != DefaultReleaseFeedInterval { + t.Fatalf("default = %s", got) + } + env[ReleaseFeedIntervalEnv] = "5m" + if got := releaseFeedInterval(nil, getenv); got != 5*time.Minute { + t.Fatalf("env = %s", got) + } + env[ReleaseFeedIntervalEnv] = "bogus" + if got := releaseFeedInterval(nil, getenv); got != DefaultReleaseFeedInterval { + t.Fatalf("bad env must fall back to default, got %s", got) + } + env[ReleaseFeedIntervalEnv] = "1s" + if got := releaseFeedInterval(nil, getenv); got != MinReleaseFeedInterval { + t.Fatalf("interval below the floor must clamp, got %s", got) + } + cfg := releaseFeedTestConfig() + cfg.Governor.ReleaseFeeds.IntervalS = 600 + if got := releaseFeedInterval(cfg, getenv); got != 10*time.Minute { + t.Fatalf("config must win over env, got %s", got) + } + if got := cfg.ReleaseFeedInterval(); got != 10*time.Minute { + t.Fatalf("ReleaseFeedInterval = %s", got) + } +} + +func TestReleaseFeedCooldown_Precedence(t *testing.T) { + env := map[string]string{} + getenv := func(k string) string { return env[k] } + if got := releaseFeedCooldown(nil, "deps", getenv); got != DefaultReleaseFeedCooldown { + t.Fatalf("default = %s", got) + } + env[ReleaseFeedCooldownEnv] = "20m" + if got := releaseFeedCooldown(nil, "deps", getenv); got != 20*time.Minute { + t.Fatalf("env = %s", got) + } + cfg := releaseFeedTestConfig() + cfg.Governor.ReleaseFeeds.CooldownS = 120 + if got := releaseFeedCooldown(cfg, "deps", getenv); got != 2*time.Minute { + t.Fatalf("block cooldown = %s", got) + } + a := cfg.Governor.ReleaseFeeds.Agents["deps"] + a.CooldownS = 30 + cfg.Governor.ReleaseFeeds.Agents["deps"] = a + if got := releaseFeedCooldown(cfg, "deps", getenv); got != 30*time.Second { + t.Fatalf("agent cooldown = %s", got) + } + if got := releaseFeedCooldown(cfg, "other", getenv); got != 2*time.Minute { + t.Fatalf("other agent takes the block cooldown, got %s", got) + } + if got := cfg.ReleaseFeedCooldown("deps"); got != 30*time.Second { + t.Fatalf("ReleaseFeedCooldown = %s", got) + } + if _, ok := envDuration(nil, ReleaseFeedCooldownEnv); ok { + t.Fatal("nil getenv must report unset") + } +} + +func TestValidateReleaseFeeds(t *testing.T) { + if err := validateReleaseFeeds(nil); err != nil { + t.Fatal(err) + } + ok := []ReleaseFeedSource{ + {Kind: ReleaseFeedKindGitHubRelease, Repo: "psf/requests"}, + {Kind: ReleaseFeedKindGitHubTag, Repo: "psf/requests"}, + {Kind: ReleaseFeedKindPyPI, Package: "requests", AuthVar: "PYPI_TOKEN"}, + {Kind: ReleaseFeedKindIndex, URL: "https://index.example.com/latest"}, + } + if err := validateReleaseFeeds(&ReleaseFeedsConfig{Agents: map[string]ReleaseFeedAgentConfig{"deps": {Sources: ok}}}); err != nil { + t.Fatalf("valid sources rejected: %v", err) + } + cases := map[string]*ReleaseFeedsConfig{ + "interval_s": {IntervalS: -1}, + "cooldown_s": {CooldownS: -1}, + "agents.deps.cooldown_s": {Agents: map[string]ReleaseFeedAgentConfig{"deps": {CooldownS: -1}}}, + "owner/repo": {Agents: map[string]ReleaseFeedAgentConfig{"deps": {Sources: []ReleaseFeedSource{{Kind: ReleaseFeedKindGitHubRelease, Repo: "requests"}}}}}, + "needs package": {Agents: map[string]ReleaseFeedAgentConfig{"deps": {Sources: []ReleaseFeedSource{{Kind: ReleaseFeedKindPyPI}}}}}, + "http(s) url": {Agents: map[string]ReleaseFeedAgentConfig{"deps": {Sources: []ReleaseFeedSource{{Kind: ReleaseFeedKindIndex, URL: "ftp://x"}}}}}, + "kind must be": {Agents: map[string]ReleaseFeedAgentConfig{"deps": {Sources: []ReleaseFeedSource{{Kind: "npm"}}}}}, + "auth_var": {Agents: map[string]ReleaseFeedAgentConfig{"deps": {Sources: []ReleaseFeedSource{{Kind: ReleaseFeedKindPyPI, Package: "x", AuthVar: "bad-name"}}}}}, + } + for want, rf := range cases { + err := validateReleaseFeeds(rf) + if err == nil || !strings.Contains(err.Error(), want) { + t.Errorf("%s: err = %v", want, err) + } + } +} + +// Config.Validate rejects a bad release_feeds block, so a typo fails at load +// instead of the watcher silently skipping the feed. +func TestValidate_RejectsBadReleaseFeeds(t *testing.T) { + cfg := validatableConfig(nil) + cfg.Governor.ReleaseFeeds = &ReleaseFeedsConfig{Agents: map[string]ReleaseFeedAgentConfig{ + "deps": {Sources: []ReleaseFeedSource{{Kind: "npm"}}}, + }} + if err := cfg.ValidateWithOptions(ValidateOptions{}); err == nil || !strings.Contains(err.Error(), "release_feeds") { + t.Fatalf("Validate = %v, want a release_feeds error", err) + } +} diff --git a/src/pkg/config/validate.go b/src/pkg/config/validate.go index 46d16f27a1..3de7bce454 100644 --- a/src/pkg/config/validate.go +++ b/src/pkg/config/validate.go @@ -269,6 +269,9 @@ func (c *Config) ValidateWithOptions(opts ValidateOptions) error { if c.Governor.CIFailureKickCooldownS < 0 { return fmt.Errorf("governor.ci_failure_kick_cooldown_s must be at least 0") } + if err := validateReleaseFeeds(c.Governor.ReleaseFeeds); err != nil { + return err + } if c.Governor.MergeTrainStallWarnS < 0 { return fmt.Errorf("governor.merge_train_stall_warn_s must be at least 0") } diff --git a/src/pkg/dashboard/settings_parity_allowlist.txt b/src/pkg/dashboard/settings_parity_allowlist.txt index 2329c6aa3b..63bafeb092 100644 --- a/src/pkg/dashboard/settings_parity_allowlist.txt +++ b/src/pkg/dashboard/settings_parity_allowlist.txt @@ -266,6 +266,17 @@ governor.ci.auto_approve_workflow_actors[] # trusted workflow-approval actors (# governor.ci.auto_approve_workflows # gated workflow auto-approval flag (#16603); yaml-only governor.ci_failure_kick_cooldown_s # webhook CI-failure kick cooldown (#15291); yaml-only governor.ci_failure_webhook_kick # webhook CI-failure kick flag (#15291); yaml-only +governor.release_feeds.agents..cooldown_s # release-feed watcher (#18494); yaml-only +governor.release_feeds.agents..sources[].auth_var # release-feed watcher (#18494); yaml-only +governor.release_feeds.agents..sources[].kind # release-feed watcher (#18494); yaml-only +governor.release_feeds.agents..sources[].name # release-feed watcher (#18494); yaml-only +governor.release_feeds.agents..sources[].package # release-feed watcher (#18494); yaml-only +governor.release_feeds.agents..sources[].repo # release-feed watcher (#18494); yaml-only +governor.release_feeds.agents..sources[].url # release-feed watcher (#18494); yaml-only +governor.release_feeds.agents..sources[].version_field # release-feed watcher (#18494); yaml-only +governor.release_feeds.cooldown_s # release-feed watcher (#18494); yaml-only +governor.release_feeds.enabled # release-feed watcher (#18494); yaml-only +governor.release_feeds.interval_s # release-feed watcher (#18494); yaml-only governor.claims.contributor_flap_threshold # contributor flap cool-off (#13579); yaml-only governor.claims.contributor_flap_window_s # contributor flap cool-off (#13579); yaml-only governor.claims.contributor_flap_cooloff_s # contributor flap cool-off (#13579); yaml-only diff --git a/src/pkg/governor/governor.go b/src/pkg/governor/governor.go index fb24f398da..53ec54717f 100644 --- a/src/pkg/governor/governor.go +++ b/src/pkg/governor/governor.go @@ -72,6 +72,10 @@ type KickRecord struct { // Empty while the turn is running or when the manager never classified it. Outcome string `json:"outcome,omitempty"` OutcomeReason string `json:"outcomeReason,omitempty"` + // Reason is why an event-driven kick went out (for example "new release + // requests 2.32.0 -> 2.33.0"), set by RecordKickWithReason. Empty for + // cadence kicks, whose reason is the schedule itself. + Reason string `json:"reason,omitempty"` } type AgentReportRecord struct { @@ -1401,6 +1405,17 @@ func (g *Governor) RecordKickForRepo(agentName, repo string) { // — the hosted scanner sat at its prompt for 45 minutes this way after a // 3-minute turn. In that case the kick is counted but the schedule is kept. func (g *Governor) RecordKickForRepoAt(agentName, repo string, kickedAt time.Time) { + g.recordKick(agentName, repo, "", kickedAt) +} + +// RecordKickWithReason records an event-driven kick of agentName and keeps +// reason on its kick-history entry, so the dashboard and persisted history +// say why the kick went out (the release-feed watcher, #18494). +func (g *Governor) RecordKickWithReason(agentName, reason string) { + g.recordKick(agentName, "", reason, time.Time{}) +} + +func (g *Governor) recordKick(agentName, repo, reason string, kickedAt time.Time) { report, hasReport := g.validateAgentReportIfPresent(agentName) g.mu.Lock() @@ -1434,7 +1449,7 @@ func (g *Governor) RecordKickForRepoAt(agentName, repo string, kickedAt time.Tim } g.state.Continuous[agentName] = st } - g.appendKickHistory(KickRecord{Timestamp: kickedAt, Agent: agentName, Repo: repo}) + g.appendKickHistory(KickRecord{Timestamp: kickedAt, Agent: agentName, Repo: repo, Reason: reason}) if hasReport { g.agentReports[agentName] = report } diff --git a/src/pkg/governor/kick_reason_test.go b/src/pkg/governor/kick_reason_test.go new file mode 100644 index 0000000000..4a0814b022 --- /dev/null +++ b/src/pkg/governor/kick_reason_test.go @@ -0,0 +1,27 @@ +package governor + +import "testing" + +// RecordKickWithReason (#18494) counts the kick like RecordKick and keeps the +// reason on the kick-history entry; a plain RecordKick leaves it empty. +func TestRecordKickWithReason_KeepsReasonOnHistory(t *testing.T) { + cfg, agents := standardConfig("scanner") + g := New(cfg, agents, testLogger()) + + g.RecordKick("scanner") + g.RecordKickWithReason("scanner", "new release: requests 2.32.0 -> 2.33.0") + + hist := g.KickHistory() + if len(hist) != 2 { + t.Fatalf("history = %d entries, want 2", len(hist)) + } + if hist[0].Reason != "" { + t.Fatalf("cadence kick reason = %q, want empty", hist[0].Reason) + } + if hist[1].Agent != "scanner" || hist[1].Reason != "new release: requests 2.32.0 -> 2.33.0" { + t.Fatalf("release kick entry = %+v", hist[1]) + } + if g.GetState().LastKick["scanner"].IsZero() { + t.Fatal("RecordKickWithReason must stamp LastKick like RecordKick") + } +} diff --git a/src/pkg/releasefeed/egress.go b/src/pkg/releasefeed/egress.go new file mode 100644 index 0000000000..7cbd5b0253 --- /dev/null +++ b/src/pkg/releasefeed/egress.go @@ -0,0 +1,41 @@ +package releasefeed + +import ( + "net/http" + "net/url" + "time" + + "github.com/hivecommons/hive/crux/upstreamproxy" +) + +// Egress client timeouts. +const ( + egressRequestTimeout = 30 * time.Second + egressTLSTimeout = 10 * time.Second + egressResponseTimeout = 20 * time.Second + egressIdleConnTimeout = 90 * time.Second + egressMaxIdlePerHost = 2 + egressMaxResponseBytes = 64 << 10 +) + +// NewEgressClient returns the HTTP client feed polls use. Every request +// follows the spoke's upstream proxy rules (crux/upstreamproxy.Default, which +// boot resolves from network.upstream_proxy, HIVE_UPSTREAM_PROXY_* or +// HTTPS_PROXY), read per request so a proxy installed after the watcher +// starts still applies. A redirect to another host drops the Authorization +// header (net/http's own rule), so a private-index token never leaves its +// host. +func NewEgressClient() *http.Client { + tr := &http.Transport{ + Proxy: func(req *http.Request) (*url.URL, error) { + return upstreamproxy.Default().ProxyFunc()(req) + }, + TLSHandshakeTimeout: egressTLSTimeout, + ResponseHeaderTimeout: egressResponseTimeout, + IdleConnTimeout: egressIdleConnTimeout, + MaxIdleConnsPerHost: egressMaxIdlePerHost, + MaxResponseHeaderBytes: egressMaxResponseBytes, + ForceAttemptHTTP2: true, + } + return &http.Client{Transport: tr, Timeout: egressRequestTimeout} +} diff --git a/src/pkg/releasefeed/fetch.go b/src/pkg/releasefeed/fetch.go new file mode 100644 index 0000000000..56f390da29 --- /dev/null +++ b/src/pkg/releasefeed/fetch.go @@ -0,0 +1,365 @@ +// Package releasefeed is the governor's release-feed watcher +// (hivecommons/hive#18494). It polls upstream release feeds (GitHub releases +// or tags, a PyPI package, a generic index URL) with conditional requests, +// remembers the last version each subscribed agent was told about, and hands +// new versions to a Kicker so the agent is kicked within one poll interval of +// an upstream release instead of finding it on its own timer. +package releasefeed + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "unicode" +) + +// Source kinds; they match pkg/config's ReleaseFeedKind* values. +const ( + KindGitHubRelease = "github_release" + KindGitHubTag = "github_tag" + KindPyPI = "pypi" + KindIndex = "index" +) + +// Default endpoints. Tests point a Fetcher at a fake feed instead. +const ( + DefaultGitHubAPI = "https://api.github.com" + DefaultGitHubWeb = "https://github.com" + DefaultPyPIBase = "https://pypi.org" + defaultIndexField = "version" +) + +// Response-size caps. PyPI's project JSON lists every release file, so a +// large project's document runs to tens of MiB; every other feed answers in +// a few KiB. +const ( + maxPyPIBodyBytes = 64 << 20 + maxOtherBodyBytes = 4 << 20 + // maxVersionLen bounds an upstream version string before it reaches a + // kick prompt; real versions are far shorter. + maxVersionLen = 128 + // redactDigestBytes is how much of a SHA-256 stands in for a redacted + // URL query. + redactDigestBytes = 4 +) + +const userAgent = "hive-release-feed-watcher" + +// Source is one upstream feed an agent subscribes to. +type Source struct { + Kind string + Repo string + Package string + URL string + VersionField string + Name string + // AuthVar names the per-agent variable whose value is the bearer token. + AuthVar string +} + +// Key identifies the feed in the state file. The auth variable is part of +// it: the same URL read with different credentials may answer differently. +func (s Source) Key() string { + id := s.Repo + switch s.Kind { + case KindPyPI: + id = strings.ToLower(s.Package) + case KindIndex: + id = redactedURL(s.URL) + } + key := s.Kind + ":" + strings.TrimSpace(id) + if s.AuthVar != "" { + key += "#" + s.AuthVar + } + return key +} + +// Label is the feed's name in the kick block. +func (s Source) Label() string { + if n := strings.TrimSpace(s.Name); n != "" { + return n + } + switch s.Kind { + case KindPyPI: + return s.Package + case KindIndex: + return redactedURL(s.URL) + } + return s.Repo +} + +// redactedURL drops an index URL's userinfo and replaces its query with a +// short digest, so a credential carried in the URL never reaches the state +// file, a log line or a kick prompt, while two URLs that differ only in the +// query still get distinct keys. +func redactedURL(raw string) string { + raw = strings.TrimSpace(raw) + u, err := url.Parse(raw) + if err != nil { + sum := sha256.Sum256([]byte(raw)) + return "invalid-url-" + hex.EncodeToString(sum[:redactDigestBytes]) + } + u.User = nil + u.Fragment = "" + if u.RawQuery != "" { + sum := sha256.Sum256([]byte(u.RawQuery)) + u.RawQuery = "q-" + hex.EncodeToString(sum[:redactDigestBytes]) + } + return u.String() +} + +// SourceState is what the watcher remembers about one feed between polls. +type SourceState struct { + ETag string `json:"etag,omitempty"` + LastModified string `json:"last_modified,omitempty"` + Version string `json:"version,omitempty"` + URL string `json:"url,omitempty"` +} + +// FetchResult is one poll of one feed. +type FetchResult struct { + // NotModified is true when the server answered 304; Version and URL are + // then empty and the previous state stands. + NotModified bool + Version string + URL string + ETag string + LastModified string +} + +// Fetcher reads feeds over HTTP. Client must be the spoke's egress client +// (see NewEgressClient) so polls follow the configured upstream proxy. +type Fetcher struct { + Client *http.Client + GitHubAPI string + GitHubWeb string + PyPIBase string +} + +// ErrNoVersion means the feed answered but carried no version (a repo with +// no releases or tags yet, an index without the field). +var ErrNoVersion = errors.New("feed has no version") + +// Fetch polls src. prev supplies the validators for a conditional request; +// token, when non-empty, is sent as a bearer credential. Errors never carry +// the token. +func (f *Fetcher) Fetch(ctx context.Context, src Source, prev SourceState, token string) (FetchResult, error) { + endpoint, limit, err := f.endpoint(src) + if err != nil { + return FetchResult{}, err + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) + if err != nil { + return FetchResult{}, fmt.Errorf("build request for %s: %w", src.Key(), err) + } + req.Header.Set("User-Agent", userAgent) + switch src.Kind { + case KindGitHubRelease, KindGitHubTag: + req.Header.Set("Accept", "application/vnd.github+json") + default: + req.Header.Set("Accept", "application/json, text/plain;q=0.9, */*;q=0.1") + } + if prev.ETag != "" { + req.Header.Set("If-None-Match", prev.ETag) + } + if prev.LastModified != "" { + req.Header.Set("If-Modified-Since", prev.LastModified) + } + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + client := f.Client + if client == nil { + client = NewEgressClient() + } + resp, err := client.Do(req) + if err != nil { + return FetchResult{}, fmt.Errorf("fetch %s: %w", src.Key(), redactURLError(err)) + } + defer func() { _ = resp.Body.Close() }() + res := FetchResult{ETag: resp.Header.Get("ETag"), LastModified: resp.Header.Get("Last-Modified")} + if resp.StatusCode == http.StatusNotModified { + res.NotModified = true + if res.ETag == "" { + res.ETag = prev.ETag + } + if res.LastModified == "" { + res.LastModified = prev.LastModified + } + return res, nil + } + if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { + return FetchResult{}, fmt.Errorf("fetch %s: HTTP %d", src.Key(), resp.StatusCode) + } + body, err := io.ReadAll(io.LimitReader(resp.Body, limit+1)) + if err != nil { + return FetchResult{}, fmt.Errorf("read %s: %w", src.Key(), err) + } + if int64(len(body)) > limit { + return FetchResult{}, fmt.Errorf("read %s: response larger than %d bytes", src.Key(), limit) + } + version, link, err := f.parse(src, body) + if err != nil { + return FetchResult{}, err + } + res.Version, res.URL = version, link + return res, nil +} + +func (f *Fetcher) endpoint(src Source) (string, int64, error) { + switch src.Kind { + case KindGitHubRelease, KindGitHubTag: + owner, repo, ok := strings.Cut(strings.TrimSpace(src.Repo), "/") + if !ok || owner == "" || repo == "" { + return "", 0, fmt.Errorf("source %s: repo must be owner/repo", src.Key()) + } + base := strings.TrimRight(orDefault(f.GitHubAPI, DefaultGitHubAPI), "/") + path := "/repos/" + url.PathEscape(owner) + "/" + url.PathEscape(repo) + if src.Kind == KindGitHubRelease { + return base + path + "/releases/latest", maxOtherBodyBytes, nil + } + return base + path + "/tags?per_page=1", maxOtherBodyBytes, nil + case KindPyPI: + pkg := strings.TrimSpace(src.Package) + if pkg == "" { + return "", 0, fmt.Errorf("source %s: package is required", src.Key()) + } + base := strings.TrimRight(orDefault(f.PyPIBase, DefaultPyPIBase), "/") + return base + "/pypi/" + url.PathEscape(pkg) + "/json", maxPyPIBodyBytes, nil + case KindIndex: + u, err := url.Parse(strings.TrimSpace(src.URL)) + if err != nil || (u.Scheme != "https" && u.Scheme != "http") || u.Host == "" { + return "", 0, fmt.Errorf("source %s: url must be http(s)", src.Kind) + } + return u.String(), maxOtherBodyBytes, nil + } + return "", 0, fmt.Errorf("source kind %q is not supported", src.Kind) +} + +func (f *Fetcher) parse(src Source, body []byte) (string, string, error) { + web := strings.TrimRight(orDefault(f.GitHubWeb, DefaultGitHubWeb), "/") + var version, link string + switch src.Kind { + case KindGitHubRelease: + var rel struct { + TagName string `json:"tag_name"` + HTMLURL string `json:"html_url"` + } + if err := json.Unmarshal(body, &rel); err != nil { + return "", "", fmt.Errorf("parse %s: %w", src.Key(), err) + } + version, link = rel.TagName, rel.HTMLURL + case KindGitHubTag: + var tags []struct { + Name string `json:"name"` + } + if err := json.Unmarshal(body, &tags); err != nil { + return "", "", fmt.Errorf("parse %s: %w", src.Key(), err) + } + if len(tags) > 0 { + version = tags[0].Name + } + if v := cleanVersion(version); v != "" { + link = web + "/" + strings.TrimSpace(src.Repo) + "/releases/tag/" + url.PathEscape(v) + } + case KindPyPI: + var doc struct { + Info struct { + Version string `json:"version"` + } `json:"info"` + } + if err := json.Unmarshal(body, &doc); err != nil { + return "", "", fmt.Errorf("parse %s: %w", src.Key(), err) + } + version = doc.Info.Version + if v := cleanVersion(version); v != "" { + link = DefaultPyPIBase + "/project/" + url.PathEscape(strings.TrimSpace(src.Package)) + "/" + url.PathEscape(v) + "/" + } + case KindIndex: + version = indexVersion(body, orDefault(src.VersionField, defaultIndexField)) + link = redactedURL(src.URL) + } + version = cleanVersion(version) + if version == "" { + return "", "", fmt.Errorf("parse %s: %w", src.Key(), ErrNoVersion) + } + return version, cleanLink(link), nil +} + +// indexVersion reads a generic index: a JSON object's field, else the first +// non-empty line of a plain-text body. +func indexVersion(body []byte, field string) string { + trimmed := strings.TrimSpace(string(body)) + if strings.HasPrefix(trimmed, "{") { + var doc map[string]any + if err := json.Unmarshal(body, &doc); err != nil { + return "" + } + switch v := doc[field].(type) { + case string: + return v + case float64: + return fmt.Sprint(v) + } + return "" + } + for line := range strings.SplitSeq(trimmed, "\n") { + if l := strings.TrimSpace(line); l != "" { + return l + } + } + return "" +} + +// cleanVersion keeps an upstream version safe to place in a prompt: one line, +// printable, bounded. +func cleanVersion(v string) string { + v = strings.TrimSpace(v) + v = strings.Map(func(r rune) rune { + if unicode.IsControl(r) || unicode.IsSpace(r) { + return -1 + } + return r + }, v) + if len(v) > maxVersionLen { + v = v[:maxVersionLen] + } + return v +} + +// cleanLink keeps only http(s) links and drops any userinfo. +func cleanLink(raw string) string { + u, err := url.Parse(strings.TrimSpace(raw)) + if err != nil || (u.Scheme != "https" && u.Scheme != "http") || u.Host == "" { + return "" + } + u.User = nil + return u.String() +} + +// redactURLError strips the request URL's query from a transport error so an +// index URL with a credential in its query string is never logged. +func redactURLError(err error) error { + var ue *url.Error + if errors.As(err, &ue) { + if u, perr := url.Parse(ue.URL); perr == nil { + u.RawQuery, u.User = "", nil + return &url.Error{Op: ue.Op, URL: u.String(), Err: ue.Err} + } + } + return err +} + +func orDefault(v, d string) string { + if strings.TrimSpace(v) == "" { + return d + } + return v +} diff --git a/src/pkg/releasefeed/fetch_test.go b/src/pkg/releasefeed/fetch_test.go new file mode 100644 index 0000000000..bafd488d17 --- /dev/null +++ b/src/pkg/releasefeed/fetch_test.go @@ -0,0 +1,309 @@ +package releasefeed + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "sync" + "testing" +) + +// fakeFeed serves every feed kind from one test server and records the +// request headers it saw. +type fakeFeed struct { + mu sync.Mutex + t *testing.T + srv *httptest.Server + version string + etag string + lastMod string + status int + body string + lastAuth string + lastINM string + lastIMS string + hits int +} + +func newFakeFeed(t *testing.T) *fakeFeed { + t.Helper() + f := &fakeFeed{t: t, version: "1.0.0", etag: `"v1"`, lastMod: "Mon, 16 Nov 2026 10:00:00 GMT"} + f.srv = httptest.NewServer(http.HandlerFunc(f.serve)) + t.Cleanup(f.srv.Close) + return f +} + +// set mutates the feed under its lock; the handler runs on the server's +// goroutines. +func (f *fakeFeed) set(fn func(*fakeFeed)) { + f.mu.Lock() + defer f.mu.Unlock() + fn(f) +} + +// seenReq is a copy of what the handler last recorded. +type seenReq struct { + hits int + auth, ifNoneMatch, ifSince string +} + +func (f *fakeFeed) seen() seenReq { + f.mu.Lock() + defer f.mu.Unlock() + return seenReq{hits: f.hits, auth: f.lastAuth, ifNoneMatch: f.lastINM, ifSince: f.lastIMS} +} + +func (f *fakeFeed) serve(w http.ResponseWriter, r *http.Request) { + f.mu.Lock() + defer f.mu.Unlock() + f.hits++ + f.lastAuth = r.Header.Get("Authorization") + f.lastINM = r.Header.Get("If-None-Match") + f.lastIMS = r.Header.Get("If-Modified-Since") + if f.status != 0 { + w.WriteHeader(f.status) + return + } + if f.lastINM != "" && f.lastINM == f.etag { + w.WriteHeader(http.StatusNotModified) + return + } + w.Header().Set("ETag", f.etag) + w.Header().Set("Last-Modified", f.lastMod) + if f.body != "" { + _, _ = w.Write([]byte(f.body)) + return + } + switch { + case strings.HasSuffix(r.URL.Path, "/releases/latest"): + _, _ = w.Write([]byte(`{"tag_name":"` + f.version + `","html_url":"https://github.com/psf/requests/releases/tag/` + f.version + `"}`)) + case strings.HasSuffix(r.URL.Path, "/tags"): + if r.URL.Query().Get("per_page") != "1" { + f.t.Errorf("tags request without per_page=1: %s", r.URL) + } + _, _ = w.Write([]byte(`[{"name":"` + f.version + `"}]`)) + case strings.HasPrefix(r.URL.Path, "/pypi/"): + _, _ = w.Write([]byte(`{"info":{"version":"` + f.version + `"},"releases":{}}`)) + default: + _, _ = w.Write([]byte(`{"version":"` + f.version + `"}`)) + } +} + +func (f *fakeFeed) fetcher() *Fetcher { + return &Fetcher{Client: f.srv.Client(), GitHubAPI: f.srv.URL, GitHubWeb: "https://github.example", PyPIBase: f.srv.URL} +} + +func TestFetch_EachKind(t *testing.T) { + f := newFakeFeed(t) + ctx := context.Background() + cases := []struct { + src Source + wantURL string + wantPath string + }{ + {Source{Kind: KindGitHubRelease, Repo: "psf/requests"}, "https://github.com/psf/requests/releases/tag/1.0.0", ""}, + {Source{Kind: KindGitHubTag, Repo: "psf/requests"}, "https://github.example/psf/requests/releases/tag/1.0.0", ""}, + {Source{Kind: KindPyPI, Package: "requests"}, "https://pypi.org/project/requests/1.0.0/", ""}, + {Source{Kind: KindIndex, URL: f.srv.URL + "/index.json"}, f.srv.URL + "/index.json", ""}, + } + for _, tc := range cases { + got, err := f.fetcher().Fetch(ctx, tc.src, SourceState{}, "") + if err != nil { + t.Fatalf("%s: %v", tc.src.Kind, err) + } + if got.NotModified || got.Version != "1.0.0" || got.URL != tc.wantURL || got.ETag != `"v1"` || got.LastModified == "" { + t.Fatalf("%s: %+v", tc.src.Kind, got) + } + if f.seen().auth != "" { + t.Fatalf("%s: unauthenticated source sent Authorization", tc.src.Kind) + } + } +} + +func TestFetch_ConditionalAndAuth(t *testing.T) { + f := newFakeFeed(t) + src := Source{Kind: KindPyPI, Package: "requests", AuthVar: "PYPI_TOKEN"} + prev := SourceState{ETag: `"v1"`, LastModified: f.lastMod, Version: "1.0.0"} + got, err := f.fetcher().Fetch(context.Background(), src, prev, "s3cret") + if err != nil { + t.Fatal(err) + } + if !got.NotModified || got.ETag != `"v1"` || got.LastModified != f.lastMod { + t.Fatalf("304 result = %+v", got) + } + if seen := f.seen(); seen.ifNoneMatch != `"v1"` || seen.ifSince != f.lastMod { + t.Fatalf("validators not sent: %+v", seen) + } + if got := f.seen().auth; got != "Bearer s3cret" { + t.Fatalf("Authorization = %q", got) + } +} + +func TestFetch_Errors(t *testing.T) { + f := newFakeFeed(t) + ctx := context.Background() + + f.set(func(f *fakeFeed) { f.status = http.StatusUnauthorized }) + _, err := f.fetcher().Fetch(ctx, Source{Kind: KindPyPI, Package: "requests"}, SourceState{}, "s3cret") + if err == nil || !strings.Contains(err.Error(), "HTTP 401") || strings.Contains(err.Error(), "s3cret") { + t.Fatalf("401: %v", err) + } + f.set(func(f *fakeFeed) { f.status, f.body = 0, `{"info":{"version":""}}` }) + if _, err := f.fetcher().Fetch(ctx, Source{Kind: KindPyPI, Package: "requests"}, SourceState{}, ""); !errors.Is(err, ErrNoVersion) { + t.Fatalf("empty version: %v", err) + } + f.set(func(f *fakeFeed) { f.body = `not json` }) + for _, src := range []Source{ + {Kind: KindGitHubRelease, Repo: "a/b"}, + {Kind: KindGitHubTag, Repo: "a/b"}, + {Kind: KindPyPI, Package: "p"}, + } { + if _, err := f.fetcher().Fetch(ctx, src, SourceState{}, ""); err == nil || !strings.Contains(err.Error(), "parse") { + t.Fatalf("%s bad json: %v", src.Kind, err) + } + } + f.set(func(f *fakeFeed) { f.body = `[]` }) + if _, err := f.fetcher().Fetch(ctx, Source{Kind: KindGitHubTag, Repo: "a/b"}, SourceState{}, ""); !errors.Is(err, ErrNoVersion) { + t.Fatalf("no tags: %v", err) + } + f.set(func(f *fakeFeed) { f.body = "" }) + + for _, src := range []Source{ + {Kind: KindGitHubRelease, Repo: "nope"}, + {Kind: KindPyPI}, + {Kind: KindIndex, URL: "ftp://x"}, + {Kind: "npm"}, + } { + if _, err := f.fetcher().Fetch(ctx, src, SourceState{}, ""); err == nil { + t.Fatalf("%+v: want an endpoint error", src) + } + } + + // A transport failure names the feed but never the query string. + dead := &Fetcher{Client: f.srv.Client()} + _, err = dead.Fetch(ctx, Source{Kind: KindIndex, URL: "http://127.0.0.1:1/latest?token=s3cret"}, SourceState{}, "") + if err == nil || strings.Contains(err.Error(), "token=s3cret") { + t.Fatalf("transport error leaked the query: %v", err) + } +} + +func TestFetch_BodyTooLarge(t *testing.T) { + f := newFakeFeed(t) + f.set(func(f *fakeFeed) { f.body = `{"version":"1"}` + strings.Repeat(" ", maxOtherBodyBytes) }) + _, err := f.fetcher().Fetch(context.Background(), Source{Kind: KindIndex, URL: f.srv.URL + "/big"}, SourceState{}, "") + if err == nil || !strings.Contains(err.Error(), "larger than") { + t.Fatalf("oversized body: %v", err) + } +} + +func TestFetch_DefaultClientAndEndpoints(t *testing.T) { + f := newFakeFeed(t) + // A zero Fetcher builds the egress client itself; the index kind needs + // no base URL, so it reaches the fake directly. + got, err := (&Fetcher{}).Fetch(context.Background(), Source{Kind: KindIndex, URL: f.srv.URL + "/v"}, SourceState{}, "") + if err != nil || got.Version != "1.0.0" { + t.Fatalf("default client: %+v %v", got, err) + } + ep, _, err := (&Fetcher{}).endpoint(Source{Kind: KindGitHubRelease, Repo: "psf/requests"}) + if err != nil || ep != DefaultGitHubAPI+"/repos/psf/requests/releases/latest" { + t.Fatalf("default github endpoint = %q %v", ep, err) + } + ep, _, err = (&Fetcher{}).endpoint(Source{Kind: KindPyPI, Package: "requests"}) + if err != nil || ep != DefaultPyPIBase+"/pypi/requests/json" { + t.Fatalf("default pypi endpoint = %q %v", ep, err) + } +} + +func TestIndexVersion(t *testing.T) { + cases := map[string]string{ + `{"version":"2.1"}`: "2.1", + `{"version":3}`: "3", + `{"latest":"x"}`: "", + `{"version":["x"]}`: "", + `{bad`: "", + "\n\n 4.5.6 \nold\n": "4.5.6", + "": "", + } + for body, want := range cases { + if got := indexVersion([]byte(body), "version"); got != want { + t.Errorf("indexVersion(%q) = %q, want %q", body, got, want) + } + } + if got := indexVersion([]byte(`{"latest":"9"}`), "latest"); got != "9" { + t.Errorf("custom field = %q", got) + } +} + +func TestCleanVersionAndLink(t *testing.T) { + if got := cleanVersion(" v1.2\n\tIGNORE ALL "); got != "v1.2IGNOREALL" { + t.Fatalf("cleanVersion = %q", got) + } + if got := cleanVersion(strings.Repeat("9", maxVersionLen+10)); len(got) != maxVersionLen { + t.Fatalf("cleanVersion length = %d", len(got)) + } + if got := cleanLink("https://user:pw@example.com/x"); got != "https://example.com/x" { + t.Fatalf("cleanLink userinfo = %q", got) + } + for _, bad := range []string{"javascript:alert(1)", "", "::"} { + if got := cleanLink(bad); got != "" { + t.Fatalf("cleanLink(%q) = %q", bad, got) + } + } + plain := errors.New("plain") + if redactURLError(plain) != plain { + t.Fatal("non-url errors pass through") + } + ue := &url.Error{Op: "Get", URL: "https://u:p@x.example/a?k=v", Err: plain} + if got := redactURLError(ue).Error(); strings.Contains(got, "k=v") || strings.Contains(got, "u:p") { + t.Fatalf("redacted = %q", got) + } +} + +func TestSourceKeyAndLabel(t *testing.T) { + cases := []struct { + src Source + key string + label string + }{ + {Source{Kind: KindGitHubRelease, Repo: "psf/requests"}, "github_release:psf/requests", "psf/requests"}, + {Source{Kind: KindPyPI, Package: "Requests", AuthVar: "TOK"}, "pypi:requests#TOK", "Requests"}, + {Source{Kind: KindIndex, URL: "https://i/x", Name: "internal"}, "index:https://i/x", "internal"}, + {Source{Kind: KindIndex, URL: "https://i/x"}, "index:https://i/x", "https://i/x"}, + } + for _, tc := range cases { + if tc.src.Key() != tc.key || tc.src.Label() != tc.label { + t.Errorf("%+v: key=%q label=%q", tc.src, tc.src.Key(), tc.src.Label()) + } + } + // A credential in an index URL never reaches the key or the label, and + // two URLs that differ only in the query keep distinct keys. + a := Source{Kind: KindIndex, URL: "https://bot:pw@i.example/x?token=s3cret"} + b := Source{Kind: KindIndex, URL: "https://i.example/x?token=other"} + for _, s := range []string{a.Key(), a.Label()} { + if strings.Contains(s, "s3cret") || strings.Contains(s, "pw") { + t.Fatalf("credential leaked: %q", s) + } + } + if a.Key() == b.Key() { + t.Fatalf("distinct queries share key %q", a.Key()) + } + if got := redactedURL("http://[::1"); !strings.HasPrefix(got, "invalid-url-") { + t.Fatalf("unparseable url = %q", got) + } +} + +func TestNewEgressClient_UsesUpstreamProxyFunc(t *testing.T) { + c := NewEgressClient() + tr, ok := c.Transport.(*http.Transport) + if !ok || tr.Proxy == nil { + t.Fatalf("egress client transport = %T without a proxy func", c.Transport) + } + req, _ := http.NewRequest(http.MethodGet, "https://pypi.org/pypi/x/json", nil) + // With no upstream proxy installed the selector reports direct. + if u, err := tr.Proxy(req); err != nil || u != nil { + t.Fatalf("no proxy installed: %v %v", u, err) + } +} diff --git a/src/pkg/releasefeed/store.go b/src/pkg/releasefeed/store.go new file mode 100644 index 0000000000..c6511e8b11 --- /dev/null +++ b/src/pkg/releasefeed/store.go @@ -0,0 +1,134 @@ +package releasefeed + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "sync" + "time" +) + +// stateFileVersion is bumped when the on-disk shape changes incompatibly. +const stateFileVersion = 1 + +// The state file holds versions and validators, never credentials. +const ( + stateDirPerm os.FileMode = 0o755 + stateFilePerm os.FileMode = 0o644 +) + +// AgentState is what the watcher remembers for one subscribed agent. +type AgentState struct { + // Seen maps a Source.Key to the version the agent was last told about + // (or the baseline recorded the first time the feed answered). + Seen map[string]string `json:"seen,omitempty"` + // Pending are new versions not yet delivered in a kick: waiting out the + // cooldown, or refused by a gate (agent paused, governor budget). + Pending []Release `json:"pending,omitempty"` + // LastKick is when the last release-feed kick for this agent went out. + LastKick time.Time `json:"last_kick,omitzero"` +} + +// State is the persisted watcher state. +type State struct { + Version int `json:"version"` + Sources map[string]SourceState `json:"sources,omitempty"` + Agents map[string]*AgentState `json:"agents,omitempty"` +} + +func newState() *State { + return &State{Version: stateFileVersion, Sources: map[string]SourceState{}, Agents: map[string]*AgentState{}} +} + +func (s *State) agent(name string) *AgentState { + a := s.Agents[name] + if a == nil { + a = &AgentState{} + s.Agents[name] = a + } + if a.Seen == nil { + a.Seen = map[string]string{} + } + return a +} + +// Store persists State. +type Store interface { + Load() (*State, error) + Save(*State) error +} + +// FileStore is a JSON-file Store written atomically (temp file + rename). +type FileStore struct { + path string + mu sync.Mutex +} + +// NewFileStore returns a FileStore at path. +func NewFileStore(path string) *FileStore { return &FileStore{path: path} } + +// Load reads the state; a missing file is an empty state. +func (f *FileStore) Load() (*State, error) { + f.mu.Lock() + defer f.mu.Unlock() + data, err := os.ReadFile(f.path) + if errors.Is(err, os.ErrNotExist) { + return newState(), nil + } + if err != nil { + return nil, err + } + st := newState() + if err := json.Unmarshal(data, st); err != nil { + return nil, fmt.Errorf("parse %s: %w", f.path, err) + } + if st.Version > stateFileVersion { + return nil, fmt.Errorf("%s has state version %d, this hive understands up to %d", f.path, st.Version, stateFileVersion) + } + if st.Sources == nil { + st.Sources = map[string]SourceState{} + } + if st.Agents == nil { + st.Agents = map[string]*AgentState{} + } + return st, nil +} + +// Save writes st atomically. +func (f *FileStore) Save(st *State) error { + f.mu.Lock() + defer f.mu.Unlock() + st.Version = stateFileVersion + data, err := json.MarshalIndent(st, "", " ") + if err != nil { + return err + } + if err := os.MkdirAll(filepath.Dir(f.path), stateDirPerm); err != nil { + return err + } + tmp, err := os.CreateTemp(filepath.Dir(f.path), filepath.Base(f.path)+".tmp-*") + if err != nil { + return err + } + name := tmp.Name() + if _, err := tmp.Write(data); err != nil { + _ = tmp.Close() + _ = os.Remove(name) + return err + } + if err := tmp.Close(); err != nil { + _ = os.Remove(name) + return err + } + if err := os.Chmod(name, stateFilePerm); err != nil { + _ = os.Remove(name) + return err + } + if err := os.Rename(name, f.path); err != nil { + _ = os.Remove(name) + return err + } + return nil +} diff --git a/src/pkg/releasefeed/store_test.go b/src/pkg/releasefeed/store_test.go new file mode 100644 index 0000000000..e05ac0050c --- /dev/null +++ b/src/pkg/releasefeed/store_test.go @@ -0,0 +1,92 @@ +package releasefeed + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func TestFileStore_LoadSaveRoundTrip(t *testing.T) { + path := filepath.Join(t.TempDir(), "nested", "release-feeds.json") + fs := NewFileStore(path) + st, err := fs.Load() + if err != nil || len(st.Sources) != 0 || len(st.Agents) != 0 { + t.Fatalf("missing file: %+v %v", st, err) + } + st.Sources["pypi:requests"] = SourceState{ETag: `"e"`, Version: "1.0"} + st.agent("deps").Seen["pypi:requests"] = "1.0" + if err := fs.Save(st); err != nil { + t.Fatal(err) + } + got, err := fs.Load() + if err != nil || got.Sources["pypi:requests"].Version != "1.0" || got.Agents["deps"].Seen["pypi:requests"] != "1.0" { + t.Fatalf("round trip: %+v %v", got, err) + } + info, err := os.Stat(path) + if err != nil || info.Mode().Perm() != stateFilePerm { + t.Fatalf("state file mode: %v %v", info, err) + } +} + +func TestFileStore_LoadErrors(t *testing.T) { + dir := t.TempDir() + corrupt := filepath.Join(dir, "corrupt.json") + if err := os.WriteFile(corrupt, []byte("{"), 0o600); err != nil { + t.Fatal(err) + } + if _, err := NewFileStore(corrupt).Load(); err == nil || !strings.Contains(err.Error(), "parse") { + t.Fatalf("corrupt: %v", err) + } + future := filepath.Join(dir, "future.json") + if err := os.WriteFile(future, []byte(`{"version":99}`), 0o600); err != nil { + t.Fatal(err) + } + if _, err := NewFileStore(future).Load(); err == nil || !strings.Contains(err.Error(), "state version") { + t.Fatalf("future: %v", err) + } + empty := filepath.Join(dir, "empty.json") + if err := os.WriteFile(empty, []byte(`{"version":1}`), 0o600); err != nil { + t.Fatal(err) + } + st, err := NewFileStore(empty).Load() + if err != nil || st.Sources == nil || st.Agents == nil { + t.Fatalf("empty maps must be initialised: %+v %v", st, err) + } + if _, err := NewFileStore(dir).Load(); err == nil { + t.Fatal("reading a directory must fail") + } +} + +// A corrupt state file is reported and the watcher starts fresh rather than +// stopping: the next answers are baselines, so at worst one release is not +// kicked. +func TestWatcher_CorruptStateStartsFresh(t *testing.T) { + r := newWatcherRig(t) + if err := os.MkdirAll(filepath.Dir(r.store.path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(r.store.path, []byte("{"), 0o600); err != nil { + t.Fatal(err) + } + res := r.w.Poll(t.Context()) + if res.Fetched != 1 || len(res.Kicked) != 0 { + t.Fatalf("fresh start: %+v", res) + } +} + +func TestFileStore_SaveError(t *testing.T) { + dir := t.TempDir() + blocker := filepath.Join(dir, "file") + if err := os.WriteFile(blocker, nil, 0o600); err != nil { + t.Fatal(err) + } + // The parent "directory" is a regular file, so MkdirAll fails. + if err := NewFileStore(filepath.Join(blocker, "state.json")).Save(newState()); err == nil { + t.Fatal("save under a file must fail") + } + // The watcher logs a failed save and carries on. + w := New(Options{Store: NewFileStore(filepath.Join(blocker, "state.json")), Logger: quietLogger()}) + w.ensureLoaded() + w.save() +} diff --git a/src/pkg/releasefeed/watcher.go b/src/pkg/releasefeed/watcher.go new file mode 100644 index 0000000000..592c2c83e6 --- /dev/null +++ b/src/pkg/releasefeed/watcher.go @@ -0,0 +1,420 @@ +package releasefeed + +import ( + "context" + "errors" + "fmt" + "log/slog" + "sort" + "strings" + "sync" + "time" +) + +// Release is one new upstream version waiting to be (or just) delivered. +type Release struct { + // Source is the feed's Source.Key. + Source string `json:"source"` + Package string `json:"package"` + Old string `json:"old"` + New string `json:"new"` + URL string `json:"url,omitempty"` +} + +// Kicker delivers a release-feed kick. Returning an error wrapping ErrGated +// means a gate (agent paused, governor budget, repo pause) refused the kick: +// the releases stay pending and go out on a later poll once the gate opens. +// Any other error is a delivery failure and is retried the same way. +type Kicker interface { + Kick(ctx context.Context, agent string, releases []Release) error +} + +// ErrGated marks a kick a gate refused. +var ErrGated = errors.New("release-feed kick gated") + +// Options wires a Watcher. Every func is read on each poll, so a config +// reload takes effect without restarting the watcher. +type Options struct { + // Enabled reports whether the watcher should poll at all. + Enabled func() bool + // Subscriptions maps agent name to the feeds it subscribes to. + Subscriptions func() map[string][]Source + // Interval is the poll interval. + Interval func() time.Duration + // Cooldown is the minimum gap between two release-feed kicks of agent. + Cooldown func(agent string) time.Duration + // Token resolves a source's auth variable for agent; "" when unset. + Token func(agent, varName string) string + Fetcher *Fetcher + Store Store + Kicker Kicker + Logger *slog.Logger + Now func() time.Time +} + +// PollResult summarizes one poll, for logs and tests. +type PollResult struct { + Fetched int + NotModified int + Errors int + Detected int + Kicked []string +} + +// Watcher polls feeds and kicks subscribed agents on new versions. +type Watcher struct { + opts Options + + mu sync.Mutex + state *State + loaded bool +} + +// New returns a Watcher. +func New(opts Options) *Watcher { + if opts.Fetcher == nil { + opts.Fetcher = &Fetcher{Client: NewEgressClient()} + } + if opts.Logger == nil { + opts.Logger = slog.Default() + } + if opts.Now == nil { + opts.Now = time.Now + } + return &Watcher{opts: opts} +} + +// fallbackPollInterval is Run's spacing when Options.Interval is unset or +// returns a non-positive value. +const fallbackPollInterval = time.Minute + +// Run polls at once and then every Interval until ctx ends. +func (w *Watcher) Run(ctx context.Context) { + for { + w.Poll(ctx) + d := fallbackPollInterval + if w.opts.Interval != nil { + if v := w.opts.Interval(); v > 0 { + d = v + } + } + t := time.NewTimer(d) + select { + case <-ctx.Done(): + t.Stop() + return + case <-t.C: + } + } +} + +// Block renders the ${NEW_RELEASES} block for agent from its pending +// releases; "" when there are none. +func (w *Watcher) Block(agent string) string { + if w == nil { + return "" + } + w.mu.Lock() + defer w.mu.Unlock() + if w.state == nil { + return "" + } + a := w.state.Agents[agent] + if a == nil { + return "" + } + return FormatBlock(a.Pending) +} + +// BlockHeader opens every ${NEW_RELEASES} block. +const BlockHeader = "NEW UPSTREAM RELEASES (release-feed watcher):" + +// FormatBlock renders releases as the ${NEW_RELEASES} block. +func FormatBlock(releases []Release) string { + if len(releases) == 0 { + return "" + } + var b strings.Builder + b.WriteString(BlockHeader) + b.WriteByte('\n') + for _, r := range releases { + fmt.Fprintf(&b, "- %s: %s -> %s", r.Package, r.Old, r.New) + if r.URL != "" { + fmt.Fprintf(&b, " (%s)", r.URL) + } + b.WriteByte('\n') + } + return b.String() +} + +// maxReasonReleases is how many releases a kick-history reason names before +// it summarizes the rest as a count. +const maxReasonReleases = 3 + +// Reason is the kick-history reason for a release-feed kick. +func Reason(releases []Release) string { + parts := make([]string, 0, min(len(releases), maxReasonReleases)) + for i, r := range releases { + if i == maxReasonReleases { + break + } + parts = append(parts, fmt.Sprintf("%s %s -> %s", r.Package, r.Old, r.New)) + } + reason := "new release: " + strings.Join(parts, ", ") + if extra := len(releases) - maxReasonReleases; extra > 0 { + reason += fmt.Sprintf(" (+%d more)", extra) + } + return reason +} + +type fetchJob struct { + src Source + agent string +} + +// Poll runs one pass: fetch every subscribed feed once, record new versions +// as pending for each subscriber, and kick every agent whose cooldown has +// passed. +func (w *Watcher) Poll(ctx context.Context) PollResult { + var res PollResult + if w.opts.Enabled != nil && !w.opts.Enabled() { + return res + } + subs := map[string][]Source{} + if w.opts.Subscriptions != nil { + subs = w.opts.Subscriptions() + } + w.ensureLoaded() + + agents := make([]string, 0, len(subs)) + for a := range subs { + agents = append(agents, a) + } + sort.Strings(agents) + jobs := map[string]fetchJob{} + for _, a := range agents { + for _, src := range subs[a] { + if _, ok := jobs[src.Key()]; !ok { + jobs[src.Key()] = fetchJob{src: src, agent: a} + } + } + } + keys := make([]string, 0, len(jobs)) + for k := range jobs { + keys = append(keys, k) + } + sort.Strings(keys) + + for _, key := range keys { + if ctx.Err() != nil { + return res + } + job := jobs[key] + token := "" + if job.src.AuthVar != "" { + if w.opts.Token != nil { + token = w.opts.Token(job.agent, job.src.AuthVar) + } + if token == "" { + res.Errors++ + w.opts.Logger.Warn("release feed: auth variable is not set; feed skipped", + "feed", key, "agent", job.agent, "auth_var", job.src.AuthVar) + continue + } + } + w.mu.Lock() + prev := w.state.Sources[key] + w.mu.Unlock() + got, err := w.opts.Fetcher.Fetch(ctx, job.src, prev, token) + if err != nil { + res.Errors++ + w.opts.Logger.Warn("release feed poll failed", "feed", key, "error", err) + continue + } + next := SourceState{ETag: got.ETag, LastModified: got.LastModified, Version: prev.Version, URL: prev.URL} + if got.NotModified { + res.NotModified++ + } else { + res.Fetched++ + next.Version, next.URL = got.Version, got.URL + } + w.mu.Lock() + w.state.Sources[key] = next + w.mu.Unlock() + } + + now := w.opts.Now() + due, detected := w.diff(subs, now) + res.Detected = detected + for _, d := range due { + if ctx.Err() != nil || w.opts.Kicker == nil { + break + } + err := w.opts.Kicker.Kick(ctx, d.agent, d.releases) + if err != nil { + if errors.Is(err, ErrGated) { + w.opts.Logger.Info("release feed kick held", "agent", d.agent, "reason", err.Error(), "pending", len(d.releases)) + } else { + w.opts.Logger.Warn("release feed kick failed", "agent", d.agent, "error", err, "pending", len(d.releases)) + } + continue + } + w.delivered(d.agent, d.releases, now) + res.Kicked = append(res.Kicked, d.agent) + w.opts.Logger.Info("release feed kicked agent", "agent", d.agent, "reason", Reason(d.releases)) + } + w.save() + return res +} + +type dueKick struct { + agent string + releases []Release +} + +// diff records new versions as pending and returns the agents due a kick. +func (w *Watcher) diff(subs map[string][]Source, now time.Time) ([]dueKick, int) { + w.mu.Lock() + defer w.mu.Unlock() + detected := 0 + live := map[string]bool{} + for agent := range w.state.Agents { + if _, ok := subs[agent]; !ok { + delete(w.state.Agents, agent) + } + } + agents := make([]string, 0, len(subs)) + for agent, srcs := range subs { + agents = append(agents, agent) + a := w.state.agent(agent) + subscribed := map[string]bool{} + for _, src := range srcs { + key := src.Key() + live[key] = true + subscribed[key] = true + cur := w.state.Sources[key] + if cur.Version == "" { + continue + } + seen := a.Seen[key] + if seen == "" { + // First answer: a baseline, not a release. + a.Seen[key] = cur.Version + continue + } + if seen == cur.Version { + continue + } + addPending(a, Release{Source: key, Package: src.Label(), Old: seen, New: cur.Version, URL: cur.URL}) + a.Seen[key] = cur.Version + detected++ + } + for key := range a.Seen { + if !subscribed[key] { + delete(a.Seen, key) + } + } + kept := a.Pending[:0] + for _, r := range a.Pending { + if subscribed[r.Source] { + kept = append(kept, r) + } + } + a.Pending = kept + } + for key := range w.state.Sources { + if !live[key] { + delete(w.state.Sources, key) + } + } + sort.Strings(agents) + var due []dueKick + for _, agent := range agents { + a := w.state.Agents[agent] + if len(a.Pending) == 0 { + continue + } + cooldown := time.Duration(0) + if w.opts.Cooldown != nil { + cooldown = w.opts.Cooldown(agent) + } + if !a.LastKick.IsZero() && now.Sub(a.LastKick) < cooldown { + continue + } + due = append(due, dueKick{agent: agent, releases: append([]Release(nil), a.Pending...)}) + } + return due, detected +} + +// addPending folds r into the agent's pending list: a feed that moved again +// before its kick keeps the original old version and takes the newest one. +func addPending(a *AgentState, r Release) { + for i := range a.Pending { + if a.Pending[i].Source != r.Source { + continue + } + a.Pending[i].New, a.Pending[i].URL, a.Pending[i].Package = r.New, r.URL, r.Package + if a.Pending[i].Old == a.Pending[i].New { + a.Pending = append(a.Pending[:i], a.Pending[i+1:]...) + } + return + } + a.Pending = append(a.Pending, r) +} + +// delivered drops the releases a kick carried and starts the cooldown. A +// release that moved again while the kick was in flight stays pending. +func (w *Watcher) delivered(agent string, sent []Release, now time.Time) { + w.mu.Lock() + defer w.mu.Unlock() + a := w.state.Agents[agent] + if a == nil { + return + } + done := map[string]string{} + for _, r := range sent { + done[r.Source] = r.New + } + kept := a.Pending[:0] + for _, r := range a.Pending { + if v, ok := done[r.Source]; ok && v == r.New { + continue + } + if v, ok := done[r.Source]; ok { + r.Old = v + } + kept = append(kept, r) + } + a.Pending = kept + a.LastKick = now +} + +func (w *Watcher) ensureLoaded() { + w.mu.Lock() + defer w.mu.Unlock() + if w.loaded { + return + } + w.loaded = true + w.state = newState() + if w.opts.Store == nil { + return + } + st, err := w.opts.Store.Load() + if err != nil { + w.opts.Logger.Warn("release feed state unreadable; starting fresh (first answers are baselines)", "error", err) + return + } + w.state = st +} + +func (w *Watcher) save() { + if w.opts.Store == nil { + return + } + w.mu.Lock() + defer w.mu.Unlock() + if err := w.opts.Store.Save(w.state); err != nil { + w.opts.Logger.Warn("release feed state not saved", "error", err) + } +} diff --git a/src/pkg/releasefeed/watcher_test.go b/src/pkg/releasefeed/watcher_test.go new file mode 100644 index 0000000000..30470e4c12 --- /dev/null +++ b/src/pkg/releasefeed/watcher_test.go @@ -0,0 +1,359 @@ +package releasefeed + +import ( + "context" + "errors" + "fmt" + "io" + "log/slog" + "net/http" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +type kickCall struct { + agent string + releases []Release + block string +} + +type fakeKicker struct { + w *Watcher + calls []kickCall + err error +} + +func (k *fakeKicker) Kick(_ context.Context, agent string, releases []Release) error { + if k.err != nil { + return k.err + } + // The kicker renders the agent's prompt while the watcher is unlocked, + // so ${NEW_RELEASES} (Block) must already carry the pending releases. + k.calls = append(k.calls, kickCall{agent: agent, releases: releases, block: k.w.Block(agent)}) + return nil +} + +type watcherRig struct { + feed *fakeFeed + w *Watcher + kicker *fakeKicker + now time.Time + subs map[string][]Source + store *FileStore + token string + tokens []string +} + +func quietLogger() *slog.Logger { return slog.New(slog.NewTextHandler(io.Discard, nil)) } + +func newWatcherRig(t *testing.T) *watcherRig { + t.Helper() + r := &watcherRig{ + feed: newFakeFeed(t), + now: time.Date(2026, 11, 16, 12, 0, 0, 0, time.UTC), + store: NewFileStore(filepath.Join(t.TempDir(), "state", "release-feeds.json")), + } + r.subs = map[string][]Source{"deps": {{Kind: KindPyPI, Package: "requests"}}} + r.kicker = &fakeKicker{} + r.w = r.newWatcher() + return r +} + +func (r *watcherRig) newWatcher() *Watcher { + w := New(Options{ + Subscriptions: func() map[string][]Source { return r.subs }, + Interval: func() time.Duration { return time.Minute }, + Cooldown: func(string) time.Duration { return time.Hour }, + Token: func(agent, name string) string { + r.tokens = append(r.tokens, agent+":"+name) + return r.token + }, + Fetcher: r.feed.fetcher(), + Store: r.store, + Kicker: r.kicker, + Logger: quietLogger(), + Now: func() time.Time { return r.now }, + }) + r.kicker.w = w + return w +} + +// The done-when of #18494: the first answer is a baseline (no kick), an +// unchanged feed kicks nothing, and a new version kicks the subscriber in +// the same poll with the release block in its prompt. +func TestWatcher_BaselineThenKickOnNewVersion(t *testing.T) { + r := newWatcherRig(t) + ctx := context.Background() + + res := r.w.Poll(ctx) + if res.Fetched != 1 || len(res.Kicked) != 0 || len(r.kicker.calls) != 0 { + t.Fatalf("baseline poll: %+v, calls %d", res, len(r.kicker.calls)) + } + res = r.w.Poll(ctx) + if res.NotModified != 1 || len(r.kicker.calls) != 0 { + t.Fatalf("unchanged poll must not kick: %+v, calls %d", res, len(r.kicker.calls)) + } + if r.feed.seen().ifNoneMatch != `"v1"` { + t.Fatalf("second poll was not conditional: %+v", r.feed.seen()) + } + + r.feed.set(func(f *fakeFeed) { f.version, f.etag = "1.1.0", `"v2"` }) + res = r.w.Poll(ctx) + if res.Detected != 1 || len(res.Kicked) != 1 || len(r.kicker.calls) != 1 { + t.Fatalf("new version poll: %+v, calls %d", res, len(r.kicker.calls)) + } + call := r.kicker.calls[0] + want := Release{Source: "pypi:requests", Package: "requests", Old: "1.0.0", New: "1.1.0", URL: "https://pypi.org/project/requests/1.1.0/"} + if call.agent != "deps" || len(call.releases) != 1 || call.releases[0] != want { + t.Fatalf("kick = %+v", call) + } + if !strings.Contains(call.block, BlockHeader) || !strings.Contains(call.block, "- requests: 1.0.0 -> 1.1.0 (https://pypi.org/project/requests/1.1.0/)") { + t.Fatalf("block = %q", call.block) + } + if got := r.w.Block("deps"); got != "" { + t.Fatalf("delivered releases still pending: %q", got) + } + + // Nothing changed again: no second kick. + r.w.Poll(ctx) + if len(r.kicker.calls) != 1 { + t.Fatalf("unchanged feed kicked again: %d calls", len(r.kicker.calls)) + } +} + +// A second release inside the cooldown waits; it goes out once the cooldown +// has passed, folded into one entry that keeps the original old version. +func TestWatcher_CooldownHoldsThenDelivers(t *testing.T) { + r := newWatcherRig(t) + ctx := context.Background() + r.w.Poll(ctx) + r.feed.set(func(f *fakeFeed) { f.version, f.etag = "1.1.0", `"v2"` }) + r.w.Poll(ctx) + if len(r.kicker.calls) != 1 { + t.Fatalf("first release: %d calls", len(r.kicker.calls)) + } + + r.now = r.now.Add(10 * time.Minute) + r.feed.set(func(f *fakeFeed) { f.version, f.etag = "1.2.0", `"v3"` }) + r.w.Poll(ctx) + r.feed.set(func(f *fakeFeed) { f.version, f.etag = "1.3.0", `"v4"` }) + r.w.Poll(ctx) + if len(r.kicker.calls) != 1 { + t.Fatalf("cooldown did not hold: %d calls", len(r.kicker.calls)) + } + if b := r.w.Block("deps"); !strings.Contains(b, "1.1.0 -> 1.3.0") { + t.Fatalf("pending block = %q", b) + } + + r.now = r.now.Add(time.Hour) + r.w.Poll(ctx) + if len(r.kicker.calls) != 2 { + t.Fatalf("after cooldown: %d calls", len(r.kicker.calls)) + } + if got := r.kicker.calls[1].releases; len(got) != 1 || got[0].Old != "1.1.0" || got[0].New != "1.3.0" { + t.Fatalf("folded release = %+v", got) + } +} + +// A gated or failed kick leaves the release pending for the next poll. +func TestWatcher_GatedKickStaysPending(t *testing.T) { + r := newWatcherRig(t) + ctx := context.Background() + r.w.Poll(ctx) + r.feed.set(func(f *fakeFeed) { f.version, f.etag = "2.0.0", `"v2"` }) + r.kicker.err = fmt.Errorf("%w: agent paused", ErrGated) + if res := r.w.Poll(ctx); len(res.Kicked) != 0 { + t.Fatalf("gated poll kicked: %+v", res) + } + r.kicker.err = errors.New("pane gone") + r.w.Poll(ctx) + if b := r.w.Block("deps"); !strings.Contains(b, "1.0.0 -> 2.0.0") { + t.Fatalf("pending lost after refusals: %q", b) + } + r.kicker.err = nil + if res := r.w.Poll(ctx); len(res.Kicked) != 1 { + t.Fatalf("open gate did not deliver: %+v", res) + } +} + +// Last-seen versions survive a restart: a fresh watcher on the same state +// file does not re-baseline, so a release published while the hive was down +// still kicks. +func TestWatcher_StatePersistsAcrossRestart(t *testing.T) { + r := newWatcherRig(t) + ctx := context.Background() + r.w.Poll(ctx) + + r.feed.set(func(f *fakeFeed) { f.version, f.etag = "1.5.0", `"v5"` }) + r.w = r.newWatcher() + r.w.Poll(ctx) + if len(r.kicker.calls) != 1 || r.kicker.calls[0].releases[0].Old != "1.0.0" { + t.Fatalf("restart re-baselined: %+v", r.kicker.calls) + } + data, err := os.ReadFile(r.store.path) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(data), `"1.5.0"`) { + t.Fatalf("state file missing the version: %s", data) + } +} + +// A source with auth_var resolves the token for its subscriber and sends it; +// an unresolved variable skips the feed instead of polling anonymously. +func TestWatcher_AuthVar(t *testing.T) { + r := newWatcherRig(t) + r.subs = map[string][]Source{"deps": {{Kind: KindIndex, URL: r.feed.srv.URL + "/private", AuthVar: "INDEX_TOKEN"}}} + ctx := context.Background() + + res := r.w.Poll(ctx) + if res.Errors != 1 || r.feedHits() != 0 { + t.Fatalf("unset token must skip the feed: %+v hits=%d", res, r.feedHits()) + } + r.token = "s3cret" + r.w.Poll(ctx) + if got := r.feed.seen().auth; got != "Bearer s3cret" { + t.Fatalf("token not sent: %q", got) + } + if len(r.tokens) == 0 || r.tokens[0] != "deps:INDEX_TOKEN" { + t.Fatalf("token lookups = %v", r.tokens) + } + data, _ := os.ReadFile(r.store.path) + if strings.Contains(string(data), "s3cret") { + t.Fatal("token reached the state file") + } +} + +// Feeds shared by two agents are fetched once and kick both; an agent that +// drops its subscription is forgotten, and a feed error is counted. +func TestWatcher_SharedFeedPruneAndErrors(t *testing.T) { + r := newWatcherRig(t) + src := Source{Kind: KindGitHubRelease, Repo: "psf/requests"} + r.subs = map[string][]Source{"a": {src}, "b": {src}} + ctx := context.Background() + r.w.Poll(ctx) + if r.feedHits() != 1 { + t.Fatalf("shared feed fetched %d times", r.feedHits()) + } + r.feed.set(func(f *fakeFeed) { f.version, f.etag = "1.1.0", `"v2"` }) + if res := r.w.Poll(ctx); len(res.Kicked) != 2 { + t.Fatalf("both subscribers should be kicked: %+v", res) + } + + r.subs = map[string][]Source{"a": {{Kind: KindPyPI, Package: "other"}}} + r.w.Poll(ctx) + r.w.mu.Lock() + _, bKept := r.w.state.Agents["b"] + _, oldKept := r.w.state.Sources[src.Key()] + _, oldSeen := r.w.state.Agents["a"].Seen[src.Key()] + r.w.mu.Unlock() + if bKept || oldKept || oldSeen { + t.Fatalf("unsubscribed state kept: agent b=%v source=%v seen=%v", bKept, oldKept, oldSeen) + } + + r.feed.set(func(f *fakeFeed) { f.status = http.StatusInternalServerError }) + if res := r.w.Poll(ctx); res.Errors != 1 { + t.Fatalf("feed error not counted: %+v", res) + } +} + +func TestWatcher_DisabledNilKickerAndRun(t *testing.T) { + r := newWatcherRig(t) + off := New(Options{Enabled: func() bool { return false }, Fetcher: r.feed.fetcher()}) + if res := off.Poll(context.Background()); res.Fetched != 0 || res.Errors != 0 || r.feedHits() != 0 { + t.Fatalf("disabled watcher polled: %+v", res) + } + var nilW *Watcher + if nilW.Block("x") != "" || off.Block("x") != "" { + t.Fatal("Block on an unloaded watcher must be empty") + } + + // No kicker and no store: detection still works, nothing panics. + w := New(Options{ + Subscriptions: func() map[string][]Source { return r.subs }, + Fetcher: r.feed.fetcher(), + }) + w.Poll(context.Background()) + r.feed.set(func(f *fakeFeed) { f.version, f.etag = "3.0.0", `"v3"` }) + if res := w.Poll(context.Background()); res.Detected != 1 || len(res.Kicked) != 0 { + t.Fatalf("nil kicker: %+v", res) + } + + // Run polls at once and returns when the context ends. + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + hits := r.feedHits() + go func() { w.Run(ctx); close(done) }() + deadline := time.After(5 * time.Second) + for r.feedHits() == hits { + select { + case <-deadline: + t.Fatal("Run did not poll") + case <-time.After(10 * time.Millisecond): + } + } + cancel() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("Run did not stop on cancel") + } + + // A cancelled context stops a poll before it fetches. + cctx, ccancel := context.WithCancel(context.Background()) + ccancel() + before := r.feedHits() + w.Poll(cctx) + if r.feedHits() != before { + t.Fatal("cancelled poll fetched") + } +} + +func (r *watcherRig) feedHits() int { return r.feed.seen().hits } + +func TestFormatBlockAndReason(t *testing.T) { + if FormatBlock(nil) != "" { + t.Fatal("empty block must be empty") + } + rels := []Release{ + {Package: "a", Old: "1", New: "2"}, + {Package: "b", Old: "1", New: "2", URL: "https://x"}, + {Package: "c", Old: "1", New: "2"}, + {Package: "d", Old: "1", New: "2"}, + } + b := FormatBlock(rels) + if !strings.HasPrefix(b, BlockHeader+"\n- a: 1 -> 2\n- b: 1 -> 2 (https://x)\n") { + t.Fatalf("block = %q", b) + } + if got := Reason(rels); got != "new release: a 1 -> 2, b 1 -> 2, c 1 -> 2 (+1 more)" { + t.Fatalf("reason = %q", got) + } + if got := Reason(rels[:1]); got != "new release: a 1 -> 2" { + t.Fatalf("reason = %q", got) + } +} + +func TestAddPendingRevertDropsEntry(t *testing.T) { + a := &AgentState{} + addPending(a, Release{Source: "k", Old: "1", New: "2"}) + addPending(a, Release{Source: "k", Old: "2", New: "1"}) + if len(a.Pending) != 0 { + t.Fatalf("a version that moved back must drop the entry: %+v", a.Pending) + } +} + +func TestDelivered_KeepsNewerPending(t *testing.T) { + w := New(Options{}) + w.ensureLoaded() + a := w.state.agent("deps") + a.Pending = []Release{{Source: "k", Old: "1", New: "3"}, {Source: "j", Old: "1", New: "2"}} + now := time.Now() + w.delivered("deps", []Release{{Source: "k", Old: "1", New: "2"}, {Source: "j", Old: "1", New: "2"}}, now) + if len(a.Pending) != 1 || a.Pending[0].Old != "2" || a.Pending[0].New != "3" || !a.LastKick.Equal(now) { + t.Fatalf("pending after delivery = %+v", a.Pending) + } + w.delivered("ghost", nil, now) +} diff --git a/src/pkg/scheduler/new_releases.go b/src/pkg/scheduler/new_releases.go new file mode 100644 index 0000000000..d759780fa8 --- /dev/null +++ b/src/pkg/scheduler/new_releases.go @@ -0,0 +1,33 @@ +package scheduler + +// NewReleasesSource renders the ${NEW_RELEASES} block for an agent: the +// upstream releases the release-feed watcher (hivecommons/hive#18494) found +// and has not yet delivered. "" means none. +type NewReleasesSource func(agentName string) string + +// newReleasesHolder wraps the installed source so atomic.Pointer can hold a +// func value. +type newReleasesHolder struct{ fn NewReleasesSource } + +// SetNewReleasesSource installs (or with nil, removes) the ${NEW_RELEASES} +// source. It is an atomic swap, not s.mu, because template substitution may +// run while s.mu is held. +func (s *Scheduler) SetNewReleasesSource(fn NewReleasesSource) { + if fn == nil { + s.newReleases.Store(nil) + return + } + s.newReleases.Store(&newReleasesHolder{fn: fn}) +} + +// newReleasesBlock is the ${NEW_RELEASES} value for agentName. +func (s *Scheduler) newReleasesBlock(agentName string) string { + if s == nil { + return "" + } + h := s.newReleases.Load() + if h == nil || h.fn == nil { + return "" + } + return h.fn(agentName) +} diff --git a/src/pkg/scheduler/new_releases_test.go b/src/pkg/scheduler/new_releases_test.go new file mode 100644 index 0000000000..09c2b87314 --- /dev/null +++ b/src/pkg/scheduler/new_releases_test.go @@ -0,0 +1,41 @@ +package scheduler + +import ( + "strings" + "testing" + + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/github" +) + +// ${NEW_RELEASES} (#18494) renders the installed source's block for the +// kicked agent, renders empty with no source, and clears when the source is +// removed. +func TestSubstituteTemplate_NewReleases(t *testing.T) { + s := New(&config.Config{Project: config.ProjectConfig{Org: "acme", Repos: []string{"widgets"}}}, testLogger()) + tmpl := "before\n${NEW_RELEASES}after" + + got, _ := s.substituteTemplateWithPolicy(tmpl, &github.ActionableResult{}, "deps", nil) + if got != "before\nafter" { + t.Fatalf("no source: %q", got) + } + + var asked string + s.SetNewReleasesSource(func(agent string) string { + asked = agent + return "NEW UPSTREAM RELEASES:\n- requests: 2.32.0 -> 2.33.0\n" + }) + got, _ = s.substituteTemplateWithPolicy(tmpl, &github.ActionableResult{}, "deps", nil) + if asked != "deps" || !strings.Contains(got, "- requests: 2.32.0 -> 2.33.0") { + t.Fatalf("with source: asked=%q got=%q", asked, got) + } + + s.SetNewReleasesSource(nil) + if got := s.newReleasesBlock("deps"); got != "" { + t.Fatalf("removed source still renders %q", got) + } + var nilSched *Scheduler + if got := nilSched.newReleasesBlock("deps"); got != "" { + t.Fatalf("nil scheduler renders %q", got) + } +} diff --git a/src/pkg/scheduler/scheduler.go b/src/pkg/scheduler/scheduler.go index e0f652e1a9..b46e56351f 100644 --- a/src/pkg/scheduler/scheduler.go +++ b/src/pkg/scheduler/scheduler.go @@ -50,19 +50,22 @@ type Scheduler struct { prLease PRLeaseLookup maintainerNotes MaintainerNotesLookup topicClaims TopicClaimLookup - runAdmitter RunAdmitter - triageCommenter TriageCommenter - clarity clarityGateState - priorArt priorArtGateState - priorArtSearcher PriorArtSearcher - questionAutocloser QuestionAutocloser - lifecycle timeline.Recorder - scanned bool - deferredKicks map[string]func(string) - firstScanDeferred map[string]bool - ioscanWarned sync.Map - lastClassifierErr atomic.Value // string: latest fail-closed classifier error - mu sync.RWMutex + // newReleases feeds ${NEW_RELEASES} (new_releases.go); atomic because + // template substitution may run with s.mu held. + newReleases atomic.Pointer[newReleasesHolder] + runAdmitter RunAdmitter + triageCommenter TriageCommenter + clarity clarityGateState + priorArt priorArtGateState + priorArtSearcher PriorArtSearcher + questionAutocloser QuestionAutocloser + lifecycle timeline.Recorder + scanned bool + deferredKicks map[string]func(string) + firstScanDeferred map[string]bool + ioscanWarned sync.Map + lastClassifierErr atomic.Value // string: latest fail-closed classifier error + mu sync.RWMutex } // SetRunTriageDeps attaches the narrow surfaces the scheduler needs for the diff --git a/src/pkg/scheduler/templates.go b/src/pkg/scheduler/templates.go index 0d5b3cd21d..24a6e86b76 100644 --- a/src/pkg/scheduler/templates.go +++ b/src/pkg/scheduler/templates.go @@ -428,6 +428,9 @@ func (s *Scheduler) substituteTemplateWithVars(template string, actionable *gith "INCEPTION_REPO_URL": lit(""), "MERGE_ELIGIBLE": lit(mergeEligibleList), "CI_FAILING": lit(ciFailingList), + // Upstream releases the release-feed watcher found for this agent + // and has not yet delivered (hivecommons/hive#18494); "" when none. + "NEW_RELEASES": func() string { return s.newReleasesBlock(agentName) }, }} // Caller-supplied vars lose a name clash: one already claimed by a built-in // above is left alone, so no call site can redefine ${GH_AUTH}. diff --git a/src/pkg/snapshot/state.go b/src/pkg/snapshot/state.go index 419489f0bc..cde7308daa 100644 --- a/src/pkg/snapshot/state.go +++ b/src/pkg/snapshot/state.go @@ -8,8 +8,8 @@ import ( "path/filepath" "time" - "github.com/hivecommons/hive/pkg/config" "github.com/hivecommons/hive/crux/watchdog" + "github.com/hivecommons/hive/pkg/config" ) const maxStateAge = 7 * 24 * time.Hour @@ -169,6 +169,8 @@ type GovKickEntry struct { // "stand-down", "no-op", "ended" — or empty when never classified. Outcome string `json:"outcome,omitempty"` OutcomeReason string `json:"outcome_reason,omitempty"` + // Reason is why an event-driven kick went out (governor.KickRecord.Reason). + Reason string `json:"reason,omitempty"` } type AgentKickEntry struct { From 1689e63528beae53faee00a99953d682fa9f41da Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Mon, 16 Nov 2026 19:38:24 -0500 Subject: [PATCH 2/2] =?UTF-8?q?=F0=9F=90=9B=20release-feed=20watcher:=20re?= =?UTF-8?q?ad=20the=20published=20config=20snapshot,=20not=20b.cfg?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A config reload overwrites b.cfg in place under cfgReloadMu, so the watcher goroutine reading it raced the reload (caught by the race detector in TestRunLoopWithAppliesMidCycleReloadWithoutClobbering). Read b.sweepConfig, the immutable snapshot the loop republishes after every reload, instead. Signed-off-by: Andy Anderson --- src/cmd/hive/release_feed_watcher.go | 18 ++++++++++++------ src/cmd/hive/release_feed_watcher_test.go | 6 +++++- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/src/cmd/hive/release_feed_watcher.go b/src/cmd/hive/release_feed_watcher.go index 8fbb2011c9..c4058fb2a0 100644 --- a/src/cmd/hive/release_feed_watcher.go +++ b/src/cmd/hive/release_feed_watcher.go @@ -135,16 +135,22 @@ func releaseFeedToken(cfg *config.Config, agent, name string) string { return strings.TrimSpace(v) } +// releaseFeedConfig is the config the watcher goroutine reads: the immutable +// snapshot the governor loop publishes after every reload (b.sweepConfig), +// never b.cfg itself, which a reload overwrites in place under cfgReloadMu. +// Nil (not yet published) reads as disabled. +func (b *boot) releaseFeedConfig() *config.Config { return b.sweepConfig.Load() } + // newReleaseFeedWatcher builds the watcher from boot state. Config, governor, // scheduler and runtime are resolved per call so a reload or late wiring is // picked up without restarting it. func (b *boot) newReleaseFeedWatcher() *releasefeed.Watcher { return releasefeed.New(releasefeed.Options{ - Enabled: func() bool { return b.cfg.ReleaseFeedsEnabled() }, - Subscriptions: func() map[string][]releasefeed.Source { return releaseFeedSubscriptions(b.cfg) }, - Interval: func() time.Duration { return b.cfg.ReleaseFeedInterval() }, - Cooldown: func(agent string) time.Duration { return b.cfg.ReleaseFeedCooldown(agent) }, - Token: func(agent, name string) string { return releaseFeedToken(b.cfg, agent, name) }, + Enabled: func() bool { return b.releaseFeedConfig().ReleaseFeedsEnabled() }, + Subscriptions: func() map[string][]releasefeed.Source { return releaseFeedSubscriptions(b.releaseFeedConfig()) }, + Interval: func() time.Duration { return b.releaseFeedConfig().ReleaseFeedInterval() }, + Cooldown: func(agent string) time.Duration { return b.releaseFeedConfig().ReleaseFeedCooldown(agent) }, + Token: func(agent, name string) string { return releaseFeedToken(b.releaseFeedConfig(), agent, name) }, Store: releasefeed.NewFileStore(releaseFeedStatePath), Kicker: b.newReleaseFeedKicker(), Logger: b.logger, @@ -154,7 +160,7 @@ func (b *boot) newReleaseFeedWatcher() *releasefeed.Watcher { // newReleaseFeedKicker wires the kicker to boot state. func (b *boot) newReleaseFeedKicker() *releaseFeedKicker { return &releaseFeedKicker{ - cfg: func() *config.Config { return b.cfg }, + cfg: b.releaseFeedConfig, gov: func() *governor.Governor { return b.gov }, isPaused: func(agent string) bool { return b.agentMgr != nil && b.agentMgr.IsPaused(agent) }, render: func(agent string) string { diff --git a/src/cmd/hive/release_feed_watcher_test.go b/src/cmd/hive/release_feed_watcher_test.go index 571affbd55..e05d371e03 100644 --- a/src/cmd/hive/release_feed_watcher_test.go +++ b/src/cmd/hive/release_feed_watcher_test.go @@ -205,8 +205,12 @@ func TestNewReleaseFeedKicker_ResolvesBootState(t *testing.T) { logger := slog.New(slog.NewTextHandler(io.Discard, nil)) b := &boot{cfg: cfg, logger: logger} k := b.newReleaseFeedKicker() + if k.config() != nil { + t.Fatal("kicker must read the published snapshot, not b.cfg") + } + b.sweepConfig.Store(cfg) if k.config() != cfg || k.governor() != nil { - t.Fatal("kicker not bound to boot config/governor") + t.Fatal("kicker not bound to the published config/governor") } if k.isPaused("quality") { t.Fatal("no runtime must read as not paused")