diff --git a/changelog.d/fixed-14678-reviewer-gateway-model.md b/changelog.d/fixed-14678-reviewer-gateway-model.md new file mode 100644 index 0000000000..c8960efb2e --- /dev/null +++ b/changelog.d/fixed-14678-reviewer-gateway-model.md @@ -0,0 +1 @@ +- ACMM pack agents moved onto an inference gateway that has no default model now get the model the hive's other agents on that gateway run, instead of being left with no model, failing to start and being paused. diff --git a/src/docs/agent-configuration.md b/src/docs/agent-configuration.md index 10738aa28c..1c0d0962c7 100644 --- a/src/docs/agent-configuration.md +++ b/src/docs/agent-configuration.md @@ -683,7 +683,7 @@ backends: Names compare case-insensitively; an empty `backend:` (the hive default) is always allowed. The list is enforced where placements are decided: - **ACMM pack apply** never places an agent on a disallowed backend. A pack agent whose pack backend is disallowed gets the first allowed `allow` entry instead (or the hive default when `allow` is empty), without the pack's model, so the replacement backend uses its own default model. A pack-owned agent already on a now-disallowed backend is moved the same way on the next apply. Backends an operator explicitly chose (operator-owned) are left alone. -- **ACMM pack agents follow the hive's inference gateway** ([#14678](https://github.com/hivecommons/hive/issues/14678)). When every backend you picked yourself (operator-owned — and any inference backend or gateway name with no `backend_owner`, e.g. one written straight into `hive.yaml`, since a pack never writes one) names the same inference backend or configured gateway, and more than half of the other enabled agents already run on it, a pack agent whose pack backend is a subscription CLI is placed on that gateway instead, with no `model:` so the gateway's default model applies. This applies to agents a level creates and, on every apply (each restart), to existing agents whose backend is still pack-owned — so after an upgrade the L2 `supervisor`/`brainstorm` may move from `copilot` onto your LiteLLM gateway alongside the new `reviewer`. An operator-owned `backend:` or a legacy `backend:` without an owner is never moved, a hive with an operator-picked CLI agent or with only a minority of agents on a gateway is left as is, and a pack that itself names a gateway or whose backend the `backends:` list replaced is not redirected. +- **ACMM pack agents follow the hive's inference gateway** ([#14678](https://github.com/hivecommons/hive/issues/14678)). When every backend you picked yourself (operator-owned — and any inference backend or gateway name with no `backend_owner`, e.g. one written straight into `hive.yaml`, since a pack never writes one) names the same inference backend or configured gateway, and more than half of the other enabled agents already run on it, a pack agent whose pack backend is a subscription CLI is placed on that gateway instead, with no `model:` so the gateway's default model applies — or, when the gateway has no default model, with the model your agents on it run (from their `model:` or the model picked on their dashboard card; the most common one if they differ), so it never launches without a model. An agent the watchdog paused for crash-looping without a model is resumed once it gets one. This applies to agents a level creates and, on every apply (each restart), to existing agents whose backend is still pack-owned — so after an upgrade the L2 `supervisor`/`brainstorm` may move from `copilot` onto your LiteLLM gateway alongside the new `reviewer`. An operator-owned `backend:` or a legacy `backend:` without an owner is never moved, a hive with an operator-picked CLI agent or with only a minority of agents on a gateway is left as is, and a pack that itself names a gateway or whose backend the `backends:` list replaced is not redirected. - **`PUT /api/config/agent/{name}/models`** rejects a placement onto a disallowed backend with `400` and a message naming it. ### Changing backend and model together: use the atomic endpoint diff --git a/src/pkg/dashboard/api_packs.go b/src/pkg/dashboard/api_packs.go index a4f763608e..b673fa193f 100644 --- a/src/pkg/dashboard/api_packs.go +++ b/src/pkg/dashboard/api_packs.go @@ -11,6 +11,7 @@ import ( "time" "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/watchdog" "github.com/hivecommons/hive/pkg/config" ghpkg "github.com/hivecommons/hive/pkg/github" "github.com/hivecommons/hive/pkg/hooks" @@ -147,6 +148,9 @@ func (s *Server) applyPack(level int, forceGovernor bool) (*ApplyPackResult, err // the manager's process table and were re-registered. A repair, not a // creation — it never feeds `created` or isFirstApplyOrExpansion (#5632). var readded []string + // gatewayModelFilled collects existing gateway-routed agents this apply + // gave a model they had none of; see resumeGatewayModelCrashLoops. + var gatewayModelFilled []string // Read once, before this apply creates or moves anything, so every pack // agent judges the hive's gateway on the same pre-apply roster rather // than on agents this apply already moved. @@ -226,7 +230,8 @@ func (s *Server) applyPack(level int, forceGovernor bool) (*ApplyPackResult, err "agent", pa.Name, "backend", existing.Backend, "gateway", gw) existing.Backend = gw if !existing.ModelIsOperatorOwned() { - existing.Model = "" + existing.Model = s.hiveGatewayModel(hiveBackends, pa.Name, gw) + existing.ModelOwner = config.FieldOwnerPack } changed = true } @@ -246,10 +251,17 @@ func (s *Server) applyPack(level int, forceGovernor bool) (*ApplyPackResult, err existing.ModelOwner = config.FieldOwnerPack changed = true } - if onForeignGateway && existing.Model != "" && !existing.ModelIsOperatorOwned() { - s.logger.Info("clearing pack model on gateway-routed agent so the gateway default_model applies", - "agent", pa.Name, "backend", existing.Backend, "model", existing.Model, "pack_backend", packBackend) - existing.Model = "" + // With no gateway default_model the agent takes the model the + // operator's agents run there instead (hiveGatewayModel), so an + // agent an earlier apply left with none is repaired too. + if gwModel := s.hiveGatewayModel(hiveBackends, pa.Name, existing.Backend); onForeignGateway && + existing.Model != gwModel && !existing.ModelIsOperatorOwned() { + s.logger.Info("setting pack model on gateway-routed agent to the gateway's model", + "agent", pa.Name, "backend", existing.Backend, "model", existing.Model, "gateway_model", gwModel, "pack_backend", packBackend) + if existing.Model == "" { + gatewayModelFilled = append(gatewayModelFilled, pa.Name) + } + existing.Model = gwModel existing.ModelOwner = config.FieldOwnerPack changed = true } @@ -399,10 +411,10 @@ func (s *Server) applyPack(level int, forceGovernor bool) (*ApplyPackResult, err packModel = "" } // The pack model is an id from the pack CLI's catalogue; on the - // gateway it is cleared so the gateway's default_model applies (#9945). + // gateway it is replaced by the gateway's model (#9945, #14678). if gw := gatewayPlacement(packBackend, backendReplaced, hiveGateway); gw != "" { packBackend = gw - packModel = "" + packModel = s.hiveGatewayModel(hiveBackends, pa.Name, gw) } agentCfg := config.AgentConfig{ Backend: packBackend, @@ -454,6 +466,8 @@ func (s *Server) applyPack(level int, forceGovernor bool) (*ApplyPackResult, err created = append(created, pa.Name) } + s.resumeGatewayModelCrashLoops(gatewayModelFilled) + s.deps.Config.ACMMLevel = &level if err := s.saveConfig(); err != nil { s.logger.Error("failed to save ACMM level to hive.yaml", "error", err) @@ -1169,6 +1183,10 @@ type agentBackendVote struct { backend string operatorOwned bool packCLI bool + // model is the model the agent runs on backend when an operator chose + // it (a live ModelOverride, or a config model the pack did not write), + // else "". hiveGatewayModel reads it. + model string } // hiveAgentBackends snapshots the backend of every enabled agent, keyed by @@ -1182,7 +1200,10 @@ type agentBackendVote struct { // LiteLLM on every card while this vote still read their pack-owned copilot // config backend, found no gateway, and left the reviewer on copilot. An // override differing from the config backend is an explicit placement, so it -// counts as an operator pick. +// counts as an operator pick. So does a gateway override over a pack-owned +// config backend equal to it: an earlier apply moved the config onto the +// gateway the override already named, and the operator's pick must keep +// counting on the next restart. func (s *Server) hiveAgentBackends() map[string]agentBackendVote { backends := map[string]agentBackendVote{} for name, agent := range s.deps.Config.Agents { @@ -1195,25 +1216,34 @@ func (s *Server) hiveAgentBackends() map[string]agentBackendVote { operatorOwned: agent.BackendIsOperatorOwned() || s.unownedGatewayBackend(agent), packCLI: agent.BackendIsPackOwned() && config.IsCLIBackend(backend), } - if override := s.agentBackendOverride(name); override != "" && !strings.EqualFold(override, backend) { - vote = agentBackendVote{backend: override, operatorOwned: true} + if agent.ModelOwner != config.FieldOwnerPack { + vote.model = strings.TrimSpace(agent.Model) + } + backendOverride, modelOverride := s.agentRuntimeOverrides(name) + if backendOverride != "" && (!strings.EqualFold(backendOverride, backend) || + agent.BackendIsPackOwned() && s.agentOnForeignGatewayBackend(backendOverride, "")) { + vote = agentBackendVote{backend: backendOverride, operatorOwned: true} + } + if modelOverride != "" { + vote.model = modelOverride } backends[name] = vote } return backends } -// agentBackendOverride is name's live BackendOverride, or "" when the agent -// has none or is not in the manager's process table. -func (s *Server) agentBackendOverride(name string) string { +// agentRuntimeOverrides is name's live BackendOverride and ModelOverride, "" +// for each the agent has none of or when it is not in the manager's process +// table. +func (s *Server) agentRuntimeOverrides(name string) (backend, model string) { if s.deps.AgentMgr == nil { - return "" + return "", "" } proc, err := s.deps.AgentMgr.GetStatus(name) if err != nil || proc == nil { - return "" + return "", "" } - return strings.TrimSpace(proc.BackendOverride) + return strings.TrimSpace(proc.BackendOverride), strings.TrimSpace(proc.ModelOverride) } // unownedGatewayBackend reports whether agent sits on an inference backend or @@ -1315,6 +1345,60 @@ func (s *Server) hiveGatewayBackend(backends map[string]agentBackendVote, except return shared } +// hiveGatewayModel is the model a pack-owned agent on gateway is given in +// place of the pack's model, or "" to leave it unset so the gateway's +// default_model applies. +// +// When the gateway has no default_model, an agent with no model launches with +// none and cannot run (#14678): the operator's agents each ran on a model +// picked in the dashboard, while the L2 reviewer the pack moved onto the same +// LiteLLM gateway had its model cleared, crash-looped and was paused. It +// therefore takes the model the operator's agents on that gateway run (the +// most common, ties broken by name), excluding except, the agent being placed. +func (s *Server) hiveGatewayModel(backends map[string]agentBackendVote, except, gateway string) string { + if gateway == "" || s.deps.Config.Governor.DefaultInferenceModel(gateway) != "" { + return "" + } + counts := map[string]int{} + for name, vote := range backends { + if strings.EqualFold(name, except) || !vote.operatorOwned || vote.model == "" || + !strings.EqualFold(vote.backend, gateway) { + continue + } + counts[vote.model]++ + } + best := "" + for model, n := range counts { + if best == "" || n > counts[best] || (n == counts[best] && model < best) { + best = model + } + } + return best +} + +// resumeGatewayModelCrashLoops resumes each of names the watchdog paused for +// crash-looping. These are gateway-routed agents this apply just gave a model +// they had none of: launched with no model they could not start, which is +// what the crash-loop pause was for (#14678), and nothing else would resume +// them now that the cause is fixed. Pauses with any other trigger are left +// alone. Should the agent still fail, the watchdog pauses it again. +func (s *Server) resumeGatewayModelCrashLoops(names []string) { + if s.deps.AgentMgr == nil { + return + } + for _, name := range names { + proc, err := s.deps.AgentMgr.GetStatus(name) + if err != nil || proc == nil || !proc.Paused || proc.PausedTrigger != watchdog.CrashLoopTrigger { + continue + } + if err := s.deps.AgentMgr.Resume(s.deps.Ctx, name, "acmm-pack", "gateway model set for an agent paused for crash-looping without one"); err != nil { + s.logger.Warn("could not resume crash-looped agent after giving it the gateway's model", "agent", name, "error", err) + continue + } + s.logger.Info("resumed crash-looped agent after giving it the gateway's model", "agent", name) + } +} + // gatewayPlacement is the backend a pack-owned agent is moved to so it follows // the hive's gateway (hiveGatewayBackend), or "" to keep packBackend. Only a // pack whose own placement is a CLI is redirected: a pack that itself names a diff --git a/src/pkg/dashboard/pack_gateway_model_test.go b/src/pkg/dashboard/pack_gateway_model_test.go index e86946904c..0ce3f1118a 100644 --- a/src/pkg/dashboard/pack_gateway_model_test.go +++ b/src/pkg/dashboard/pack_gateway_model_test.go @@ -3,6 +3,7 @@ package dashboard import ( "testing" + "github.com/hivecommons/hive/crux/watchdog" "github.com/hivecommons/hive/pkg/config" ) @@ -488,3 +489,144 @@ func TestApplyPackFollowsGatewayBackendOverrides(t *testing.T) { }) } } + +// gatewayModelHive is the third #14678 reopen: scanner/quality/guide run on +// the LiteLLM gateway with a model the operator picked for each, and the +// gateway has no default_model. ownership selects how the operator placed +// them: "override" through the dashboard's live backend and model overrides +// (replayed from hive-state.json) over the pack's copilot config, "config" +// through operator-owned config fields. +func gatewayModelHive(t *testing.T, ownership, model string) *Server { + t.Helper() + if ownership == "config" { + srv := gatewayHive(t, "litellm") + for _, name := range []string{"scanner", "quality", "guide"} { + a := srv.deps.Config.Agents[name] + a.Model, a.ModelOwner = model, config.FieldOwnerOperator + srv.deps.Config.Agents[name] = a + } + return srv + } + srv := newFullServer(t) + for _, name := range []string{"scanner", "quality", "guide"} { + a := packCLIAgent(name) + srv.deps.Config.Agents[name] = a + srv.deps.AgentMgr.AddAgent(name, a) + if err := srv.deps.AgentMgr.SetBackendOverride(name, "litellm"); err != nil { + t.Fatalf("SetBackendOverride(%s): %v", name, err) + } + if err := srv.deps.AgentMgr.SetModelOverride(name, model); err != nil { + t.Fatalf("SetModelOverride(%s): %v", name, err) + } + } + return srv +} + +// TestApplyPackGivesGatewayAgentTheOperatorsModel covers the third #14678 +// reopen: the reviewer was moved onto LiteLLM with its model cleared for the +// gateway's default_model, but the gateway had none, so it launched with no +// model, crash-looped and was paused while every other agent ran on the model +// the operator picked. It must run on that model, whether the level creates +// it, moves it off copilot, or an earlier apply left it with no model. +func TestApplyPackGivesGatewayAgentTheOperatorsModel(t *testing.T) { + const model = "ibm/granite-4-h-small" + reviewers := []struct { + name string + reviewer *config.AgentConfig + }{ + {"reviewer created by the level", nil}, + {"pack-owned copilot reviewer moved", &config.AgentConfig{ + ID: "reviewer", Role: "reviewer", Enabled: true, + Backend: "copilot", BackendOwner: config.FieldOwnerPack, + Model: "claude-fable-5", ModelOwner: config.FieldOwnerPack, + }}, + {"gateway reviewer left with no model repaired", &config.AgentConfig{ + ID: "reviewer", Role: "reviewer", Enabled: true, + Backend: "litellm", BackendOwner: config.FieldOwnerPack, + ModelOwner: config.FieldOwnerPack, + }}, + } + for _, ownership := range []string{"override", "config"} { + for _, tt := range reviewers { + t.Run(ownership+"/"+tt.name, func(t *testing.T) { + srv := gatewayModelHive(t, ownership, model) + if tt.reviewer != nil { + srv.deps.Config.Agents["reviewer"] = *tt.reviewer + } + for i := 0; i < 2; i++ { + if _, err := srv.ApplyPack(2); err != nil { + t.Fatalf("ApplyPack(2) #%d: %v", i, err) + } + got := srv.deps.Config.Agents["reviewer"] + if got.Backend != "litellm" || got.Model != model { + t.Errorf("apply #%d: reviewer backend=%q model=%q, want litellm with the operator's %s", i, got.Backend, got.Model, model) + } + if got.ModelIsOperatorOwned() { + t.Errorf("apply #%d: reviewer model is operator-owned; the operator never picked it", i) + } + } + }) + } + } +} + +// TestApplyPackGatewayDefaultModelOutranksOperatorsModel: a gateway with a +// default_model keeps the #9945 behaviour, the pack agent names no model and +// the default applies. +func TestApplyPackGatewayDefaultModelOutranksOperatorsModel(t *testing.T) { + srv := gatewayHive(t, "cb") + srv.deps.Config.Governor.Gateways = []config.GatewayConfig{{ + Name: "cb", Kind: config.GatewayKindLiteLLM, Endpoint: "https://litellm.example.test", DefaultModel: "rits/zai-org/glm-5-3", + }} + for _, name := range []string{"scanner", "quality", "guide"} { + a := srv.deps.Config.Agents[name] + a.Model, a.ModelOwner = "ibm/granite-4-h-small", config.FieldOwnerOperator + srv.deps.Config.Agents[name] = a + } + if _, err := srv.ApplyPack(2); err != nil { + t.Fatalf("ApplyPack(2): %v", err) + } + if got := srv.deps.Config.Agents["reviewer"]; got.Backend != "cb" || got.Model != "" { + t.Errorf("reviewer backend=%q model=%q, want cb with no model so the gateway default_model applies", got.Backend, got.Model) + } +} + +// TestApplyPackResumesGatewayAgentCrashLoopedWithoutModel: the reviewer left +// with no model crash-looped and the watchdog paused it; once the apply gives +// it the gateway's model, the cause is gone and it is resumed. A pause with any +// other trigger is not the apply's to undo. +func TestApplyPackResumesGatewayAgentCrashLoopedWithoutModel(t *testing.T) { + for _, tt := range []struct { + trigger string + wantResume bool + }{ + {watchdog.CrashLoopTrigger, true}, + {"dashboard-api", false}, + } { + t.Run(tt.trigger, func(t *testing.T) { + srv := gatewayModelHive(t, "override", "ibm/granite-4-h-small") + reviewer := config.AgentConfig{ + ID: "reviewer", Role: "reviewer", Enabled: true, + Backend: "litellm", BackendOwner: config.FieldOwnerPack, + ModelOwner: config.FieldOwnerPack, + } + srv.deps.Config.Agents["reviewer"] = reviewer + srv.deps.AgentMgr.AddAgent("reviewer", reviewer) + if err := srv.deps.AgentMgr.Pause("reviewer", tt.trigger, "test pause"); err != nil { + t.Fatalf("pause: %v", err) + } + if _, err := srv.ApplyPack(2); err != nil { + t.Fatalf("ApplyPack(2): %v", err) + } + proc, err := srv.deps.AgentMgr.GetStatus("reviewer") + if err != nil { + t.Fatalf("GetStatus(reviewer): %v", err) + } + // Resume clears the pause marker even when the test manager cannot + // relaunch the agent (no tmux), so assert on the marker. + if resumed := proc.PausedTrigger == ""; resumed != tt.wantResume { + t.Errorf("reviewer paused by %q: resumed=%v, want %v", tt.trigger, resumed, tt.wantResume) + } + }) + } +}