From 2ae68eb0cf220c04a3d4cdf1b31a21e78b628519 Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Mon, 16 Nov 2026 22:33:00 -0500 Subject: [PATCH 1/2] =?UTF-8?q?=E2=9C=A8=20Betty=20runs=20several=20model?= =?UTF-8?q?=20calls=20per=20meeting=20at=20once?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A meeting may now have up to meetings.llm.per_meeting_in_flight (default 3, never above max_in_flight) model calls in flight, so a second request no longer waits for the first answer. A per-meeting Gate bounds the calls and reserves each in-flight call's expected cost against the token budget, so concurrent calls cannot all pass the check on the same ledger total. On a warm CLI backend the meeting keeps a pool of warm CLIs (one turn per process, since stream results carry no request id) that grows only when calls overlap. Part of #18590. Signed-off-by: Andy Anderson --- .../added-18590-betty-parallel-model-calls.md | 1 + src/cmd/hive/meetingswire.go | 25 ++- src/cmd/hive/meetingswire_cli_test.go | 4 +- src/docs/meetings.md | 3 +- src/pkg/config/meetings_config.go | 17 +- src/pkg/config/meetings_config_test.go | 7 +- src/pkg/meetings/betty/answer.go | 5 +- src/pkg/meetings/meetllm/anthropic.go | 2 +- src/pkg/meetings/meetllm/cli.go | 24 ++- src/pkg/meetings/meetllm/cli_test.go | 3 + src/pkg/meetings/meetllm/gate.go | 98 ++++++++++ src/pkg/meetings/meetllm/gate_test.go | 154 +++++++++++++++ src/pkg/meetings/meetllm/meetllm.go | 70 ++++++- src/pkg/meetings/meetllm/pool.go | 178 ++++++++++++++++++ src/pkg/meetings/meetllm/pool_test.go | 93 +++++++++ 15 files changed, 659 insertions(+), 25 deletions(-) create mode 100644 changelog.d/added-18590-betty-parallel-model-calls.md create mode 100644 src/pkg/meetings/meetllm/gate.go create mode 100644 src/pkg/meetings/meetllm/gate_test.go create mode 100644 src/pkg/meetings/meetllm/pool.go create mode 100644 src/pkg/meetings/meetllm/pool_test.go diff --git a/changelog.d/added-18590-betty-parallel-model-calls.md b/changelog.d/added-18590-betty-parallel-model-calls.md new file mode 100644 index 0000000000..d43280a005 --- /dev/null +++ b/changelog.d/added-18590-betty-parallel-model-calls.md @@ -0,0 +1 @@ +- **Betty works on several requests at once.** One meeting may now run up to `meetings.llm.per_meeting_in_flight` (default 3, never above `meetings.llm.max_in_flight`) model calls in parallel, so a second question no longer waits for the first answer. On the Claude CLI backend the meeting keeps a small pool of warm CLIs that grows only when requests overlap, and every call in flight reserves budget so parallel answers cannot overspend the meeting's token budget. ([#18590](https://github.com/hivecommons/hive/issues/18590)) diff --git a/src/cmd/hive/meetingswire.go b/src/cmd/hive/meetingswire.go index b14aaef683..62285b0b97 100644 --- a/src/cmd/hive/meetingswire.go +++ b/src/cmd/hive/meetingswire.go @@ -296,6 +296,9 @@ func meetingPersonaFactory(cfg *config.Config, persona config.ResolvedMeetingsPe Timeout: betty.DefaultAddressedTimeout, UsageFile: meetingUsageFile(store, mt.ID), MaxTokens: maxTokens, SlotsDir: slotsDir, MaxInFlight: llm.MaxInFlight, + // Several requests are answered at once, each call reserving + // budget against the others in flight (#18590). + Gate: meetllm.NewGate(llm.PerMeetingInFlight), } switch llm.Backend { case config.MeetingsLLMModeOpenAI: @@ -320,9 +323,10 @@ func meetingPersonaFactory(cfg *config.Config, persona config.ResolvedMeetingsPe // meetingCLIModel is the persona's model on an agent backend // (hivecommons/hive#18517, #18549), in the backend's no-tools completion -// profile with the agents' sign-in: one warm CLI for the meeting, started -// now so the first question finds it ready, on a backend with a warm mode; -// one CLI per call otherwise. It reports false when the backend has no +// profile with the agents' sign-in: a pool of warm CLIs for the meeting +// (one per concurrent answer, #18590), the first started now so the first +// question finds it ready, on a backend with a warm mode; one CLI per call +// otherwise. It reports false when the backend has no // profile. func meetingCLIModel(llm config.ResolvedMeetingsLLM, mc meetllm.Config, meetingID string, logger *slog.Logger) (betty.MeetLLM, bool) { mc.Provider, mc.Backend, mc.Model = meetllm.ProviderAgentBackend, llm.AgentBackend, llm.Model @@ -338,21 +342,30 @@ func meetingCLIModel(llm config.ResolvedMeetingsLLM, mc meetllm.Config, meetingI } return betty.MeetLLM{Config: mc}, true } - sess, err := meetllm.NewCLISession(mc) + pool, err := meetllm.NewCLIPool(mc, meetingCLIPoolSize(llm, mc)) if err != nil { if logger != nil { logger.Error("[meetings] persona model not started", "meeting", meetingID, "backend", llm.AgentBackend, "error", err) } return betty.MeetLLM{}, false } - if err := sess.Warm(); err != nil && logger != nil { + if err := pool.Warm(); err != nil && logger != nil { // The first call retries the start. logger.Warn("[meetings] persona CLI not warmed", "meeting", meetingID, "backend", llm.AgentBackend, "error", err) } - mc.Session = sess + mc.Pool = pool return betty.MeetLLM{Config: mc}, true } +// meetingCLIPoolSize is how many warm CLIs one meeting may run: its +// concurrent answers (the gate's size), else the resolved per-meeting cap. +func meetingCLIPoolSize(llm config.ResolvedMeetingsLLM, mc meetllm.Config) int { + if mc.Gate != nil { + return mc.Gate.Size() + } + return llm.PerMeetingInFlight +} + // applyMeetingCLITimeouts widens the persona's per-call timeouts to the CLI // mode's: a warm turn is slower than an API call. func applyMeetingCLITimeouts(s *betty.Settings, cli config.ResolvedMeetingsCLI) { diff --git a/src/cmd/hive/meetingswire_cli_test.go b/src/cmd/hive/meetingswire_cli_test.go index e909163ac6..5e17328c10 100644 --- a/src/cmd/hive/meetingswire_cli_test.go +++ b/src/cmd/hive/meetingswire_cli_test.go @@ -97,8 +97,8 @@ func TestMeetingPersonaFactoryAgentBackend(t *testing.T) { e.Close() } missing := cliLLM(filepath.Join(dir, "absent"), dir) - if m, ok := meetingCLIModel(missing, meetllm.Config{MaxInFlight: 1}, "m3", logger); !ok || m.Config.Session == nil { - t.Fatal("a missing binary still yields a session that retries the start") + if m, ok := meetingCLIModel(missing, meetllm.Config{MaxInFlight: 1}, "m3", logger); !ok || m.Config.Pool == nil { + t.Fatal("a missing binary still yields a pool that retries the start") } else { _ = m.Close() } diff --git a/src/docs/meetings.md b/src/docs/meetings.md index a9ae7b1bbd..233127571c 100644 --- a/src/docs/meetings.md +++ b/src/docs/meetings.md @@ -125,7 +125,7 @@ The CLI uses the sign-in the agents get, without hive logging it or writing it t A refused backend fails config validation and `PUT /api/meetings/model` with its reason, and the Settings picker lists it disabled, with the reason under "Why some backends are unavailable for meetings". A `meetings.llm.backend` that is neither a known backend nor a configured gateway is refused the same way. -The warm process is replaced after `HIVE_MEETINGS_CLI_RECYCLE_TURNS` turns (default 8, which bounds the history each turn re-reads and pays for), and at once after any error, timeout or crash, so a late reply can never answer the next request and the next call finds a fresh process. Calls from one meeting run one at a time inside the shared in-flight limit. On a CLI backend Betty's timeouts are wider than the API modes' 4 s, 2.5 s and 800 ms: `HIVE_MEETINGS_CLI_ADDRESSED_TIMEOUT_MS`, `HIVE_MEETINGS_CLI_UNPROMPTED_TIMEOUT_MS` and `HIVE_MEETINGS_CLI_RESTYLE_TIMEOUT_MS`, default 8000, 6000 and 3000 on `claude` and 20000, 15000 and 10000 on `copilot` and `omp`, which start the CLI on every call. Spek's per-prompt call uses `HIVE_MEETINGS_CLI_ONESHOT_TIMEOUT_MS` (30000). `HIVE_MEETINGS_CLI_BINARY` overrides the CLI binary. The panel shows the model in use, for example "claude CLI claude-sonnet-5-5" or "vllm model llama-4". +The warm process is replaced after `HIVE_MEETINGS_CLI_RECYCLE_TURNS` turns (default 8, which bounds the history each turn re-reads and pays for), and at once after any error, timeout or crash, so a late reply can never answer the next request and the next call finds a fresh process. Calls from one meeting run in parallel, up to `meetings.llm.per_meeting_in_flight` (default 3) inside the shared in-flight limit: a warm CLI answers one turn at a time on one stream (its results carry no request id), so the meeting keeps a pool of up to that many warm CLIs. Only the first starts with the meeting; when two requests overlap and no idle warm CLI is left, the next one is warmed in the background, so a burst of questions pays at most one cold start (a few seconds on `claude`) and the rest of the meeting answers warm. A meeting that never overlaps requests runs one CLI. On a CLI backend Betty's timeouts are wider than the API modes' 4 s, 2.5 s and 800 ms: `HIVE_MEETINGS_CLI_ADDRESSED_TIMEOUT_MS`, `HIVE_MEETINGS_CLI_UNPROMPTED_TIMEOUT_MS` and `HIVE_MEETINGS_CLI_RESTYLE_TIMEOUT_MS`, default 8000, 6000 and 3000 on `claude` and 20000, 15000 and 10000 on `copilot` and `omp`, which start the CLI on every call. Spek's per-prompt call uses `HIVE_MEETINGS_CLI_ONESHOT_TIMEOUT_MS` (30000). `HIVE_MEETINGS_CLI_BINARY` overrides the CLI binary. The panel shows the model in use, for example "claude CLI claude-sonnet-5-5" or "vllm model llama-4". Spektacular must be a version that includes the `meeting` command ([hivecommons/spektacular#126](https://github.com/hivecommons/spektacular/pull/126)). With an older binary the live mode exits at once; hive keeps the meeting going (transcript, consent, `hive leave`, research approvals, the panel) without Spek's questions and notes, and the panel says plainly that notes and clarifying questions are off for the meeting. While hive cannot reach a Spektacular child that is restarting, the panel says they are paused, and clears it when Spek writes again. @@ -323,6 +323,7 @@ Every key has an environment override that wins over `hive.yaml`. The full list | `meetings.llm.anthropic_model` | Spektacular's default | Anthropic model. | | `meetings.llm.timeout_ms` | `2500` | Per-call model timeout. | | `meetings.llm.max_in_flight` | `4` | Concurrent model calls across meetings. | +| `meetings.llm.per_meeting_in_flight` | `3` | Concurrent model calls in one meeting, so Betty answers several requests at once; never above `max_in_flight` (`HIVE_MEETINGS_LLM_PER_MEETING_IN_FLIGHT`). Each call in flight reserves its expected cost against the meeting's token budget, so parallel answers cannot overspend it. | | `meetings.email.invites_enabled` | `true` | Accept calendar invites sent to `hive@` (`HIVE_MEETINGS_EMAIL_INVITES_ENABLED`). | | `meetings.email.files_enabled` | `true` | Accept mail-in files as Inbox items (`HIVE_MEETINGS_EMAIL_FILES_ENABLED`). | | `meetings.persona.enabled` | `true` | The meeting attendee persona. | diff --git a/src/pkg/config/meetings_config.go b/src/pkg/config/meetings_config.go index 3fa746bb35..2de0614318 100644 --- a/src/pkg/config/meetings_config.go +++ b/src/pkg/config/meetings_config.go @@ -284,6 +284,10 @@ type MeetingsLLMConfig struct { // MaxInFlight caps concurrent model calls across meetings. Zero means // DefaultMeetingsLLMMaxInFlight. MaxInFlight int `yaml:"max_in_flight,omitempty" json:"max_in_flight,omitempty"` + // PerMeetingInFlight caps one meeting's concurrent model calls, so the + // persona answers several requests at once (#18590); it never exceeds + // MaxInFlight. Zero means DefaultMeetingsLLMPerMeetingInFlight. + PerMeetingInFlight int `yaml:"per_meeting_in_flight,omitempty" json:"per_meeting_in_flight,omitempty"` } // Meetings LLM modes and defaults. @@ -301,6 +305,9 @@ const ( DefaultMeetingsLLMTimeoutMS = 2500 // DefaultMeetingsLLMMaxInFlight lets a few meetings reason at once. DefaultMeetingsLLMMaxInFlight = 4 + // DefaultMeetingsLLMPerMeetingInFlight lets one meeting's persona work + // on a few requests at once. + DefaultMeetingsLLMPerMeetingInFlight = 3 // DefaultMeetingsLLMKeyFile and DefaultMeetingsAnthropicKeyFile are the // keystore-sealed secret files. DefaultMeetingsLLMKeyFile = WritableSecretsDir + "/meetings_llm_api_key" @@ -325,7 +332,9 @@ const ( MeetingsAnthropicModelEnv = "HIVE_MEETINGS_ANTHROPIC_MODEL" MeetingsLLMTimeoutEnv = "HIVE_MEETINGS_LLM_TIMEOUT_MS" MeetingsLLMMaxInFlightEnv = "HIVE_MEETINGS_LLM_MAX_IN_FLIGHT" - MeetingsLLMBackendEnv = "HIVE_MEETINGS_LLM_BACKEND" + // MeetingsLLMPerMeetingInFlightEnv overrides per_meeting_in_flight. + MeetingsLLMPerMeetingInFlightEnv = "HIVE_MEETINGS_LLM_PER_MEETING_IN_FLIGHT" + MeetingsLLMBackendEnv = "HIVE_MEETINGS_LLM_BACKEND" ) // ResolvedMeetingsLLM is the effective meetings model. Keys are secrets; @@ -343,6 +352,9 @@ type ResolvedMeetingsLLM struct { AnthropicModel string TimeoutMS int MaxInFlight int + // PerMeetingInFlight is one meeting's concurrent model calls, at most + // MaxInFlight. + PerMeetingInFlight int // Reason explains Backend none. Reason string // AgentBackend is the agent backend (claude, ...) on the agent-backend @@ -396,6 +408,8 @@ func (c *Config) ResolveMeetingsLLM(ctx context.Context) ResolvedMeetingsLLM { MaxInFlight: meetingsEnvInt(MeetingsLLMMaxInFlightEnv, l.MaxInFlight, DefaultMeetingsLLMMaxInFlight), AnthropicModel: meetingsEnvString(MeetingsAnthropicModelEnv, l.AnthropicModel, ""), } + r.PerMeetingInFlight = min(r.MaxInFlight, + meetingsEnvInt(MeetingsLLMPerMeetingInFlightEnv, l.PerMeetingInFlight, DefaultMeetingsLLMPerMeetingInFlight)) if mode == MeetingsLLMModeAgentBackend || mode == MeetingsLLMModeClaudeCLI { return c.resolveMeetingsAgentBackend(ctx, mode, r) } @@ -750,6 +764,7 @@ func (m MeetingsConfig) Validate() error { {"meetings.budget.max_intents_per_meeting", m.Budget.MaxIntentsPerMeeting}, {"meetings.llm.timeout_ms", m.LLM.TimeoutMS}, {"meetings.llm.max_in_flight", m.LLM.MaxInFlight}, + {"meetings.llm.per_meeting_in_flight", m.LLM.PerMeetingInFlight}, } { if f.v < 0 { return fmt.Errorf("%s must not be negative (got %d)", f.key, f.v) diff --git a/src/pkg/config/meetings_config_test.go b/src/pkg/config/meetings_config_test.go index 9541835c53..4096ae436f 100644 --- a/src/pkg/config/meetings_config_test.go +++ b/src/pkg/config/meetings_config_test.go @@ -352,7 +352,8 @@ func TestResolveMeetingsLLM(t *testing.T) { cfg.Governor.LiteLLM.DefaultModel = "gpt-x" r := cfg.ResolveMeetingsLLM(ctx) if r.Backend != MeetingsLLMModeOpenAI || r.Endpoint != "https://litellm.example" || r.Model != "gpt-x" || - r.TimeoutMS != DefaultMeetingsLLMTimeoutMS || r.MaxInFlight != DefaultMeetingsLLMMaxInFlight || r.Label() != "model gpt-x" { + r.TimeoutMS != DefaultMeetingsLLMTimeoutMS || r.MaxInFlight != DefaultMeetingsLLMMaxInFlight || r.Label() != "model gpt-x" || + r.PerMeetingInFlight != DefaultMeetingsLLMPerMeetingInFlight { t.Fatalf("litellm fallback %+v", r) } // Meetings' own endpoint, model and key win. @@ -366,6 +367,10 @@ func TestResolveMeetingsLLM(t *testing.T) { if r.Endpoint != "https://llm.example" || r.Model != "m1" || r.APIKey != "mkey" || r.TimeoutMS != 900 || r.MaxInFlight != 2 { t.Fatalf("meetings llm %+v", r) } + // One meeting never runs more calls than the hive-wide cap. + if r.PerMeetingInFlight != 2 { + t.Fatalf("per-meeting in flight %d above max_in_flight 2", r.PerMeetingInFlight) + } // An Anthropic key switches to anthropic mode. aFile := filepath.Join(dir, "a") if err := WriteMeetingsSecret(ctx, aFile, "akey"); err != nil { diff --git a/src/pkg/meetings/betty/answer.go b/src/pkg/meetings/betty/answer.go index ad70fe8a92..d56673ee04 100644 --- a/src/pkg/meetings/betty/answer.go +++ b/src/pkg/meetings/betty/answer.go @@ -55,8 +55,11 @@ func (m MeetLLM) Complete(ctx context.Context, system, prompt string) (string, e return meetllm.Complete(ctx, c, hc, prompt) } -// Close stops the meeting's warm CLI, if it has one. +// Close stops the meeting's warm CLIs, if it has any. func (m MeetLLM) Close() error { + if m.Config.Pool != nil { + _ = m.Config.Pool.Close() + } if m.Config.Session != nil { return m.Config.Session.Close() } diff --git a/src/pkg/meetings/meetllm/anthropic.go b/src/pkg/meetings/meetllm/anthropic.go index 0b52f4b4f4..3113363687 100644 --- a/src/pkg/meetings/meetllm/anthropic.go +++ b/src/pkg/meetings/meetllm/anthropic.go @@ -100,7 +100,7 @@ func completeAnthropic(ctx context.Context, c Config, hc *http.Client, prompt st if err := json.NewDecoder(io.LimitReader(resp.Body, maxResponseBytes)).Decode(&mr); err != nil { return "", fmt.Errorf("meeting-llm: decoding response: %w", err) } - if err := recordUsage(c.UsageFile, mr.Usage.InputTokens+mr.Usage.OutputTokens, time.Now()); err != nil { + if err := c.charge(mr.Usage.InputTokens+mr.Usage.OutputTokens, time.Now()); err != nil { return "", fmt.Errorf("meeting-llm: recording usage: %w", err) } if mr.StopReason == stopRefusal { diff --git a/src/pkg/meetings/meetllm/cli.go b/src/pkg/meetings/meetllm/cli.go index fbc5edfb8d..b5c959bc3c 100644 --- a/src/pkg/meetings/meetllm/cli.go +++ b/src/pkg/meetings/meetllm/cli.go @@ -532,6 +532,26 @@ func (s *CLISession) lock(ctx context.Context) error { func (s *CLISession) unlock() { <-s.sem } +// tryLock takes the turn lock if it is free. +func (s *CLISession) tryLock() bool { + select { + case s.sem <- struct{}{}: + return true + default: + return false + } +} + +// idle reports that no turn holds the session. +func (s *CLISession) idle() bool { return len(s.sem) == 0 } + +// running reports that the session has a live process. +func (s *CLISession) running() bool { + s.mu.Lock() + defer s.mu.Unlock() + return s.proc != nil +} + func (s *CLISession) turnLocked(ctx context.Context, system, prompt string) (string, int, error) { s.mu.Lock() p, err := s.ensureLocked() @@ -597,7 +617,7 @@ func completeSession(ctx context.Context, c Config, prompt string) (string, erro } defer release() reply, tokens, err := c.Session.turnLocked(ctx, c.System, prompt) - if uerr := recordUsage(c.UsageFile, tokens, time.Now()); uerr != nil && err == nil { + if uerr := c.charge(tokens, time.Now()); uerr != nil && err == nil { err = fmt.Errorf("meeting-llm: recording usage: %w", uerr) } return reply, err @@ -655,7 +675,7 @@ func completeCLIOnce(ctx context.Context, c Config, prompt string) (string, erro } return "", perr } - if err := recordUsage(c.UsageFile, out.tokens, time.Now()); err != nil { + if err := c.charge(out.tokens, time.Now()); err != nil { return "", fmt.Errorf("meeting-llm: recording usage: %w", err) } if out.err != nil { diff --git a/src/pkg/meetings/meetllm/cli_test.go b/src/pkg/meetings/meetllm/cli_test.go index b205a87b75..a179aa2cd8 100644 --- a/src/pkg/meetings/meetllm/cli_test.go +++ b/src/pkg/meetings/meetllm/cli_test.go @@ -23,6 +23,8 @@ const ( fakeCrash = "crash" fakeCrashOnce = "crashonce" fakeError = "error" + // fakeSlow answers after half a second, so turns overlap. + fakeSlow = "slow" ) // fakeTokensPerTurn is what the fake reports per turn: 10+5+3+2. @@ -46,6 +48,7 @@ while IFS= read -r line; do echo "$line" >> "$D/turns" case "` + behaviour + `" in hang) sleep 30 ;; + slow) sleep 0.5 ;; crash) exit 1 ;; crashonce) if [ ! -f "$D/crashed" ]; then : > "$D/crashed"; exit 1; fi ;; error) echo '{"type":"result","subtype":"error_during_execution","is_error":true,"result":"","usage":{"input_tokens":10,"output_tokens":5,"cache_creation_input_tokens":3,"cache_read_input_tokens":2}}'; continue ;; diff --git a/src/pkg/meetings/meetllm/gate.go b/src/pkg/meetings/meetllm/gate.go new file mode 100644 index 0000000000..d783ba8a15 --- /dev/null +++ b/src/pkg/meetings/meetllm/gate.go @@ -0,0 +1,98 @@ +package meetllm + +import ( + "context" + "fmt" + "sync" +) + +// Per-meeting parallelism (hivecommons/hive#18590): the meeting persona +// answers several requests at once, so one meeting may have more than one +// model call in flight. Gate bounds how many and keeps the meeting's token +// budget honest while they run. +const ( + // DefaultPerMeetingInFlight is how many model calls one meeting may + // have in flight when its gate is built with zero. + DefaultPerMeetingInFlight = 3 + // defaultCallEstimate is the tokens a call is assumed to cost before + // the meeting has seen one: a persona prompt with retrieval snippets + // and a short JSON answer. + defaultCallEstimate = 2000 +) + +// Gate is one meeting's in-process admission for model calls: at most N in +// flight, and a token reservation per call so concurrent calls cannot all +// pass the budget check on the same ledger total and overspend it. The +// ledger stays the single record of spend; the reservation only covers the +// calls whose usage is not written yet. +type Gate struct { + sem chan struct{} + + mu sync.Mutex + reserved int + // estimate is the largest call cost seen so far (defaultCallEstimate + // until a call reports usage). + estimate int +} + +// NewGate builds a gate admitting n concurrent calls; n <= 0 means +// DefaultPerMeetingInFlight. +func NewGate(n int) *Gate { + if n <= 0 { + n = DefaultPerMeetingInFlight + } + return &Gate{sem: make(chan struct{}, n), estimate: defaultCallEstimate} +} + +// Size is how many calls the gate admits at once. +func (g *Gate) Size() int { return cap(g.sem) } + +// InFlight is how many calls hold the gate now. +func (g *Gate) InFlight() int { return len(g.sem) } + +// enter waits for a free place until ctx ends. +func (g *Gate) enter(ctx context.Context) error { + select { + case g.sem <- struct{}{}: + return nil + case <-ctx.Done(): + return fmt.Errorf("meeting-llm: meeting's model calls all busy: %w", ctx.Err()) + } +} + +func (g *Gate) leave() { <-g.sem } + +// reserve admits one call against the budget: it refuses (ErrBudget) when +// the ledger total plus the calls still in flight already reach maxTokens, +// and otherwise holds this call's estimate until settle. +func (g *Gate) reserve(used, maxTokens int) (int, error) { + g.mu.Lock() + defer g.mu.Unlock() + if maxTokens > 0 && used+g.reserved >= maxTokens { + return 0, ErrBudget + } + est := g.estimate + g.reserved += est + return est, nil +} + +// settle releases a call's reservation once its usage is in the ledger and +// learns its cost for the next estimate. +func (g *Gate) settle(held, spent int) { + g.mu.Lock() + defer g.mu.Unlock() + g.reserved -= held + if g.reserved < 0 { + g.reserved = 0 + } + if spent > g.estimate { + g.estimate = spent + } +} + +// Reserved is the tokens held for calls in flight. +func (g *Gate) Reserved() int { + g.mu.Lock() + defer g.mu.Unlock() + return g.reserved +} diff --git a/src/pkg/meetings/meetllm/gate_test.go b/src/pkg/meetings/meetllm/gate_test.go new file mode 100644 index 0000000000..bdb1e58aae --- /dev/null +++ b/src/pkg/meetings/meetllm/gate_test.go @@ -0,0 +1,154 @@ +package meetllm + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "path/filepath" + "sync" + "sync/atomic" + "testing" + "time" +) + +// gateServer is a chat endpoint that holds every request until release is +// closed, counting the most requests it held at once. +func gateServer(t *testing.T, tokens string) (*httptest.Server, chan struct{}, *atomic.Int32, *atomic.Int32) { + t.Helper() + release := make(chan struct{}) + var now, peak atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + n := now.Add(1) + for { + p := peak.Load() + if n <= p || peak.CompareAndSwap(p, n) { + break + } + } + <-release + now.Add(-1) + _, _ = w.Write([]byte(`{"choices":[{"message":{"role":"assistant","content":"ok"}}],"usage":{"total_tokens":` + tokens + `}}`)) + })) + t.Cleanup(srv.Close) + t.Cleanup(func() { + select { + case <-release: + default: + close(release) + } + }) + return srv, release, &now, &peak +} + +func gateConfig(srv *httptest.Server, dir string, g *Gate) Config { + return Config{ + Endpoint: srv.URL, Model: "m", Timeout: 5 * time.Second, + UsageFile: filepath.Join(dir, "usage.jsonl"), MaxInFlight: 8, Gate: g, + } +} + +// A meeting's gate lets up to its size of calls run at once, and no more. +func TestCompleteGateRunsCallsInParallel(t *testing.T) { + for _, size := range []int{1, 3} { + srv, release, now, peak := gateServer(t, "10") + c := gateConfig(srv, t.TempDir(), NewGate(size)) + const calls = 4 + var wg sync.WaitGroup + errs := make(chan error, calls) + for range calls { + wg.Add(1) + go func() { + defer wg.Done() + _, err := Complete(context.Background(), c, srv.Client(), "q") + errs <- err + }() + } + waitFor(t, "calls in flight", func() bool { return int(now.Load()) == size }) + // Give a call past the gate's size time to show up if it could. + time.Sleep(50 * time.Millisecond) + if got := int(peak.Load()); got != size { + t.Fatalf("gate %d: %d calls in flight at once", size, got) + } + close(release) + wg.Wait() + close(errs) + for err := range errs { + if err != nil { + t.Fatalf("gate %d: %v", size, err) + } + } + if n, _ := UsageTotal(c.UsageFile); n != calls*10 { + t.Fatalf("gate %d: usage %d, want %d", size, n, calls*10) + } + } +} + +// Concurrent calls cannot all pass the budget check on the same ledger +// total: a call in flight holds a reservation, so the next one is refused +// before it reaches the model. +func TestCompleteGateReservesBudget(t *testing.T) { + srv, release, now, _ := gateServer(t, "100") + c := gateConfig(srv, t.TempDir(), NewGate(3)) + c.MaxTokens = 150 + first := make(chan error, 1) + go func() { + _, err := Complete(context.Background(), c, srv.Client(), "q") + first <- err + }() + waitFor(t, "the first call in flight", func() bool { return now.Load() == 1 }) + if _, err := Complete(context.Background(), c, srv.Client(), "q2"); !errors.Is(err, ErrBudget) { + t.Fatalf("second call while the first holds the budget: err = %v", err) + } + close(release) + if err := <-first; err != nil { + t.Fatal(err) + } + if r := c.Gate.Reserved(); r != 0 { + t.Fatalf("reservation left after the call: %d", r) + } + // Sequential calls keep the old rule: refused once the ledger is spent. + if _, err := Complete(context.Background(), c, srv.Client(), "q3"); err != nil { + t.Fatalf("a call under budget: %v", err) + } + if _, err := Complete(context.Background(), c, srv.Client(), "q4"); !errors.Is(err, ErrBudget) { + t.Fatalf("over budget err = %v", err) + } +} + +func TestGateReserveAndSettle(t *testing.T) { + g := NewGate(0) + if g.Size() != DefaultPerMeetingInFlight { + t.Fatalf("size %d", g.Size()) + } + held, err := g.reserve(0, 0) + if err != nil || held != defaultCallEstimate { + t.Fatalf("reserve without a budget = %d %v", held, err) + } + g.settle(held, defaultCallEstimate+500) + if g.Reserved() != 0 || g.estimate != defaultCallEstimate+500 { + t.Fatalf("after settle reserved %d estimate %d", g.Reserved(), g.estimate) + } + a, _ := g.reserve(0, 10_000) + if _, err := g.reserve(10_000-a, 10_000); !errors.Is(err, ErrBudget) { + t.Fatalf("reservation not counted: %v", err) + } + g.settle(a, 0) + g.settle(a, 0) + if g.Reserved() != 0 { + t.Fatalf("double settle went negative: %d", g.Reserved()) + } +} + +// A call waiting for the meeting's place gives up at its deadline. +func TestCompleteGateBusy(t *testing.T) { + srv, _, now, _ := gateServer(t, "1") + c := gateConfig(srv, t.TempDir(), NewGate(1)) + held := c + go func() { _, _ = Complete(context.Background(), held, srv.Client(), "hold") }() + waitFor(t, "the held call", func() bool { return now.Load() == 1 }) + c.Timeout = 50 * time.Millisecond + if _, err := Complete(context.Background(), c, srv.Client(), "q"); !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("busy err = %v", err) + } +} diff --git a/src/pkg/meetings/meetllm/meetllm.go b/src/pkg/meetings/meetllm/meetllm.go index 15f4b729a1..f84d16740e 100644 --- a/src/pkg/meetings/meetllm/meetllm.go +++ b/src/pkg/meetings/meetllm/meetllm.go @@ -119,8 +119,16 @@ type Config struct { // replaced; zero means DefaultRecycleTurns. RecycleTurns int // Session is the meeting's warm CLI. Nil on ProviderAgentBackend runs - // one CLI per call. + // one CLI per call. Pool, when set, takes its place. Session *CLISession + // Pool is the meeting's set of warm CLIs, so several turns run at once + // (hivecommons/hive#18590). + Pool *CLIPool + // Gate bounds the meeting's concurrent calls and reserves budget for + // the ones in flight; nil admits every call (the one-shot command). + Gate *Gate + // spent receives the tokens a call charged, for the gate. + spent *int // Getenv reads the environment the CLI inherits from; nil is os.Getenv. Getenv func(string) string // CABundle is an optional PEM file of private CAs the endpoint is @@ -193,6 +201,15 @@ func UsageTotal(path string) (int, error) { return total, sc.Err() } +// charge records a call's tokens in the ledger and reports them to the +// meeting's gate. +func (c Config) charge(tokens int, now time.Time) error { + if c.spent != nil && tokens > 0 { + *c.spent += tokens + } + return recordUsage(c.UsageFile, tokens, now) +} + func recordUsage(path string, tokens int, now time.Time) error { if path == "" || tokens <= 0 { return nil @@ -273,19 +290,52 @@ func chatURL(endpoint string) string { } // Complete makes one call and records its usage. It refuses once the -// meeting's budget is spent. +// meeting's budget is spent. With a Gate, the call first waits for one of +// the meeting's places and reserves budget against the calls still in +// flight, so concurrent answers cannot overspend it. func Complete(ctx context.Context, c Config, hc *http.Client, prompt string) (string, error) { - if c.MaxTokens > 0 && c.UsageFile != "" { - used, err := UsageTotal(c.UsageFile) - if err != nil { + used, err := c.used() + if err != nil { + return "", err + } + if c.MaxTokens > 0 && used >= c.MaxTokens { + return "", ErrBudget + } + ctx, cancel := context.WithTimeout(ctx, c.Timeout) + defer cancel() + if c.Gate != nil { + if err := c.Gate.enter(ctx); err != nil { return "", err } - if used >= c.MaxTokens { - return "", ErrBudget + defer c.Gate.leave() + // Calls that finished while this one waited are in the ledger now. + if used, err = c.used(); err != nil { + return "", err } + held, err := c.Gate.reserve(used, c.MaxTokens) + if err != nil { + return "", err + } + var spent int + c.spent = &spent + defer func() { c.Gate.settle(held, spent) }() + } + return complete(ctx, c, hc, prompt) +} + +// used is the meeting's ledger total when it has a budget. +func (c Config) used() (int, error) { + if c.MaxTokens <= 0 || c.UsageFile == "" { + return 0, nil + } + return UsageTotal(c.UsageFile) +} + +// complete is one call once admitted. +func complete(ctx context.Context, c Config, hc *http.Client, prompt string) (string, error) { + if c.Provider == ProviderAgentBackend && c.Pool != nil { + return completePool(ctx, c, prompt) } - ctx, cancel := context.WithTimeout(ctx, c.Timeout) - defer cancel() if c.Provider == ProviderAgentBackend && c.Session != nil { return completeSession(ctx, c, prompt) } @@ -330,7 +380,7 @@ func Complete(ctx context.Context, c Config, hc *http.Client, prompt string) (st if tokens == 0 { tokens = cr.Usage.PromptTokens + cr.Usage.CompletionTokens } - if err := recordUsage(c.UsageFile, tokens, time.Now()); err != nil { + if err := c.charge(tokens, time.Now()); err != nil { return "", fmt.Errorf("meeting-llm: recording usage: %w", err) } if len(cr.Choices) == 0 { diff --git a/src/pkg/meetings/meetllm/pool.go b/src/pkg/meetings/meetllm/pool.go new file mode 100644 index 0000000000..2c1250cf86 --- /dev/null +++ b/src/pkg/meetings/meetllm/pool.go @@ -0,0 +1,178 @@ +package meetllm + +import ( + "context" + "fmt" + "sync" + "time" +) + +// CLIPool is one meeting's set of warm completion CLIs, so the persona can +// run several turns at once (hivecommons/hive#18590). A warm CLI answers +// one turn at a time on one stdin/stdout stream (its result events carry no +// request id to tell turns apart), so concurrency comes from more +// processes, never from interleaving turns on one. +// +// The first CLI is warmed when the meeting starts. The others start only +// when the meeting actually overlaps requests: a call that takes the last +// idle warm CLI while another call is in flight warms the next one in the +// background, so a burst of questions pays at most one cold start and the +// rest of the meeting stays warm. Memory is bounded by Size processes. +type CLIPool struct { + cfg Config + // sem holds one place per call in the pool; a holder always finds an + // idle session (or room to start one). + sem chan struct{} + + mu sync.Mutex + sessions []*CLISession + closed bool +} + +// NewCLIPool builds a pool of up to n warm CLIs for c.Backend; n <= 0 +// means one. Warm starts the first. +func NewCLIPool(c Config, n int) (*CLIPool, error) { + if n <= 0 { + n = 1 + } + first, err := NewCLISession(c) + if err != nil { + return nil, err + } + return &CLIPool{cfg: c, sem: make(chan struct{}, n), sessions: []*CLISession{first}}, nil +} + +// Size is the most CLIs the pool runs. +func (p *CLIPool) Size() int { return cap(p.sem) } + +// Warm starts the first CLI if it is not running. +func (p *CLIPool) Warm() error { + p.mu.Lock() + s := p.sessions[0] + p.mu.Unlock() + return s.Warm() +} + +// Started is how many of the pool's CLIs have a process now. +func (p *CLIPool) Started() int { + p.mu.Lock() + defer p.mu.Unlock() + n := 0 + for _, s := range p.sessions { + if s.running() { + n++ + } + } + return n +} + +// Close stops every CLI; later turns fail with ErrSessionClosed. +func (p *CLIPool) Close() error { + p.mu.Lock() + p.closed = true + sessions := append([]*CLISession(nil), p.sessions...) + p.mu.Unlock() + for _, s := range sessions { + _ = s.Close() + } + return nil +} + +// take waits for a place, then returns an idle session holding its turn +// lock: a warm one when there is one, else an idle cold one, else a new +// session. Release it with give. +func (p *CLIPool) take(ctx context.Context) (*CLISession, error) { + select { + case p.sem <- struct{}{}: + case <-ctx.Done(): + return nil, fmt.Errorf("meeting-llm: CLI pool busy: %w", ctx.Err()) + } + p.mu.Lock() + defer p.mu.Unlock() + if p.closed { + <-p.sem + return nil, ErrSessionClosed + } + s := p.idleLocked(true) + if s == nil { + s = p.idleLocked(false) + } + if s == nil { + // The place guarantees room: every existing session is held. + var err error + if s, err = p.addLocked(); err != nil { + <-p.sem + return nil, err + } + s.tryLock() + } + p.warmAheadLocked() + return s, nil +} + +// give returns a session taken with take. +func (p *CLIPool) give(s *CLISession) { + s.unlock() + <-p.sem +} + +// idleLocked locks and returns an idle session, warm ones only when warm. +func (p *CLIPool) idleLocked(warm bool) *CLISession { + for _, s := range p.sessions { + if warm && !s.running() { + continue + } + if s.tryLock() { + return s + } + } + return nil +} + +func (p *CLIPool) addLocked() (*CLISession, error) { + s, err := NewCLISession(p.cfg) + if err != nil { + return nil, err + } + p.sessions = append(p.sessions, s) + return s, nil +} + +// warmAheadLocked starts one more CLI in the background when calls overlap +// and no idle warm CLI is left for the next one. +func (p *CLIPool) warmAheadLocked() { + if len(p.sem) < 2 || len(p.sessions) >= cap(p.sem) { + return + } + for _, s := range p.sessions { + if s.running() && s.idle() { + return + } + } + s, err := p.addLocked() + if err != nil { + return + } + go func() { _ = s.Warm() }() +} + +// completePool runs one call on an idle CLI of the meeting's pool under the +// hive-wide in-flight limit and records its usage. The pool place is taken +// before the slot, so a call queued behind its own meeting holds no slot. +func completePool(ctx context.Context, c Config, prompt string) (string, error) { + s, err := c.Pool.take(ctx) + if err != nil { + return "", err + } + defer c.Pool.give(s) + release, err := acquire(ctx, c.SlotsDir, c.MaxInFlight) + if err != nil { + return "", err + } + defer release() + reply, tokens, err := s.turnLocked(ctx, c.System, prompt) + if uerr := c.charge(tokens, time.Now()); uerr != nil && err == nil { + err = fmt.Errorf("meeting-llm: recording usage: %w", uerr) + } + return reply, err +} diff --git a/src/pkg/meetings/meetllm/pool_test.go b/src/pkg/meetings/meetllm/pool_test.go new file mode 100644 index 0000000000..0ba6f2993a --- /dev/null +++ b/src/pkg/meetings/meetllm/pool_test.go @@ -0,0 +1,93 @@ +package meetllm + +import ( + "context" + "errors" + "net/http" + "path/filepath" + "sync" + "testing" +) + +func newPool(t *testing.T, c Config, n int) *CLIPool { + t.Helper() + p, err := NewCLIPool(c, n) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = p.Close() }) + return p +} + +// Overlapping turns run on separate warm processes, never interleaved on +// one, and the pool grows no further than its size. +func TestCLIPoolParallelTurns(t *testing.T) { + dir := t.TempDir() + c := cliConfig(t, dir, fakeSlow) + c.UsageFile = filepath.Join(dir, "usage.jsonl") + c.MaxInFlight = 8 + c.Gate = NewGate(2) + c.Pool = newPool(t, c, 2) + if err := c.Pool.Warm(); err != nil { + t.Fatal(err) + } + const calls = 4 + var wg sync.WaitGroup + var mu sync.Mutex + pids := map[int]bool{} + for range calls { + wg.Add(1) + go func() { + defer wg.Done() + reply, err := Complete(context.Background(), c, http.DefaultClient, "q") + if err != nil { + t.Error(err) + return + } + mu.Lock() + pids[replyPID(t, reply)] = true + mu.Unlock() + }() + } + wg.Wait() + if len(pids) != 2 { + t.Fatalf("turns ran on %d processes, want 2: %v", len(pids), pids) + } + if n := c.Pool.Started(); n != 2 { + t.Fatalf("pool runs %d CLIs, want 2", n) + } + if n, _ := UsageTotal(c.UsageFile); n != calls*fakeTokensPerTurn { + t.Fatalf("usage %d, want %d", n, calls*fakeTokensPerTurn) + } +} + +// One request at a time keeps a single warm CLI: the pool only grows when +// calls overlap. +func TestCLIPoolSequentialStaysSingle(t *testing.T) { + dir := t.TempDir() + c := cliConfig(t, dir, fakeOK) + c.Pool = newPool(t, c, 3) + for range 3 { + if _, err := Complete(context.Background(), c, http.DefaultClient, "q"); err != nil { + t.Fatal(err) + } + } + if n := len(readLines(t, filepath.Join(dir, "starts"))); n != 1 { + t.Fatalf("sequential turns started %d CLIs", n) + } +} + +// A closed pool refuses turns. +func TestCLIPoolClosed(t *testing.T) { + c := cliConfig(t, t.TempDir(), fakeOK) + c.Pool = newPool(t, c, 2) + if err := c.Pool.Close(); err != nil { + t.Fatal(err) + } + if _, err := Complete(context.Background(), c, http.DefaultClient, "q"); !errors.Is(err, ErrSessionClosed) { + t.Fatalf("closed pool err = %v", err) + } + if _, err := NewCLIPool(Config{Backend: "codex"}, 2); err == nil { + t.Fatal("want an error for a backend with no warm mode") + } +} From 3887b44cd4cccd48f34847e040c76c5012536f6a Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Mon, 16 Nov 2026 22:51:27 -0500 Subject: [PATCH 2/2] =?UTF-8?q?=F0=9F=8C=B1=20Show=20meetings.llm.per=5Fme?= =?UTF-8?q?eting=5Fin=5Fflight=20in=20the=20meetings=20settings?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Settings parity guard needs every hive.yaml leaf referenced by the Settings UI; the meetings settings' effective values now include it. Part of #18590. Signed-off-by: Andy Anderson --- src/pkg/dashboard/api_meetings.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/pkg/dashboard/api_meetings.go b/src/pkg/dashboard/api_meetings.go index db3d049ad1..4d377028aa 100644 --- a/src/pkg/dashboard/api_meetings.go +++ b/src/pkg/dashboard/api_meetings.go @@ -470,6 +470,7 @@ func (s *Server) meetingsSettings(ctx context.Context, m config.MeetingsConfig) "meetings.llm.anthropic_model": m.LLM.AnthropicModel, "meetings.llm.timeout_ms": strconv.Itoa(m.LLM.TimeoutMS), "meetings.llm.max_in_flight": strconv.Itoa(m.LLM.MaxInFlight), + "meetings.llm.per_meeting_in_flight": strconv.Itoa(meetingsPerMeetingInFlight(m.LLM)), "meetings.persona.enabled": boolString(persona.Enabled), "meetings.persona.preset": persona.Preset, "meetings.persona.name": persona.Name, @@ -594,3 +595,16 @@ func (s *Server) handleMeetingsSettingsPost(w http.ResponseWriter, r *http.Reque out.RestartRequired = changed jsonResponse(w, out) } + +// meetingsPerMeetingInFlight is meetings.llm.per_meeting_in_flight as it +// applies: the default when unset, never above max_in_flight. +func meetingsPerMeetingInFlight(l config.MeetingsLLMConfig) int { + n, ceiling := l.PerMeetingInFlight, l.MaxInFlight + if n <= 0 { + n = config.DefaultMeetingsLLMPerMeetingInFlight + } + if ceiling <= 0 { + ceiling = config.DefaultMeetingsLLMMaxInFlight + } + return min(n, ceiling) +}