Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/added-18590-betty-parallel-model-calls.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- **Betty works on several requests at once.** One meeting may now run up to `meetings.llm.per_meeting_in_flight` (default 3, never above `meetings.llm.max_in_flight`) model calls in parallel, so a second question no longer waits for the first answer. On the Claude CLI backend the meeting keeps a small pool of warm CLIs that grows only when requests overlap, and every call in flight reserves budget so parallel answers cannot overspend the meeting's token budget. ([#18590](https://github.com/hivecommons/hive/issues/18590))
25 changes: 19 additions & 6 deletions src/cmd/hive/meetingswire.go
Original file line number Diff line number Diff line change
Expand Up @@ -296,6 +296,9 @@ func meetingPersonaFactory(cfg *config.Config, persona config.ResolvedMeetingsPe
Timeout: betty.DefaultAddressedTimeout,
UsageFile: meetingUsageFile(store, mt.ID), MaxTokens: maxTokens,
SlotsDir: slotsDir, MaxInFlight: llm.MaxInFlight,
// Several requests are answered at once, each call reserving
// budget against the others in flight (#18590).
Gate: meetllm.NewGate(llm.PerMeetingInFlight),
}
switch llm.Backend {
case config.MeetingsLLMModeOpenAI:
Expand All @@ -320,9 +323,10 @@ func meetingPersonaFactory(cfg *config.Config, persona config.ResolvedMeetingsPe

// meetingCLIModel is the persona's model on an agent backend
// (hivecommons/hive#18517, #18549), in the backend's no-tools completion
// profile with the agents' sign-in: one warm CLI for the meeting, started
// now so the first question finds it ready, on a backend with a warm mode;
// one CLI per call otherwise. It reports false when the backend has no
// profile with the agents' sign-in: a pool of warm CLIs for the meeting
// (one per concurrent answer, #18590), the first started now so the first
// question finds it ready, on a backend with a warm mode; one CLI per call
// otherwise. It reports false when the backend has no
// profile.
func meetingCLIModel(llm config.ResolvedMeetingsLLM, mc meetllm.Config, meetingID string, logger *slog.Logger) (betty.MeetLLM, bool) {
mc.Provider, mc.Backend, mc.Model = meetllm.ProviderAgentBackend, llm.AgentBackend, llm.Model
Expand All @@ -338,21 +342,30 @@ func meetingCLIModel(llm config.ResolvedMeetingsLLM, mc meetllm.Config, meetingI
}
return betty.MeetLLM{Config: mc}, true
}
sess, err := meetllm.NewCLISession(mc)
pool, err := meetllm.NewCLIPool(mc, meetingCLIPoolSize(llm, mc))
if err != nil {
if logger != nil {
logger.Error("[meetings] persona model not started", "meeting", meetingID, "backend", llm.AgentBackend, "error", err)
}
return betty.MeetLLM{}, false
}
if err := sess.Warm(); err != nil && logger != nil {
if err := pool.Warm(); err != nil && logger != nil {
// The first call retries the start.
logger.Warn("[meetings] persona CLI not warmed", "meeting", meetingID, "backend", llm.AgentBackend, "error", err)
}
mc.Session = sess
mc.Pool = pool
return betty.MeetLLM{Config: mc}, true
}

// meetingCLIPoolSize is how many warm CLIs one meeting may run: its
// concurrent answers (the gate's size), else the resolved per-meeting cap.
func meetingCLIPoolSize(llm config.ResolvedMeetingsLLM, mc meetllm.Config) int {
if mc.Gate != nil {
return mc.Gate.Size()
}
return llm.PerMeetingInFlight
}

// applyMeetingCLITimeouts widens the persona's per-call timeouts to the CLI
// mode's: a warm turn is slower than an API call.
func applyMeetingCLITimeouts(s *betty.Settings, cli config.ResolvedMeetingsCLI) {
Expand Down
4 changes: 2 additions & 2 deletions src/cmd/hive/meetingswire_cli_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -97,8 +97,8 @@ func TestMeetingPersonaFactoryAgentBackend(t *testing.T) {
e.Close()
}
missing := cliLLM(filepath.Join(dir, "absent"), dir)
if m, ok := meetingCLIModel(missing, meetllm.Config{MaxInFlight: 1}, "m3", logger); !ok || m.Config.Session == nil {
t.Fatal("a missing binary still yields a session that retries the start")
if m, ok := meetingCLIModel(missing, meetllm.Config{MaxInFlight: 1}, "m3", logger); !ok || m.Config.Pool == nil {
t.Fatal("a missing binary still yields a pool that retries the start")
} else {
_ = m.Close()
}
Expand Down
3 changes: 2 additions & 1 deletion src/docs/meetings.md
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@ The CLI uses the sign-in the agents get, without hive logging it or writing it t

A refused backend fails config validation and `PUT /api/meetings/model` with its reason, and the Settings picker lists it disabled, with the reason under "Why some backends are unavailable for meetings". A `meetings.llm.backend` that is neither a known backend nor a configured gateway is refused the same way.

The warm process is replaced after `HIVE_MEETINGS_CLI_RECYCLE_TURNS` turns (default 8, which bounds the history each turn re-reads and pays for), and at once after any error, timeout or crash, so a late reply can never answer the next request and the next call finds a fresh process. Calls from one meeting run one at a time inside the shared in-flight limit. On a CLI backend Betty's timeouts are wider than the API modes' 4 s, 2.5 s and 800 ms: `HIVE_MEETINGS_CLI_ADDRESSED_TIMEOUT_MS`, `HIVE_MEETINGS_CLI_UNPROMPTED_TIMEOUT_MS` and `HIVE_MEETINGS_CLI_RESTYLE_TIMEOUT_MS`, default 8000, 6000 and 3000 on `claude` and 20000, 15000 and 10000 on `copilot` and `omp`, which start the CLI on every call. Spek's per-prompt call uses `HIVE_MEETINGS_CLI_ONESHOT_TIMEOUT_MS` (30000). `HIVE_MEETINGS_CLI_BINARY` overrides the CLI binary. The panel shows the model in use, for example "claude CLI claude-sonnet-5-5" or "vllm model llama-4".
The warm process is replaced after `HIVE_MEETINGS_CLI_RECYCLE_TURNS` turns (default 8, which bounds the history each turn re-reads and pays for), and at once after any error, timeout or crash, so a late reply can never answer the next request and the next call finds a fresh process. Calls from one meeting run in parallel, up to `meetings.llm.per_meeting_in_flight` (default 3) inside the shared in-flight limit: a warm CLI answers one turn at a time on one stream (its results carry no request id), so the meeting keeps a pool of up to that many warm CLIs. Only the first starts with the meeting; when two requests overlap and no idle warm CLI is left, the next one is warmed in the background, so a burst of questions pays at most one cold start (a few seconds on `claude`) and the rest of the meeting answers warm. A meeting that never overlaps requests runs one CLI. On a CLI backend Betty's timeouts are wider than the API modes' 4 s, 2.5 s and 800 ms: `HIVE_MEETINGS_CLI_ADDRESSED_TIMEOUT_MS`, `HIVE_MEETINGS_CLI_UNPROMPTED_TIMEOUT_MS` and `HIVE_MEETINGS_CLI_RESTYLE_TIMEOUT_MS`, default 8000, 6000 and 3000 on `claude` and 20000, 15000 and 10000 on `copilot` and `omp`, which start the CLI on every call. Spek's per-prompt call uses `HIVE_MEETINGS_CLI_ONESHOT_TIMEOUT_MS` (30000). `HIVE_MEETINGS_CLI_BINARY` overrides the CLI binary. The panel shows the model in use, for example "claude CLI claude-sonnet-5-5" or "vllm model llama-4".

Spektacular must be a version that includes the `meeting` command ([hivecommons/spektacular#126](https://github.com/hivecommons/spektacular/pull/126)). With an older binary the live mode exits at once; hive keeps the meeting going (transcript, consent, `hive leave`, research approvals, the panel) without Spek's questions and notes, and the panel says plainly that notes and clarifying questions are off for the meeting. While hive cannot reach a Spektacular child that is restarting, the panel says they are paused, and clears it when Spek writes again.

Expand Down Expand Up @@ -323,6 +323,7 @@ Every key has an environment override that wins over `hive.yaml`. The full list
| `meetings.llm.anthropic_model` | Spektacular's default | Anthropic model. |
| `meetings.llm.timeout_ms` | `2500` | Per-call model timeout. |
| `meetings.llm.max_in_flight` | `4` | Concurrent model calls across meetings. |
| `meetings.llm.per_meeting_in_flight` | `3` | Concurrent model calls in one meeting, so Betty answers several requests at once; never above `max_in_flight` (`HIVE_MEETINGS_LLM_PER_MEETING_IN_FLIGHT`). Each call in flight reserves its expected cost against the meeting's token budget, so parallel answers cannot overspend it. |
| `meetings.email.invites_enabled` | `true` | Accept calendar invites sent to `hive@<host>` (`HIVE_MEETINGS_EMAIL_INVITES_ENABLED`). |
| `meetings.email.files_enabled` | `true` | Accept mail-in files as Inbox items (`HIVE_MEETINGS_EMAIL_FILES_ENABLED`). |
| `meetings.persona.enabled` | `true` | The meeting attendee persona. |
Expand Down
17 changes: 16 additions & 1 deletion src/pkg/config/meetings_config.go
Original file line number Diff line number Diff line change
Expand Up @@ -284,6 +284,10 @@ type MeetingsLLMConfig struct {
// MaxInFlight caps concurrent model calls across meetings. Zero means
// DefaultMeetingsLLMMaxInFlight.
MaxInFlight int `yaml:"max_in_flight,omitempty" json:"max_in_flight,omitempty"`
// PerMeetingInFlight caps one meeting's concurrent model calls, so the
// persona answers several requests at once (#18590); it never exceeds
// MaxInFlight. Zero means DefaultMeetingsLLMPerMeetingInFlight.
PerMeetingInFlight int `yaml:"per_meeting_in_flight,omitempty" json:"per_meeting_in_flight,omitempty"`
}

// Meetings LLM modes and defaults.
Expand All @@ -301,6 +305,9 @@ const (
DefaultMeetingsLLMTimeoutMS = 2500
// DefaultMeetingsLLMMaxInFlight lets a few meetings reason at once.
DefaultMeetingsLLMMaxInFlight = 4
// DefaultMeetingsLLMPerMeetingInFlight lets one meeting's persona work
// on a few requests at once.
DefaultMeetingsLLMPerMeetingInFlight = 3
// DefaultMeetingsLLMKeyFile and DefaultMeetingsAnthropicKeyFile are the
// keystore-sealed secret files.
DefaultMeetingsLLMKeyFile = WritableSecretsDir + "/meetings_llm_api_key"
Expand All @@ -325,7 +332,9 @@ const (
MeetingsAnthropicModelEnv = "HIVE_MEETINGS_ANTHROPIC_MODEL"
MeetingsLLMTimeoutEnv = "HIVE_MEETINGS_LLM_TIMEOUT_MS"
MeetingsLLMMaxInFlightEnv = "HIVE_MEETINGS_LLM_MAX_IN_FLIGHT"
MeetingsLLMBackendEnv = "HIVE_MEETINGS_LLM_BACKEND"
// MeetingsLLMPerMeetingInFlightEnv overrides per_meeting_in_flight.
MeetingsLLMPerMeetingInFlightEnv = "HIVE_MEETINGS_LLM_PER_MEETING_IN_FLIGHT"
MeetingsLLMBackendEnv = "HIVE_MEETINGS_LLM_BACKEND"
)

// ResolvedMeetingsLLM is the effective meetings model. Keys are secrets;
Expand All @@ -343,6 +352,9 @@ type ResolvedMeetingsLLM struct {
AnthropicModel string
TimeoutMS int
MaxInFlight int
// PerMeetingInFlight is one meeting's concurrent model calls, at most
// MaxInFlight.
PerMeetingInFlight int
// Reason explains Backend none.
Reason string
// AgentBackend is the agent backend (claude, ...) on the agent-backend
Expand Down Expand Up @@ -396,6 +408,8 @@ func (c *Config) ResolveMeetingsLLM(ctx context.Context) ResolvedMeetingsLLM {
MaxInFlight: meetingsEnvInt(MeetingsLLMMaxInFlightEnv, l.MaxInFlight, DefaultMeetingsLLMMaxInFlight),
AnthropicModel: meetingsEnvString(MeetingsAnthropicModelEnv, l.AnthropicModel, ""),
}
r.PerMeetingInFlight = min(r.MaxInFlight,
meetingsEnvInt(MeetingsLLMPerMeetingInFlightEnv, l.PerMeetingInFlight, DefaultMeetingsLLMPerMeetingInFlight))
if mode == MeetingsLLMModeAgentBackend || mode == MeetingsLLMModeClaudeCLI {
return c.resolveMeetingsAgentBackend(ctx, mode, r)
}
Expand Down Expand Up @@ -750,6 +764,7 @@ func (m MeetingsConfig) Validate() error {
{"meetings.budget.max_intents_per_meeting", m.Budget.MaxIntentsPerMeeting},
{"meetings.llm.timeout_ms", m.LLM.TimeoutMS},
{"meetings.llm.max_in_flight", m.LLM.MaxInFlight},
{"meetings.llm.per_meeting_in_flight", m.LLM.PerMeetingInFlight},
} {
if f.v < 0 {
return fmt.Errorf("%s must not be negative (got %d)", f.key, f.v)
Expand Down
7 changes: 6 additions & 1 deletion src/pkg/config/meetings_config_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -352,7 +352,8 @@ func TestResolveMeetingsLLM(t *testing.T) {
cfg.Governor.LiteLLM.DefaultModel = "gpt-x"
r := cfg.ResolveMeetingsLLM(ctx)
if r.Backend != MeetingsLLMModeOpenAI || r.Endpoint != "https://litellm.example" || r.Model != "gpt-x" ||
r.TimeoutMS != DefaultMeetingsLLMTimeoutMS || r.MaxInFlight != DefaultMeetingsLLMMaxInFlight || r.Label() != "model gpt-x" {
r.TimeoutMS != DefaultMeetingsLLMTimeoutMS || r.MaxInFlight != DefaultMeetingsLLMMaxInFlight || r.Label() != "model gpt-x" ||
r.PerMeetingInFlight != DefaultMeetingsLLMPerMeetingInFlight {
t.Fatalf("litellm fallback %+v", r)
}
// Meetings' own endpoint, model and key win.
Expand All @@ -366,6 +367,10 @@ func TestResolveMeetingsLLM(t *testing.T) {
if r.Endpoint != "https://llm.example" || r.Model != "m1" || r.APIKey != "mkey" || r.TimeoutMS != 900 || r.MaxInFlight != 2 {
t.Fatalf("meetings llm %+v", r)
}
// One meeting never runs more calls than the hive-wide cap.
if r.PerMeetingInFlight != 2 {
t.Fatalf("per-meeting in flight %d above max_in_flight 2", r.PerMeetingInFlight)
}
// An Anthropic key switches to anthropic mode.
aFile := filepath.Join(dir, "a")
if err := WriteMeetingsSecret(ctx, aFile, "akey"); err != nil {
Expand Down
14 changes: 14 additions & 0 deletions src/pkg/dashboard/api_meetings.go
Original file line number Diff line number Diff line change
Expand Up @@ -470,6 +470,7 @@ func (s *Server) meetingsSettings(ctx context.Context, m config.MeetingsConfig)
"meetings.llm.anthropic_model": m.LLM.AnthropicModel,
"meetings.llm.timeout_ms": strconv.Itoa(m.LLM.TimeoutMS),
"meetings.llm.max_in_flight": strconv.Itoa(m.LLM.MaxInFlight),
"meetings.llm.per_meeting_in_flight": strconv.Itoa(meetingsPerMeetingInFlight(m.LLM)),
"meetings.persona.enabled": boolString(persona.Enabled),
"meetings.persona.preset": persona.Preset,
"meetings.persona.name": persona.Name,
Expand Down Expand Up @@ -594,3 +595,16 @@ func (s *Server) handleMeetingsSettingsPost(w http.ResponseWriter, r *http.Reque
out.RestartRequired = changed
jsonResponse(w, out)
}

// meetingsPerMeetingInFlight is meetings.llm.per_meeting_in_flight as it
// applies: the default when unset, never above max_in_flight.
func meetingsPerMeetingInFlight(l config.MeetingsLLMConfig) int {
n, ceiling := l.PerMeetingInFlight, l.MaxInFlight
if n <= 0 {
n = config.DefaultMeetingsLLMPerMeetingInFlight
}
if ceiling <= 0 {
ceiling = config.DefaultMeetingsLLMMaxInFlight
}
return min(n, ceiling)
}
5 changes: 4 additions & 1 deletion src/pkg/meetings/betty/answer.go
Original file line number Diff line number Diff line change
Expand Up @@ -55,8 +55,11 @@ func (m MeetLLM) Complete(ctx context.Context, system, prompt string) (string, e
return meetllm.Complete(ctx, c, hc, prompt)
}

// Close stops the meeting's warm CLI, if it has one.
// Close stops the meeting's warm CLIs, if it has any.
func (m MeetLLM) Close() error {
if m.Config.Pool != nil {
_ = m.Config.Pool.Close()
}
if m.Config.Session != nil {
return m.Config.Session.Close()
}
Expand Down
2 changes: 1 addition & 1 deletion src/pkg/meetings/meetllm/anthropic.go
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@ func completeAnthropic(ctx context.Context, c Config, hc *http.Client, prompt st
if err := json.NewDecoder(io.LimitReader(resp.Body, maxResponseBytes)).Decode(&mr); err != nil {
return "", fmt.Errorf("meeting-llm: decoding response: %w", err)
}
if err := recordUsage(c.UsageFile, mr.Usage.InputTokens+mr.Usage.OutputTokens, time.Now()); err != nil {
if err := c.charge(mr.Usage.InputTokens+mr.Usage.OutputTokens, time.Now()); err != nil {
return "", fmt.Errorf("meeting-llm: recording usage: %w", err)
}
if mr.StopReason == stopRefusal {
Expand Down
24 changes: 22 additions & 2 deletions src/pkg/meetings/meetllm/cli.go
Original file line number Diff line number Diff line change
Expand Up @@ -532,6 +532,26 @@ func (s *CLISession) lock(ctx context.Context) error {

func (s *CLISession) unlock() { <-s.sem }

// tryLock takes the turn lock if it is free.
func (s *CLISession) tryLock() bool {
select {
case s.sem <- struct{}{}:
return true
default:
return false
}
}

// idle reports that no turn holds the session.
func (s *CLISession) idle() bool { return len(s.sem) == 0 }

// running reports that the session has a live process.
func (s *CLISession) running() bool {
s.mu.Lock()
defer s.mu.Unlock()
return s.proc != nil
}

func (s *CLISession) turnLocked(ctx context.Context, system, prompt string) (string, int, error) {
s.mu.Lock()
p, err := s.ensureLocked()
Expand Down Expand Up @@ -597,7 +617,7 @@ func completeSession(ctx context.Context, c Config, prompt string) (string, erro
}
defer release()
reply, tokens, err := c.Session.turnLocked(ctx, c.System, prompt)
if uerr := recordUsage(c.UsageFile, tokens, time.Now()); uerr != nil && err == nil {
if uerr := c.charge(tokens, time.Now()); uerr != nil && err == nil {
err = fmt.Errorf("meeting-llm: recording usage: %w", uerr)
}
return reply, err
Expand Down Expand Up @@ -655,7 +675,7 @@ func completeCLIOnce(ctx context.Context, c Config, prompt string) (string, erro
}
return "", perr
}
if err := recordUsage(c.UsageFile, out.tokens, time.Now()); err != nil {
if err := c.charge(out.tokens, time.Now()); err != nil {
return "", fmt.Errorf("meeting-llm: recording usage: %w", err)
}
if out.err != nil {
Expand Down
3 changes: 3 additions & 0 deletions src/pkg/meetings/meetllm/cli_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@ const (
fakeCrash = "crash"
fakeCrashOnce = "crashonce"
fakeError = "error"
// fakeSlow answers after half a second, so turns overlap.
fakeSlow = "slow"
)

// fakeTokensPerTurn is what the fake reports per turn: 10+5+3+2.
Expand All @@ -46,6 +48,7 @@ while IFS= read -r line; do
echo "$line" >> "$D/turns"
case "` + behaviour + `" in
hang) sleep 30 ;;
slow) sleep 0.5 ;;
crash) exit 1 ;;
crashonce) if [ ! -f "$D/crashed" ]; then : > "$D/crashed"; exit 1; fi ;;
error) echo '{"type":"result","subtype":"error_during_execution","is_error":true,"result":"","usage":{"input_tokens":10,"output_tokens":5,"cache_creation_input_tokens":3,"cache_read_input_tokens":2}}'; continue ;;
Expand Down
98 changes: 98 additions & 0 deletions src/pkg/meetings/meetllm/gate.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
package meetllm

import (
"context"
"fmt"
"sync"
)

// Per-meeting parallelism (hivecommons/hive#18590): the meeting persona
// answers several requests at once, so one meeting may have more than one
// model call in flight. Gate bounds how many and keeps the meeting's token
// budget honest while they run.
const (
// DefaultPerMeetingInFlight is how many model calls one meeting may
// have in flight when its gate is built with zero.
DefaultPerMeetingInFlight = 3
// defaultCallEstimate is the tokens a call is assumed to cost before
// the meeting has seen one: a persona prompt with retrieval snippets
// and a short JSON answer.
defaultCallEstimate = 2000
)

// Gate is one meeting's in-process admission for model calls: at most N in
// flight, and a token reservation per call so concurrent calls cannot all
// pass the budget check on the same ledger total and overspend it. The
// ledger stays the single record of spend; the reservation only covers the
// calls whose usage is not written yet.
type Gate struct {
sem chan struct{}

mu sync.Mutex
reserved int
// estimate is the largest call cost seen so far (defaultCallEstimate
// until a call reports usage).
estimate int
}

// NewGate builds a gate admitting n concurrent calls; n <= 0 means
// DefaultPerMeetingInFlight.
func NewGate(n int) *Gate {
if n <= 0 {
n = DefaultPerMeetingInFlight
}
return &Gate{sem: make(chan struct{}, n), estimate: defaultCallEstimate}
}

// Size is how many calls the gate admits at once.
func (g *Gate) Size() int { return cap(g.sem) }

// InFlight is how many calls hold the gate now.
func (g *Gate) InFlight() int { return len(g.sem) }

// enter waits for a free place until ctx ends.
func (g *Gate) enter(ctx context.Context) error {
select {
case g.sem <- struct{}{}:
return nil
case <-ctx.Done():
return fmt.Errorf("meeting-llm: meeting's model calls all busy: %w", ctx.Err())
}
}

func (g *Gate) leave() { <-g.sem }

// reserve admits one call against the budget: it refuses (ErrBudget) when
// the ledger total plus the calls still in flight already reach maxTokens,
// and otherwise holds this call's estimate until settle.
func (g *Gate) reserve(used, maxTokens int) (int, error) {
g.mu.Lock()
defer g.mu.Unlock()
if maxTokens > 0 && used+g.reserved >= maxTokens {
return 0, ErrBudget
}
est := g.estimate
g.reserved += est
return est, nil
}

// settle releases a call's reservation once its usage is in the ledger and
// learns its cost for the next estimate.
func (g *Gate) settle(held, spent int) {
g.mu.Lock()
defer g.mu.Unlock()
g.reserved -= held
if g.reserved < 0 {
g.reserved = 0
}
if spent > g.estimate {
g.estimate = spent
}
}

// Reserved is the tokens held for calls in flight.
func (g *Gate) Reserved() int {
g.mu.Lock()
defer g.mu.Unlock()
return g.reserved
}
Loading
Loading