From 6b0952118f3dedafbb3df4265ef76a5e1537e798 Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Mon, 16 Nov 2026 22:48:56 -0500 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20Betty=20times=20every=20answer=20st?= =?UTF-8?q?age=20and=20answers=20live-state=20questions=20without=20waitin?= =?UTF-8?q?g=20for=20search?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each answer carries heard, retrieval, model and answer-ready timings with its source and budget (4 s for the hive's live state, 8 s otherwise); the live session adds delivery and say times and keeps the last 20 with a latency summary in the live state served by the meeting detail API. Gather stops waiting once the hive status retriever answers. Part of #18590. Signed-off-by: Andy Anderson --- .../added-18590-betty-answer-timings.md | 1 + src/docs/meetings.md | 1 + src/pkg/meetings/betty/betty.go | 5 + src/pkg/meetings/betty/queue.go | 26 ++++- src/pkg/meetings/betty/retrieve.go | 11 +- src/pkg/meetings/betty/timing.go | 107 ++++++++++++++++++ src/pkg/meetings/betty/timing_test.go | 68 +++++++++++ src/pkg/meetings/live/session.go | 8 +- src/pkg/meetings/live/timing.go | 93 +++++++++++++++ src/pkg/meetings/live/timing_test.go | 51 +++++++++ src/pkg/meetings/live/types.go | 4 + 11 files changed, 369 insertions(+), 6 deletions(-) create mode 100644 changelog.d/added-18590-betty-answer-timings.md create mode 100644 src/pkg/meetings/betty/timing.go create mode 100644 src/pkg/meetings/betty/timing_test.go create mode 100644 src/pkg/meetings/live/timing.go create mode 100644 src/pkg/meetings/live/timing_test.go diff --git a/changelog.d/added-18590-betty-answer-timings.md b/changelog.d/added-18590-betty-answer-timings.md new file mode 100644 index 0000000000..8c46ba1d99 --- /dev/null +++ b/changelog.d/added-18590-betty-answer-timings.md @@ -0,0 +1 @@ +- **Betty's answers are timed stage by stage.** The meeting's live state (`/api/meetings/{id}/live`) now lists each recent answer's retrieval, model, answer-ready and delivery times, its source and its budget (4 s for questions about the hive's live state, 8 s otherwise), with a median and slowest summary; slow answers are logged. Questions about the hive no longer wait for slower sources such as web search once the live state is in. ([#18590](https://github.com/hivecommons/hive/issues/18590)) diff --git a/src/docs/meetings.md b/src/docs/meetings.md index a9ae7b1bbd..dfbd92696f 100644 --- a/src/docs/meetings.md +++ b/src/docs/meetings.md @@ -37,6 +37,7 @@ Hive attends meetings as a persona: **Betty** ("Betty Bee from the hive", she/he - **Deep dives.** When an addressed question needs research (the model sets `needs_research`, or its confidence is below `min_confidence`) and the meeting has a research path, she says "I'll dig into that and report back", starts a research item through the same pool, cap and approval as Spek's research (shown in the panel as `betty-N`, dispatched with `betty=1` in its meta), and posts exactly one follow-up in her own voice when it completes (the summary as is if restyling fails). - **Room questions.** Betty also answers questions people ask the room (a line ending in "?" or starting with "what", "how", "who" and the like, four words or more), but only if nobody answers within `answer_wait_s` (8 seconds), the knowledge sources had a hit, the model is at least `min_confidence` (0.75) sure, and the topic was not already answered. She never posts general commentary. - **Caps.** Unprompted answers are capped at `per_10m` (2 per ten minutes), at least `min_gap_s` (90 seconds) apart and `per_meeting` (12) per meeting, with one reply per question, at most `queue_depth` (3) questions waiting and none older than `stale_s` (60 seconds). Addressed requests skip these caps but still go through the chat rate limit (`meetings.chat_posts_per_minute`) and the meeting budget. Chat answers are at most 3 sentences and 300 characters. +- **Speed.** Every answer records its stages: when the request was heard, retrieval, the model call, answer ready, and delivery (chat post, or the meeting accepting the speech). The meeting's live state (`GET /api/meetings/{id}/live`, under `live.persona_timings`, the last 20 answers) shows them with the source (`hive-status`, `retrieval`, `model` or `fallback`) and a budget: 4 s from heard to answer for a question about the hive's live state, 8 s otherwise; `live.persona_latency` sums answers, answers over budget, median and slowest. An answer over budget is logged with its stages. Retrieval stops waiting as soon as the hive's live state answers a question about the hive, so a slow web search never delays it. - **Spek.** With Betty on, Spek's spoken replies are dropped, Spek's chat posts go through Betty (posted unless she already covered the topic, so the meeting never gets two answers), and Spek's clarifying questions are reworded in Betty's voice with one quick model call (800 ms; on failure or timeout the original text is posted). The rewording runs beside the meeting loop, so the transcript, notes and other replies keep flowing; Spek's later questions and chat posts wait their turn so they still appear in Spek's order. Notes and research are unchanged. Lines addressed to Betty and leave phrases never count as answers to Spek's questions. - **Manners.** Plain, brief language; at most one light touch of humor, never about a person and none when the conversation sounds tense. Set `meetings.persona.humor: off` to turn humor off. Meeting speech, knowledge and work items reach the model as data, never as instructions. She never claims to have done something she did not do. - **Model.** Betty answers with the meetings model through the same path as `hive meeting-llm` (an OpenAI-compatible endpoint, charged to the meeting's usage ledger and budget). On a hive with only an Anthropic key (`HIVE_MEETINGS_ANTHROPIC_API_KEY` or `meetings.llm.anthropic_key_file`) she calls the Anthropic Messages API directly with `meetings.llm.model` (default `claude-haiku-5-5`), with the same timeouts, in-flight limit and usage ledger, so her tokens count toward the meeting's budget. With no model she still introduces herself but says she cannot answer questions. With `meetings.llm.mode: agent-backend` (or `claude-cli`) she answers through one of the hive's agent backends instead; see [Agent backends](#agent-backends). diff --git a/src/pkg/meetings/betty/betty.go b/src/pkg/meetings/betty/betty.go index ced7c6d0e5..2e2f2eef08 100644 --- a/src/pkg/meetings/betty/betty.go +++ b/src/pkg/meetings/betty/betty.go @@ -55,6 +55,9 @@ type Reply struct { // query through the meeting's research path and to report back with // FollowUp when it completes. Research string `json:"research,omitempty"` + // Timing is the answer's stage timings (timing.go), on answers to a + // request. + Timing *Timing `json:"timing,omitempty"` } // Line is one final transcript segment or chat line. @@ -422,10 +425,12 @@ func (e *Engine) run() { case r := <-e.retrieved: if p := e.pending[r.id]; p != nil { p.snips, p.retrieved = r.snips, true + p.stages.retrievedAt = e.d.Now() e.proceed(p) } case a := <-e.answers: if p := e.pending[a.id]; p != nil { + p.stages.modelDone = a.done e.onAnswer(p, a) } } diff --git a/src/pkg/meetings/betty/queue.go b/src/pkg/meetings/betty/queue.go index dfc68f2dcb..ec7a91a641 100644 --- a/src/pkg/meetings/betty/queue.go +++ b/src/pkg/meetings/betty/queue.go @@ -45,6 +45,8 @@ type pending struct { snips []Snippet // thoughts: a spoken request with no question. thoughts bool + // stages are the request's stage marks (timing.go). + stages stageTimes } type retrieval struct { @@ -56,6 +58,8 @@ type answered struct { id int ans Answer err error + // done is when the model call returned. + done time.Time } // rememberTopic records a topic as answered, for dedupe. @@ -155,12 +159,17 @@ func (e *Engine) addressed(l Line, trig Trigger, now time.Time) { e.drop("budget") return } + early := e.hadEarlyAck(l.Who, now) if trig.VoiceWanted && e.d.LLM != nil && !e.takeEarlyAck(l.Who, now) { // Speak at once so the room knows she heard; the answer follows. e.emit(Reply{Kind: KindVoice, Text: e.ack(), Trigger: TriggerAck, Question: l.Text, VoiceWanted: true}) } p := e.add(l, trig, TopicKey(trig.Rest), true, now) p.waited, p.thoughts = true, thoughts + if trig.VoiceWanted && e.d.LLM != nil { + // Acknowledged from the partial transcript, or just now. + p.stages.ackAt, p.stages.earlyAck = now, early + } e.retrieve(p, trig.Rest) } @@ -203,6 +212,13 @@ func (e *Engine) onPartial(l Line) { } } +// hadEarlyAck reports whether this speaker has an early ack standing, +// without clearing it. +func (e *Engine) hadEarlyAck(who string, now time.Time) bool { + at, ok := e.earlyAck[who] + return ok && now.Sub(at) < earlyAckWindow +} + // takeEarlyAck reports whether this speaker was already acknowledged from // a partial of the line now final, and clears the mark. func (e *Engine) takeEarlyAck(who string, now time.Time) bool { @@ -352,7 +368,8 @@ func (e *Engine) proceed(p *pending) { return } if e.d.LLM == nil { - e.emit(Reply{Kind: replyKind(p.trig), Text: fallbackNoModel, Trigger: TriggerAddressed, Question: p.line.Text, VoiceWanted: p.trig.VoiceWanted}) + e.emit(Reply{Kind: replyKind(p.trig), Text: fallbackNoModel, Trigger: TriggerAddressed, Question: p.line.Text, VoiceWanted: p.trig.VoiceWanted, + Timing: e.timing(p, true, now)}) e.finish(p, "") return } @@ -365,6 +382,7 @@ func (e *Engine) proceed(p *pending) { system := SystemPrompt(e.s.Persona) prompt := answerPrompt(mode, p.line.Who, p.line.Text, p.snips, recent, p.addressed && p.trig.VoiceWanted) id := p.id + p.stages.modelStart = now go func() { ctx, cancel := context.WithTimeout(e.ctx, timeout) defer cancel() @@ -374,7 +392,7 @@ func (e *Engine) proceed(p *pending) { a, err = ParseAnswer(raw) } select { - case e.answers <- answered{id: id, ans: a, err: err}: + case e.answers <- answered{id: id, ans: a, err: err, done: e.d.Now()}: case <-e.ctx.Done(): } }() @@ -399,7 +417,8 @@ func (e *Engine) onAnswer(p *pending, a answered) { e.finish(p, reason) return } - if e.emit(Reply{Text: text, Trigger: TriggerQuestion, Question: p.line.Text, NeedsResearch: a.ans.NeedsResearch, Sources: a.ans.Sources}) { + if e.emit(Reply{Text: text, Trigger: TriggerQuestion, Question: p.line.Text, NeedsResearch: a.ans.NeedsResearch, Sources: a.ans.Sources, + Timing: e.timing(p, false, now)}) { e.posts = append(e.posts, now) e.lastPost = now e.total++ @@ -461,6 +480,7 @@ func (e *Engine) onAddressedAnswer(p *pending, a answered) { rep.NeedsResearch = true rep.Research = clip(researchQuery(p), maxResearchQuery) } + rep.Timing = e.timing(p, a.err != nil || a.ans.Answer == "", e.d.Now()) e.emit(rep) e.rememberTopic(p.topic) e.rememberTopic(TopicKey(text)) diff --git a/src/pkg/meetings/betty/retrieve.go b/src/pkg/meetings/betty/retrieve.go index 8004e0ee6e..d90940efe5 100644 --- a/src/pkg/meetings/betty/retrieve.go +++ b/src/pkg/meetings/betty/retrieve.go @@ -59,7 +59,16 @@ func Gather(ctx context.Context, rs []Retriever, query string, terms []string, t } for n := 0; n < len(rs); n++ { select { - case <-done: + case i := <-done: + // The hive's live state answers a question about the hive on + // its own: do not wait for slower sources such as web search + // (#18590, a live-state answer within HiveStatusBudget). + mu.Lock() + live := hasHiveStatus(results[i]) + mu.Unlock() + if live { + n = len(rs) + } case <-ctx.Done(): n = len(rs) } diff --git a/src/pkg/meetings/betty/timing.go b/src/pkg/meetings/betty/timing.go new file mode 100644 index 0000000000..f8593b4951 --- /dev/null +++ b/src/pkg/meetings/betty/timing.go @@ -0,0 +1,107 @@ +package betty + +import "time" + +// Per-request stage timings (hivecommons/hive#18590): every answer carries +// how long each stage took, so the meeting detail shows where the seconds +// went and whether the answer met its budget. + +// Answer sources, for Timing.Source. +const ( + // SourceHiveStatus: answered from the hive's live state. + SourceHiveStatus = "hive-status" + // SourceRetrieval: answered from knowledge, beads, code maps or web + // hits. + SourceRetrieval = "retrieval" + // SourceModel: answered by the model with no retrieval hits. + SourceModel = "model" + // SourceFallback: the model failed or is missing; a fixed reply. + SourceFallback = "fallback" +) + +// Answer budgets: the time from hearing a request to having its answer. +const ( + // HiveStatusBudget is the target for a question about the hive's live + // state, which needs no search: in-memory status and one model call. + HiveStatusBudget = 4 * time.Second + // AnswerBudget is the target for any other answer. + AnswerBudget = 8 * time.Second +) + +// Timing is one answer's stage timings, in milliseconds from when the +// request was heard (its final line). +type Timing struct { + HeardAt time.Time `json:"heard_at"` + // AckMS is when the spoken acknowledgement went out; EarlyAck marks one + // said from the partial transcript, before the final line (AckMS 0). + AckMS int64 `json:"ack_ms,omitempty"` + EarlyAck bool `json:"early_ack,omitempty"` + // RetrieveMS is how long retrieval took; ModelMS the model call. + RetrieveMS int64 `json:"retrieve_ms"` + ModelMS int64 `json:"model_ms"` + // AnswerMS is heard to answer ready. + AnswerMS int64 `json:"answer_ms"` + Source string `json:"source"` + BudgetMS int64 `json:"budget_ms"` + // OverBudget reports AnswerMS above BudgetMS. + OverBudget bool `json:"over_budget,omitempty"` +} + +// stageTimes are a pending request's stage marks. +type stageTimes struct { + ackAt time.Time + earlyAck bool + retrievedAt time.Time + modelStart time.Time + modelDone time.Time +} + +func ms(d time.Duration) int64 { + if d < 0 { + return 0 + } + return d.Milliseconds() +} + +// since is the milliseconds from a to b, zero when either is unset. +func since(a, b time.Time) int64 { + if a.IsZero() || b.IsZero() { + return 0 + } + return ms(b.Sub(a)) +} + +// timing is p's stage timings for an answer ready at now. +func (e *Engine) timing(p *pending, failed bool, now time.Time) *Timing { + t := &Timing{ + HeardAt: p.created, + EarlyAck: p.stages.earlyAck, + RetrieveMS: since(p.created, p.stages.retrievedAt), + ModelMS: since(p.stages.modelStart, p.stages.modelDone), + AnswerMS: since(p.created, now), + } + if !p.stages.earlyAck { + t.AckMS = since(p.created, p.stages.ackAt) + } + switch { + case failed: + t.Source = SourceFallback + case hasHiveStatus(p.snips): + t.Source = SourceHiveStatus + case len(p.snips) > 0: + t.Source = SourceRetrieval + default: + t.Source = SourceModel + } + budget := AnswerBudget + if t.Source == SourceHiveStatus { + budget = HiveStatusBudget + } + t.BudgetMS = budget.Milliseconds() + t.OverBudget = t.AnswerMS > t.BudgetMS + if t.OverBudget { + e.log.Info("[meetings] persona answer over budget", "source", t.Source, "answer_ms", t.AnswerMS, + "budget_ms", t.BudgetMS, "retrieve_ms", t.RetrieveMS, "model_ms", t.ModelMS) + } + return t +} diff --git a/src/pkg/meetings/betty/timing_test.go b/src/pkg/meetings/betty/timing_test.go new file mode 100644 index 0000000000..6a34527768 --- /dev/null +++ b/src/pkg/meetings/betty/timing_test.go @@ -0,0 +1,68 @@ +package betty + +import ( + "context" + "testing" + "time" +) + +// Each answer carries its stage timings: retrieval, the model call and the +// time to the answer, its source and its budget. +func TestAnswerTimingStages(t *testing.T) { + var r *rig + hook := hookRetriever{fn: func() { r.clk.add(300 * time.Millisecond) }} + llm := &fakeLLM{reply: func(string, string) (string, error) { + r.clk.add(time.Second) + return `{"answer":"It ships Friday.","confidence":0.9}`, nil + }} + r = newRig(t, testSettings(), llm, hook) + r.e.Observe(Line{Who: "Bo", Text: "Betty, when does the release ship?"}) + tm := r.reply().Timing + if tm == nil { + t.Fatal("no timing") + } + if tm.RetrieveMS != 300 || tm.ModelMS != 1000 || tm.AnswerMS != 1300 || tm.Source != SourceRetrieval || + tm.BudgetMS != AnswerBudget.Milliseconds() || tm.OverBudget { + t.Fatalf("timing %+v", tm) + } +} + +// A live-state answer has the tighter budget and is flagged when it +// misses it. +func TestHiveStatusAnswerBudget(t *testing.T) { + var r *rig + llm := &fakeLLM{reply: func(string, string) (string, error) { + r.clk.add(HiveStatusBudget + time.Second) + return `{"answer":"40 open issues.","confidence":0.9}`, nil + }} + r = newRig(t, testSettings(), llm, fakeRetriever{hits: []Snippet{{Source: HiveStatusSource, Title: "The hive right now", Text: "Open now: 40 issues."}}}) + r.e.Observe(Line{Who: "Bo", Text: "Betty, how many open issues are there?"}) + tm := r.reply().Timing + if tm == nil || tm.Source != SourceHiveStatus || tm.BudgetMS != HiveStatusBudget.Milliseconds() || !tm.OverBudget { + t.Fatalf("timing %+v", tm) + } + noModel := newRig(t, testSettings(), nil) + noModel.e.Observe(Line{Who: "Bo", Text: "Betty, when does the release ship?"}) + if tm := noModel.reply().Timing; tm == nil || tm.Source != SourceFallback { + t.Fatalf("fallback timing %+v", tm) + } +} + +// The hive's live state ends retrieval at once: a question about the hive +// does not wait for slower sources. +func TestGatherStopsAtHiveStatus(t *testing.T) { + live := fakeRetriever{hits: []Snippet{{Source: HiveStatusSource, Text: "Open now: 40 issues."}}} + slow := fakeRetriever{hits: []Snippet{{Source: "web", Text: "slow"}}, delay: time.Second} + start := time.Now() + got := Gather(context.Background(), []Retriever{slow, live}, "how many open issues", []string{"issue"}, 1500*time.Millisecond) + if d := time.Since(start); d > 500*time.Millisecond { + t.Fatalf("gather waited %v for slower sources", d) + } + if len(got) != 1 || got[0].Source != HiveStatusSource { + t.Fatalf("got %+v", got) + } + // Without the live state every source is waited for. + if got := Gather(context.Background(), []Retriever{slow}, "q", []string{"q"}, 1500*time.Millisecond); len(got) != 1 { + t.Fatalf("slow source dropped: %+v", got) + } +} diff --git a/src/pkg/meetings/live/session.go b/src/pkg/meetings/live/session.go index 36ecaad3da..b8ff0d8d73 100644 --- a/src/pkg/meetings/live/session.go +++ b/src/pkg/meetings/live/session.go @@ -76,6 +76,8 @@ type session struct { persona *betty.Engine // fileTokens is the usage-ledger total already charged. fileTokens int + // lastSay is how long the meeting's last say call took (timing.go). + lastSay time.Duration // spekQueue keeps Spek's asks and chat offers in order while an ask is // being restyled by the persona off the session goroutine; restyled // carries the results back. @@ -439,7 +441,7 @@ func (s *session) say(ctx context.Context, text string) { m, err := s.r.opts.Meetings.Get(s.id) if err == nil && m.VoiceEnabled && s.r.opts.Policy.VoiceOnAddress { cctx, cancel := context.WithTimeout(ctx, s.r.opts.CallTimeout) - err = s.r.opts.Meetings.Say(cctx, s.id, text) + err = s.timedSay(func() error { return s.r.opts.Meetings.Say(cctx, s.id, text) }) cancel() if err == nil { return @@ -707,6 +709,7 @@ func (s *session) onPersonaReply(ctx context.Context, rep betty.Reply) { return } if s.speak(ctx, text) { + s.recordAckSay() s.emitPersona(rep) } else { s.drop("ack_no_voice") @@ -745,6 +748,7 @@ func (s *session) onPersonaReply(ctx context.Context, rep betty.Reply) { } func (s *session) emitPersona(rep betty.Reply) { + s.recordTiming(rep) raw, _ := json.Marshal(rep) s.emit(Event{Kind: EventPersona, Reason: rep.Trigger, Intent: raw}) } @@ -785,7 +789,7 @@ func (s *session) speak(ctx context.Context, text string) bool { } cctx, cancel := context.WithTimeout(ctx, s.r.opts.CallTimeout) defer cancel() - return s.r.opts.Meetings.Say(cctx, s.id, text) == nil + return s.timedSay(func() error { return s.r.opts.Meetings.Say(cctx, s.id, text) }) == nil } // queuedSpek is one of Spek's asks or chat offers waiting its turn while an diff --git a/src/pkg/meetings/live/timing.go b/src/pkg/meetings/live/timing.go new file mode 100644 index 0000000000..eef4493c26 --- /dev/null +++ b/src/pkg/meetings/live/timing.go @@ -0,0 +1,93 @@ +package live + +import ( + "slices" + "time" + + "github.com/hivecommons/hive/pkg/meetings/betty" +) + +// Persona answer timings in the live state (hivecommons/hive#18590): the +// meeting detail API (GET /api/meetings/{id}/live, `live.persona_timings` +// and `live.persona_latency`) shows every recent answer's stages, heard to +// retrieval to model to answer ready to sent, and a summary against the +// answer budgets. + +// maxPersonaTimings bounds the answers kept with their timings. +const maxPersonaTimings = 20 + +// PersonaTiming is one persona answer's stages. +type PersonaTiming struct { + betty.Timing + Trigger string `json:"trigger"` + Kind string `json:"kind"` + // DeliverMS is answer ready to sent (chat) or handed to the meeting's + // voice (spoken), including any wait behind the chat rate. + DeliverMS int64 `json:"deliver_ms"` + // SayMS is how long the meeting took to accept the speech (TTS on + // Discord synthesises before it accepts), for a spoken answer. + SayMS int64 `json:"say_ms,omitempty"` + // TotalMS is heard to sent. + TotalMS int64 `json:"total_ms"` +} + +// PersonaLatency sums the persona's answer timings: counts over the +// meeting, percentiles over the kept answers. +type PersonaLatency struct { + Answers int `json:"answers"` + OverBudget int `json:"over_budget"` + P50MS int64 `json:"p50_ms"` + MaxMS int64 `json:"max_ms"` + // LastAckSayMS is how long the last spoken acknowledgement took to be + // accepted. + LastAckSayMS int64 `json:"last_ack_say_ms,omitempty"` +} + +// recordTiming adds a persona answer's timings once it is out. +func (s *session) recordTiming(rep betty.Reply) { + if rep.Timing == nil { + return + } + now := s.now() + ready := rep.Timing.HeardAt.Add(time.Duration(rep.Timing.AnswerMS) * time.Millisecond) + t := PersonaTiming{Timing: *rep.Timing, Trigger: rep.Trigger, Kind: rep.Kind, + DeliverMS: max(0, now.Sub(ready).Milliseconds()), TotalMS: max(0, now.Sub(rep.Timing.HeardAt).Milliseconds())} + if rep.Kind == betty.KindVoice { + t.SayMS = s.lastSay.Milliseconds() + } + s.state.PersonaTimings = append(s.state.PersonaTimings, t) + if n := len(s.state.PersonaTimings); n > maxPersonaTimings { + s.state.PersonaTimings = s.state.PersonaTimings[n-maxPersonaTimings:] + } + l := s.latency() + l.Answers++ + if t.OverBudget { + l.OverBudget++ + } + totals := make([]int64, 0, len(s.state.PersonaTimings)) + for _, x := range s.state.PersonaTimings { + totals = append(totals, x.TotalMS) + } + slices.Sort(totals) + l.P50MS, l.MaxMS = totals[len(totals)/2], totals[len(totals)-1] +} + +// recordAckSay notes how long a spoken acknowledgement took to go out. +func (s *session) recordAckSay() { + s.latency().LastAckSayMS = s.lastSay.Milliseconds() +} + +func (s *session) latency() *PersonaLatency { + if s.state.PersonaLatency == nil { + s.state.PersonaLatency = &PersonaLatency{} + } + return s.state.PersonaLatency +} + +// timedSay calls the meeting's say and keeps how long it took. +func (s *session) timedSay(say func() error) error { + start := time.Now() + err := say() + s.lastSay = time.Since(start) + return err +} diff --git a/src/pkg/meetings/live/timing_test.go b/src/pkg/meetings/live/timing_test.go new file mode 100644 index 0000000000..ee08829804 --- /dev/null +++ b/src/pkg/meetings/live/timing_test.go @@ -0,0 +1,51 @@ +package live + +import ( + "testing" + + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/meetings" + "github.com/hivecommons/hive/pkg/meetings/betty" +) + +// The live state (the meeting detail API's `live`) keeps each answer's +// stage timings and a latency summary. +func TestPersonaTimingsInLiveState(t *testing.T) { + h := personaHarness(t, Policy{ChatPostsPerMinute: 20}, config.MeetingsPersonaConfig{}, &personaLLM{}) + h.live() + h.line(meetings.EventChat, "Bo", "Betty, when does the release ship?", false) + h.eventually("answer", h.posted("The release ships Friday.")) + h.eventually("timing recorded", func() bool { return len(h.state().PersonaTimings) == 1 }) + st := h.state() + tm := st.PersonaTimings[0] + if tm.Source != betty.SourceRetrieval || tm.Trigger != betty.TriggerAddressed || tm.Kind != betty.KindChat || + tm.TotalMS < tm.AnswerMS || tm.BudgetMS == 0 { + t.Fatalf("timing %+v", tm) + } + if l := st.PersonaLatency; l == nil || l.Answers != 1 || l.MaxMS != tm.TotalMS || l.P50MS != tm.TotalMS { + t.Fatalf("latency %+v", st.PersonaLatency) + } +} + +func TestRecordTimingKeepsTheLastAnswers(t *testing.T) { + _, s := timingSession(t) + for i := range maxPersonaTimings + 5 { + s.recordTiming(betty.Reply{Trigger: betty.TriggerAddressed, Timing: &betty.Timing{HeardAt: s.now(), AnswerMS: int64(i), OverBudget: i%2 == 0}}) + } + if n := len(s.state.PersonaTimings); n != maxPersonaTimings { + t.Fatalf("kept %d timings", n) + } + if l := s.state.PersonaLatency; l.Answers != maxPersonaTimings+5 || l.OverBudget != (maxPersonaTimings+5+1)/2 { + t.Fatalf("latency %+v", l) + } + s.recordTiming(betty.Reply{Trigger: betty.TriggerAck}) + if l := s.state.PersonaLatency; l.Answers != maxPersonaTimings+5 { + t.Fatal("a reply without timings is not counted") + } +} + +func timingSession(t *testing.T) (*harness, *session) { + t.Helper() + h := newHarness(t, Policy{}, nil) + return h, h.r.newSession(h.m, h.spek) +} diff --git a/src/pkg/meetings/live/types.go b/src/pkg/meetings/live/types.go index 3e62d15d47..8e6137a8d8 100644 --- a/src/pkg/meetings/live/types.go +++ b/src/pkg/meetings/live/types.go @@ -136,6 +136,10 @@ type State struct { Persona string `json:"persona,omitempty"` // PersonaReplies counts the persona's posted replies. PersonaReplies int `json:"persona_replies,omitempty"` + // PersonaTimings are the last answers' stage timings, newest last; + // PersonaLatency sums them (timing.go, #18590). + PersonaTimings []PersonaTiming `json:"persona_timings,omitempty"` + PersonaLatency *PersonaLatency `json:"persona_latency,omitempty"` } // StateDoc is the side document name the state is stored under.