From 75190ce52ccc3abec1bd7d3707f6d0c4982cc719 Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Mon, 16 Nov 2026 22:53:25 -0500 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20Betty=20cites=20the=20sources=20of?= =?UTF-8?q?=20chat=20answers=20and=20says=20when=20she=20is=20not=20sure?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Chat answers end with up to two sources the model reported using (web link, work item id, knowledge fact, code map or hive live status); the prompt asks for issue and PR numbers and forbids guessing; a low-confidence addressed answer not from the live state is hedged. Part of #18590. Signed-off-by: Andy Anderson --- .../added-18590-betty-cites-sources.md | 1 + src/docs/meetings.md | 1 + src/pkg/meetings/betty/betty_test.go | 4 +- src/pkg/meetings/betty/cite.go | 133 ++++++++++++++++++ src/pkg/meetings/betty/cite_test.go | 73 ++++++++++ src/pkg/meetings/betty/persona.go | 3 + src/pkg/meetings/betty/queue.go | 21 ++- 7 files changed, 233 insertions(+), 3 deletions(-) create mode 100644 changelog.d/added-18590-betty-cites-sources.md create mode 100644 src/pkg/meetings/betty/cite.go create mode 100644 src/pkg/meetings/betty/cite_test.go diff --git a/changelog.d/added-18590-betty-cites-sources.md b/changelog.d/added-18590-betty-cites-sources.md new file mode 100644 index 0000000000..e5646e8985 --- /dev/null +++ b/changelog.d/added-18590-betty-cites-sources.md @@ -0,0 +1 @@ +- **Betty says where her answers come from and when she is not sure.** Chat answers end with the sources they used (a web link, a work item id, the knowledge fact or "hive live status"), the model is told to give issue and PR numbers and never guess, and a low-confidence answer opens with "I'm not sure, but ..." instead of sounding certain. ([#18590](https://github.com/hivecommons/hive/issues/18590)) diff --git a/src/docs/meetings.md b/src/docs/meetings.md index a9ae7b1bbd..f6a34cd27e 100644 --- a/src/docs/meetings.md +++ b/src/docs/meetings.md @@ -37,6 +37,7 @@ Hive attends meetings as a persona: **Betty** ("Betty Bee from the hive", she/he - **Deep dives.** When an addressed question needs research (the model sets `needs_research`, or its confidence is below `min_confidence`) and the meeting has a research path, she says "I'll dig into that and report back", starts a research item through the same pool, cap and approval as Spek's research (shown in the panel as `betty-N`, dispatched with `betty=1` in its meta), and posts exactly one follow-up in her own voice when it completes (the summary as is if restyling fails). - **Room questions.** Betty also answers questions people ask the room (a line ending in "?" or starting with "what", "how", "who" and the like, four words or more), but only if nobody answers within `answer_wait_s` (8 seconds), the knowledge sources had a hit, the model is at least `min_confidence` (0.75) sure, and the topic was not already answered. She never posts general commentary. - **Caps.** Unprompted answers are capped at `per_10m` (2 per ten minutes), at least `min_gap_s` (90 seconds) apart and `per_meeting` (12) per meeting, with one reply per question, at most `queue_depth` (3) questions waiting and none older than `stale_s` (60 seconds). Addressed requests skip these caps but still go through the chat rate limit (`meetings.chat_posts_per_minute`) and the meeting budget. Chat answers are at most 3 sentences and 300 characters. +- **Sources and honesty.** A chat answer ends with the sources it used, from the context entries the model reports: a web hit's link, a work item's id, or the knowledge fact, code map or "hive live status" it came from, at most two, skipped when the answer already names them (`(Source: https://...; hive-42)`). The model is told to give issue and PR numbers as the context has them and never to guess numbers, names, dates or links. An addressed answer below `unprompted.min_confidence` that is not from the hive's live state, and that will not become a deep dive, opens with "I'm not sure, but ..." unless it already says so. Spoken answers carry no sources. - **Spek.** With Betty on, Spek's spoken replies are dropped, Spek's chat posts go through Betty (posted unless she already covered the topic, so the meeting never gets two answers), and Spek's clarifying questions are reworded in Betty's voice with one quick model call (800 ms; on failure or timeout the original text is posted). The rewording runs beside the meeting loop, so the transcript, notes and other replies keep flowing; Spek's later questions and chat posts wait their turn so they still appear in Spek's order. Notes and research are unchanged. Lines addressed to Betty and leave phrases never count as answers to Spek's questions. - **Manners.** Plain, brief language; at most one light touch of humor, never about a person and none when the conversation sounds tense. Set `meetings.persona.humor: off` to turn humor off. Meeting speech, knowledge and work items reach the model as data, never as instructions. She never claims to have done something she did not do. - **Model.** Betty answers with the meetings model through the same path as `hive meeting-llm` (an OpenAI-compatible endpoint, charged to the meeting's usage ledger and budget). On a hive with only an Anthropic key (`HIVE_MEETINGS_ANTHROPIC_API_KEY` or `meetings.llm.anthropic_key_file`) she calls the Anthropic Messages API directly with `meetings.llm.model` (default `claude-haiku-5-5`), with the same timeouts, in-flight limit and usage ledger, so her tokens count toward the meeting's budget. With no model she still introduces herself but says she cannot answer questions. With `meetings.llm.mode: agent-backend` (or `claude-cli`) she answers through one of the hive's agent backends instead; see [Agent backends](#agent-backends). diff --git a/src/pkg/meetings/betty/betty_test.go b/src/pkg/meetings/betty/betty_test.go index fc22525fb4..9991c60f76 100644 --- a/src/pkg/meetings/betty/betty_test.go +++ b/src/pkg/meetings/betty/betty_test.go @@ -316,7 +316,7 @@ func TestEngineAddressedAnswer(t *testing.T) { r.e.Observe(Line{Who: "Ana", Text: "we moved the release to Friday"}) r.e.Observe(Line{Who: "Bo", Text: "Betty, when does the release ship?"}) rep := r.reply() - if rep.Trigger != TriggerAddressed || rep.Text != "It ships Friday." || len(rep.Sources) != 1 { + if rep.Trigger != TriggerAddressed || rep.Text != `It ships Friday. (Source: knowledge "Release")` || len(rep.Sources) != 1 { t.Fatalf("addressed %+v", rep) } c := llm.last() @@ -350,7 +350,7 @@ func TestEngineRoomQuestionAnsweredAfterWait(t *testing.T) { start := time.Now() r.e.Observe(Line{Who: "Ana", Text: "When does the release ship?"}) rep := r.reply() - if rep.Trigger != TriggerQuestion || rep.Text != "It ships Friday." || time.Since(start) < testWait { + if rep.Trigger != TriggerQuestion || rep.Text != `It ships Friday. (Source: knowledge "Release")` || time.Since(start) < testWait { t.Fatalf("question %+v after %v", rep, time.Since(start)) } if !strings.Contains(llm.last().prompt, "nobody has answered") { diff --git a/src/pkg/meetings/betty/cite.go b/src/pkg/meetings/betty/cite.go new file mode 100644 index 0000000000..b4bf6e11be --- /dev/null +++ b/src/pkg/meetings/betty/cite.go @@ -0,0 +1,133 @@ +package betty + +import ( + "regexp" + "strconv" + "strings" + "unicode/utf8" +) + +// Citations (hivecommons/hive#18590): a chat answer says where its facts +// came from, from the context entries the model reported using: the link +// of a web hit, a work item's id, or the name of a knowledge fact, code +// map or the hive's live status. Spoken answers carry none (no links +// aloud). +const ( + // maxCitations bounds the sources named after one answer. + maxCitations = 2 + // maxCitationChars bounds one named source. + maxCitationChars = 80 +) + +// citeFormat is the citation suffix by primary language subtag. +var citeFormat = map[string]string{ + "en": "Source: ", "es": "Fuente: ", "fr": "Source : ", "de": "Quelle: ", "pt": "Fonte: ", "it": "Fonte: ", +} + +// trailingURL is the link a web snippet's text ends with. +var trailingURL = regexp.MustCompile(`\((https?://[^\s()]+)\)\s*$`) + +// citation names one snippet as a source. +func citation(s Snippet) string { + switch { + case strings.HasPrefix(s.Source, webSource+" "): + if m := trailingURL.FindStringSubmatch(s.Text); m != nil { + return m[1] + } + return strings.TrimPrefix(s.Source, webSource+" ") + case strings.HasPrefix(s.Source, "work item "): + return strings.TrimPrefix(s.Source, "work item ") + case s.Source == HiveStatusSource: + return "hive live status" + case s.Title != "": + return s.Source + " \"" + s.Title + "\"" + } + return s.Source +} + +// citeSources appends the sources an answer used to a chat answer, unless +// the answer already names them. Sources are the model's context numbers +// (1-based); unknown ones are skipped. +func citeSources(lang, text string, used []string, snips []Snippet) string { + if strings.TrimSpace(text) == "" || len(used) == 0 { + return text + } + var names []string + seen := map[string]bool{} + for _, u := range used { + i, err := strconv.Atoi(strings.Trim(strings.TrimSpace(u), `"[]`)) + if err != nil || i < 1 || i > len(snips) { + continue + } + c := strings.TrimSpace(citation(snips[i-1])) + if c == "" || seen[c] || strings.Contains(text, c) { + continue + } + seen[c] = true + if utf8.RuneCountInString(c) > maxCitationChars { + c = string([]rune(c)[:maxCitationChars-1]) + "…" + } + names = append(names, c) + if len(names) == maxCitations { + break + } + } + if len(names) == 0 { + return text + } + prefix := citeFormat[primaryLang(lang)] + if prefix == "" { + prefix = citeFormat["en"] + } + return strings.TrimSpace(text) + " (" + prefix + strings.Join(names, "; ") + ")" +} + +func primaryLang(lang string) string { + return strings.ToLower(strings.SplitN(strings.TrimSpace(lang), "-", 2)[0]) +} + +// unsurePhrases already say she is not sure, in the persona languages. +var unsurePhrases = []string{ + "not sure", "don't know", "do not know", "no sé", "no estoy segur", "pas sûr", "je ne sais pas", + "nicht sicher", "weiß ich nicht", "não sei", "não tenho certeza", "non sono sicur", "non lo so", +} + +// unsurePrefix opens a low-confidence answer, by language. +var unsurePrefix = map[string]string{ + "en": "I'm not sure, but ", "es": "No estoy segura, pero ", "fr": "Je ne suis pas sûre, mais ", + "de": "Ich bin nicht sicher, aber ", "pt": "Não tenho certeza, mas ", "it": "Non ne sono sicura, ma ", +} + +// hedge marks a low-confidence answer as unsure, unless it already says +// so, rather than stating a guess as fact. +func hedge(lang, text string) string { + t := strings.TrimSpace(text) + if t == "" { + return text + } + low := strings.ToLower(t) + for _, p := range unsurePhrases { + if strings.Contains(low, p) { + return text + } + } + prefix := unsurePrefix[primaryLang(lang)] + if prefix == "" { + prefix = unsurePrefix["en"] + } + return prefix + lowerInitial(t) +} + +// lowerInitial lowercases the first letter unless it starts "I " or an +// acronym. +func lowerInitial(s string) string { + r, n := utf8.DecodeRuneInString(s) + if n == 0 || r < 'A' || r > 'Z' { + return s + } + next, _ := utf8.DecodeRuneInString(s[n:]) + if (r == 'I' && (next == ' ' || next == '\'')) || (next >= 'A' && next <= 'Z') || (next >= '0' && next <= '9') { + return s + } + return string(r+('a'-'A')) + s[n:] +} diff --git a/src/pkg/meetings/betty/cite_test.go b/src/pkg/meetings/betty/cite_test.go new file mode 100644 index 0000000000..3eabad49ef --- /dev/null +++ b/src/pkg/meetings/betty/cite_test.go @@ -0,0 +1,73 @@ +package betty + +import ( + "strings" + "testing" +) + +func TestCiteSources(t *testing.T) { + snips := []Snippet{ + {Source: "web example.com", Title: "Cert docs", Text: "Renews nightly (https://example.com/certs)"}, + {Source: "work item hive-42", Title: "Rotate keys", Text: "open"}, + {Source: "knowledge", Title: "Release plan", Text: "Friday"}, + {Source: HiveStatusSource, Title: "The hive right now", Text: "40 issues"}, + } + for _, c := range []struct { + text string + used []string + want string + }{ + {"It renews nightly.", []string{"1", "2"}, "It renews nightly. (Source: https://example.com/certs; hive-42)"}, + {"It renews nightly.", []string{"1", "2", "3"}, "It renews nightly. (Source: https://example.com/certs; hive-42)"}, + {"Friday.", []string{"[3]"}, `Friday. (Source: knowledge "Release plan")`}, + {"40 open issues.", []string{"4"}, "40 open issues. (Source: hive live status)"}, + {"See https://example.com/certs.", []string{"1"}, "See https://example.com/certs."}, + {"No sources.", nil, "No sources."}, + {"Bad index.", []string{"9", "x"}, "Bad index."}, + } { + if got := citeSources("en", c.text, c.used, snips); got != c.want { + t.Errorf("citeSources(%q, %v) = %q, want %q", c.text, c.used, got, c.want) + } + } + if got := citeSources("de-DE", "Freitag.", []string{"3"}, snips); !strings.HasPrefix(got, "Freitag. (Quelle: ") { + t.Fatalf("localized %q", got) + } +} + +func TestHedge(t *testing.T) { + for in, want := range map[string]string{ + "It ships Friday.": "I'm not sure, but it ships Friday.", + "Not sure, maybe Friday.": "Not sure, maybe Friday.", + "I don't know yet.": "I don't know yet.", + "PR 12 fixed it.": "I'm not sure, but PR 12 fixed it.", + "I think it ships on Friday.": "I'm not sure, but I think it ships on Friday.", + } { + if got := hedge("en", in); got != want { + t.Errorf("hedge(%q) = %q, want %q", in, got, want) + } + } +} + +// A low-confidence addressed answer is said as unsure, not as fact; the +// hive's live state is never hedged; spoken answers carry no citation. +func TestAddressedAnswerHedgedAndCited(t *testing.T) { + llm := &fakeLLM{reply: func(string, string) (string, error) { + return `{"answer":"It ships Friday.","confidence":0.4,"sources":[1]}`, nil + }} + r := newRig(t, testSettings(), llm) + r.e.Observe(Line{Who: "Bo", Text: "Betty, when does the release ship?"}) + if a := r.reply(); a.Text != `I'm not sure, but it ships Friday. (Source: knowledge "Release")` { + t.Fatalf("hedged %q", a.Text) + } + r.e.Observe(Line{Who: "Bo", Text: "Betty, when does the release ship, say it out loud?", Spoken: true}) + for { + a := r.reply() + if a.Trigger == TriggerAck { + continue + } + if a.Text != "I'm not sure, but it ships Friday." { + t.Fatalf("spoken %q", a.Text) + } + break + } +} diff --git a/src/pkg/meetings/betty/persona.go b/src/pkg/meetings/betty/persona.go index 46baf09fe7..07a54d6cc3 100644 --- a/src/pkg/meetings/betty/persona.go +++ b/src/pkg/meetings/betty/persona.go @@ -87,7 +87,10 @@ func answerPrompt(mode, who, question string, snips []Snippet, recent []recentLi } if spoken { b.WriteString("Your answer will be spoken aloud in the meeting: at most 2 sentences and 200 characters, no links, no lists, no Markdown.\n") + } else { + b.WriteString("Name where facts come from: give issue and PR numbers (#123) and work item ids as the context has them, and list the context numbers you used in sources.\n") } + b.WriteString("Only state what the context or the transcript supports. If they do not answer it, say you are not sure and set confidence below 0.5; never guess numbers, names, dates or links.\n") if hasHiveStatus(snips) { b.WriteString("Context marked \"" + HiveStatusSource + "\" is the hive's own state read just now: answer questions about the hive's issues, PRs, agents and work from it directly, with its numbers, and set needs_research to false.\n") } diff --git a/src/pkg/meetings/betty/queue.go b/src/pkg/meetings/betty/queue.go index dfc68f2dcb..84ccdb9709 100644 --- a/src/pkg/meetings/betty/queue.go +++ b/src/pkg/meetings/betty/queue.go @@ -399,7 +399,8 @@ func (e *Engine) onAnswer(p *pending, a answered) { e.finish(p, reason) return } - if e.emit(Reply{Text: text, Trigger: TriggerQuestion, Question: p.line.Text, NeedsResearch: a.ans.NeedsResearch, Sources: a.ans.Sources}) { + said := citeSources(e.s.Persona.Language, text, a.ans.Sources, p.snips) + if e.emit(Reply{Text: said, Trigger: TriggerQuestion, Question: p.line.Text, NeedsResearch: a.ans.NeedsResearch, Sources: a.ans.Sources}) { e.posts = append(e.posts, now) e.lastPost = now e.total++ @@ -461,12 +462,30 @@ func (e *Engine) onAddressedAnswer(p *pending, a answered) { rep.NeedsResearch = true rep.Research = clip(researchQuery(p), maxResearchQuery) } + if a.err == nil && rep.Research == "" && text != "" { + rep.Text = e.vouch(p, a.ans, rep.Text, live) + } e.emit(rep) e.rememberTopic(p.topic) e.rememberTopic(TopicKey(text)) e.finish(p, "") } +// vouch is an addressed model answer as said (#18590 accuracy): a +// low-confidence answer that is not from the hive's live state is marked +// unsure rather than stated as fact, and a chat answer names the sources +// it used. +func (e *Engine) vouch(p *pending, ans Answer, text string, live bool) string { + lang := e.s.Persona.Language + if !live && ans.Confidence < e.s.MinConfidence { + text = hedge(lang, text) + } + if !p.trig.VoiceWanted { + text = citeSources(lang, text, ans.Sources, p.snips) + } + return text +} + // maxResearchQuery bounds a deep dive's query. const maxResearchQuery = 300