From 0303ff6a0f9b853babf0f6ad787e85b9b9d64acb Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Wed, 18 Nov 2026 14:36:17 -0500 Subject: [PATCH 1/5] =?UTF-8?q?=F0=9F=90=9B=20Filter=20em-dashes=20on=20ev?= =?UTF-8?q?ery=20outbound=20path,=20agent=20proxy=20writes=20included?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The #18970 filter covered hive's own GitHub writes but not text agents send themselves through the egress proxy, so a gh api comment kept its em-dashes. Move the one shared filter into the crux module so the proxy uses it, rewrite JSON text fields of allowed agent writes (REST and the comment, review, PR and issue GraphQL mutations), rewrite the HEAD commit message in the push broker, and route notifications, Discord, meeting chat and speech, hub posts, Linear, Jira and DIBS writes through it. A new inventory test fails CI when an outbound HTTP send is added without the filter or a stated reason. Fixes #19103 Signed-off-by: Andy Anderson --- .../fixed-19103-agent-writes-no-em-dash.md | 1 + src/crux/prosefilter/json.go | 154 +++++++ src/crux/prosefilter/json_test.go | 68 +++ src/crux/prosefilter/prosefilter.go | 217 +++++++++ src/crux/prosefilter/prosefilter_test.go | 62 +++ src/crux/proxy/github_proxy.go | 6 + src/crux/proxy/github_proxy_mitm.go | 8 + src/crux/proxy/outbound_prose.go | 196 ++++++++ src/crux/proxy/outbound_prose_test.go | 253 ++++++++++ src/crux/pushbroker/commit_message.go | 82 ++++ src/crux/pushbroker/commit_message_test.go | 113 +++++ src/crux/pushbroker/pushbroker.go | 13 +- src/docs/documentation-map.md | 1 + src/docs/env-vars.md | 1 + src/docs/github-write-surface.md | 5 + src/docs/outbound-prose.md | 119 +++++ src/pkg/dashboard/api_meetings_persona.go | 2 + src/pkg/discord/bot.go | 15 + src/pkg/discord/bot_prose_test.go | 38 ++ src/pkg/github/pullrequest.go | 2 + src/pkg/hub/auth_audit.go | 3 + src/pkg/hub/discord_request_notify.go | 11 + src/pkg/hub/feedback.go | 2 + src/pkg/hub/github_activity.go | 3 + src/pkg/hub/selfreport_relay.go | 2 + src/pkg/hub/slack.go | 3 + src/pkg/inbox/dibs.go | 6 +- src/pkg/linearagent/client.go | 7 + src/pkg/meetings/client.go | 9 + src/pkg/meetings/client_prose_test.go | 42 ++ src/pkg/notify/notify.go | 4 + src/pkg/prosefilter/emit_guard_test.go | 4 + src/pkg/prosefilter/outbound_sites_test.go | 432 ++++++++++++++++++ src/pkg/prosefilter/prosefilter.go | 235 ++-------- src/pkg/prosefilter/prosefilter_test.go | 64 +-- .../testdata/emit_dash_baseline.json | 6 + src/pkg/worksource/jira.go | 11 + src/pkg/worksource/linear.go | 7 + 38 files changed, 1950 insertions(+), 257 deletions(-) create mode 100644 changelog.d/fixed-19103-agent-writes-no-em-dash.md create mode 100644 src/crux/prosefilter/json.go create mode 100644 src/crux/prosefilter/json_test.go create mode 100644 src/crux/prosefilter/prosefilter.go create mode 100644 src/crux/prosefilter/prosefilter_test.go create mode 100644 src/crux/proxy/outbound_prose.go create mode 100644 src/crux/proxy/outbound_prose_test.go create mode 100644 src/crux/pushbroker/commit_message.go create mode 100644 src/crux/pushbroker/commit_message_test.go create mode 100644 src/docs/outbound-prose.md create mode 100644 src/pkg/discord/bot_prose_test.go create mode 100644 src/pkg/meetings/client_prose_test.go create mode 100644 src/pkg/prosefilter/outbound_sites_test.go diff --git a/changelog.d/fixed-19103-agent-writes-no-em-dash.md b/changelog.d/fixed-19103-agent-writes-no-em-dash.md new file mode 100644 index 0000000000..bf065fc955 --- /dev/null +++ b/changelog.d/fixed-19103-agent-writes-no-em-dash.md @@ -0,0 +1 @@ +- Em-dashes no longer reach GitHub or any other outbound channel through paths the first filter missed ([#19103](https://github.com/hivecommons/hive/issues/19103)). The shared dash filter moved into the `crux` module so the GitHub egress proxy now applies it to agent writes it relays (`gh api` comments, review replies, PR and issue titles and bodies, contents-API commit messages, and the text inputs of comment, review, PR and issue GraphQL mutations, plus Linear mutation variables), touching only JSON text fields and never git traffic or non-JSON bodies. The push broker rewrites the agent's HEAD commit message, and notifications (ntfy, Slack, Discord), Discord bot messages, meeting chat and spoken text, hub-filed issues and comments, Linear, Jira and DIBS writes, and squash commit messages hive writes all pass through the same filter. A new CI inventory test fails when an outbound HTTP send is added without the filter or a stated reason. Set `HIVE_AGENT_PROSE_FILTER=false` to stop rewriting agent-authored text. diff --git a/src/crux/prosefilter/json.go b/src/crux/prosefilter/json.go new file mode 100644 index 0000000000..1517f8ffcd --- /dev/null +++ b/src/crux/prosefilter/json.go @@ -0,0 +1,154 @@ +package prosefilter + +import ( + "bytes" + "encoding/json" + "os" + "strings" +) + +// Kind says how a JSON field's text is filtered. +type Kind int + +const ( + // Prose is multi-line Markdown text: Sanitize. + Prose Kind = iota + 1 + // Title is single-line text: SanitizeTitle. + Title +) + +// Keys maps a JSON object key to the filter its string values get. +type Keys map[string]Kind + +// Stats counts what SanitizeJSON changed: text fields rewritten, and dashes +// removed. It never carries the text itself, so it is safe to log. +type Stats struct { + Fields int + Dashes int +} + +// AgentWritesEnv is the operator switch for rewriting text that an agent +// wrote itself (its GitHub writes through the egress proxy, its HEAD commit +// message in the push broker). It is on unless set to one of +// agentWritesOffValues. Text hive itself composes is always filtered. +const AgentWritesEnv = "HIVE_AGENT_PROSE_FILTER" + +var agentWritesOffValues = map[string]bool{"false": true, "0": true, "off": true, "no": true} + +// AgentWritesEnabled reports whether AgentWritesEnv leaves the agent-write +// rewrite on. +func AgentWritesEnabled() bool { + return !agentWritesOffValues[strings.ToLower(strings.TrimSpace(os.Getenv(AgentWritesEnv)))] +} + +// SanitizeJSON rewrites, at any depth of a JSON document, the string values +// (and the strings of a string array) stored under a key in keys, using the +// same rule as Sanitize. Every other value, number formatting included, is +// preserved. A body that is not JSON, holds no dash, or has nothing to change +// is returned unchanged with zero Stats; only a changed body is re-encoded +// (keys sorted, no HTML escaping). +func SanitizeJSON(body []byte, keys Keys) ([]byte, Stats) { + if len(keys) == 0 || !mayHoldDash(body) { + return body, Stats{} + } + dec := json.NewDecoder(bytes.NewReader(body)) + dec.UseNumber() + var doc any + if dec.Decode(&doc) != nil || dec.More() { + return body, Stats{} + } + var st Stats + doc = sanitizeTree(doc, keys, &st) + if st.Fields == 0 { + return body, Stats{} + } + out, err := marshalNoEscape(doc) + if err != nil { + return body, Stats{} + } + return out, st +} + +// mayHoldDash reports whether body could contain an em- or en-dash, raw or as +// a JSON \u escape. +func mayHoldDash(body []byte) bool { + if bytes.Contains(body, []byte(emDash)) || bytes.Contains(body, []byte(enDash)) { + return true + } + lower := bytes.ToLower(body) + return bytes.Contains(lower, []byte(`—`)) || bytes.Contains(lower, []byte(`–`)) +} + +func sanitizeTree(v any, keys Keys, st *Stats) any { + switch t := v.(type) { + case map[string]any: + for k, child := range t { + if kind, ok := keys[k]; ok { + switch c := child.(type) { + case string: + t[k] = sanitizeCounted(c, kind, st) + continue + case []any: + if allStrings(c) { + for i, s := range c { + c[i] = sanitizeCounted(s.(string), kind, st) + } + continue + } + } + } + t[k] = sanitizeTree(child, keys, st) + } + return t + case []any: + for i, child := range t { + t[i] = sanitizeTree(child, keys, st) + } + return t + default: + return v + } +} + +func allStrings(items []any) bool { + for _, it := range items { + if _, ok := it.(string); !ok { + return false + } + } + return len(items) > 0 +} + +func sanitizeCounted(s string, kind Kind, st *Stats) string { + if !HasDash(s) { + return s + } + var clean string + if kind == Title { + clean = SanitizeTitle(s) + } else { + clean = Sanitize(s) + } + if clean != s { + st.Fields++ + st.Dashes += DashCount(s) - DashCount(clean) + } + return clean +} + +// DashCount is the number of em-dashes and en-dashes in s. +func DashCount(s string) int { + return strings.Count(s, emDash) + strings.Count(s, enDash) +} + +// marshalNoEscape is json.Marshal without HTML escaping, so a body that only +// had its dashes rewritten does not also gain < escapes. +func marshalNoEscape(v any) ([]byte, error) { + var buf bytes.Buffer + enc := json.NewEncoder(&buf) + enc.SetEscapeHTML(false) + if err := enc.Encode(v); err != nil { + return nil, err + } + return bytes.TrimRight(buf.Bytes(), "\n"), nil +} diff --git a/src/crux/prosefilter/json_test.go b/src/crux/prosefilter/json_test.go new file mode 100644 index 0000000000..c6e3c1ae54 --- /dev/null +++ b/src/crux/prosefilter/json_test.go @@ -0,0 +1,68 @@ +package prosefilter + +import ( + "encoding/json" + "testing" +) + +func TestSanitizeJSON(t *testing.T) { + keys := Keys{"body": Prose, "title": Title, "texts": Prose} + cases := []struct { + name, in, want string + fields int + }{ + {"top-level body", `{"body":"a ` + em + ` b"}`, `{"body":"a - b"}`, 1}, + {"escaped dash", `{"body":"a — b"}`, `{"body":"a - b"}`, 1}, + {"title", `{"title":"x ` + em + ` y"}`, `{"title":"x - y"}`, 1}, + {"nested", `{"input":{"body":"a ` + em + ` b","id":"T_1"}}`, `{"input":{"body":"a - b","id":"T_1"}}`, 1}, + {"array items", `{"comments":[{"body":"a ` + em + ` b","line":3}]}`, `{"comments":[{"body":"a - b","line":3}]}`, 1}, + {"string array under prose key", `{"texts":["a ` + em + ` b","c"]}`, `{"texts":["a - b","c"]}`, 1}, + {"numbers kept exactly", `{"body":"a ` + em + ` b","n":12345678901234567890,"f":1.50}`, `{"body":"a - b","f":1.50,"n":12345678901234567890}`, 1}, + {"no HTML escaping", `{"body":" & ` + em + `"}`, `{"body":" & -"}`, 1}, + {"other key untouched", `{"ref":"a ` + em + ` b"}`, `{"ref":"a ` + em + ` b"}`, 0}, + {"non-string under key untouched", `{"body":{"ref":"a ` + em + ` b"}}`, `{"body":{"ref":"a ` + em + ` b"}}`, 0}, + {"no dash", `{"body":"plain"}`, `{"body":"plain"}`, 0}, + {"not JSON", `plain ` + em + ` text`, `plain ` + em + ` text`, 0}, + {"trailing data", `{"body":"a ` + em + ` b"} {}`, `{"body":"a ` + em + ` b"} {}`, 0}, + {"code span kept", `{"body":"` + "`a " + em + " b`" + `"}`, `{"body":"` + "`a " + em + " b`" + `"}`, 0}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, st := SanitizeJSON([]byte(tc.in), keys) + if string(got) != tc.want { + t.Fatalf("SanitizeJSON = %s, want %s", got, tc.want) + } + if st.Fields != tc.fields { + t.Fatalf("Fields = %d, want %d", st.Fields, tc.fields) + } + if tc.fields > 0 && st.Dashes == 0 { + t.Fatal("Dashes not counted") + } + if tc.fields > 0 && !json.Valid(got) { + t.Fatalf("output is not JSON: %s", got) + } + }) + } +} + +func TestSanitizeJSONNoKeys(t *testing.T) { + in := []byte(`{"body":"a ` + em + ` b"}`) + if got, st := SanitizeJSON(in, nil); string(got) != string(in) || st.Fields != 0 { + t.Fatalf("nil keys changed the body: %s", got) + } +} + +func TestDashCount(t *testing.T) { + if got := DashCount("a " + em + " b " + en + " c"); got != 2 { + t.Fatalf("DashCount = %d, want 2", got) + } +} + +func TestAgentWritesEnabled(t *testing.T) { + for v, want := range map[string]bool{"": true, "true": true, "false": false, "0": false, "OFF": false, "no": false} { + t.Setenv(AgentWritesEnv, v) + if got := AgentWritesEnabled(); got != want { + t.Errorf("%s=%q: AgentWritesEnabled = %v, want %v", AgentWritesEnv, v, got, want) + } + } +} diff --git a/src/crux/prosefilter/prosefilter.go b/src/crux/prosefilter/prosefilter.go new file mode 100644 index 0000000000..9d9ce0faae --- /dev/null +++ b/src/crux/prosefilter/prosefilter.go @@ -0,0 +1,217 @@ +// Package prosefilter is the one place hive rewrites dashes in text that +// leaves the hive (PR and issue titles and bodies, comments, review bodies, +// commit messages, notifications, meeting chat and spoken text). +// +// The owner's rule is that em-dashes are never acceptable in anything hive or +// its agents write. Agents are told so in the shared base prompt, but a model +// can still emit one, so every outward write path also passes its prose +// through Sanitize before it is sent: hive's own sends in the root module +// (via src/pkg/prosefilter, which delegates here) and agent-originated GitHub +// writes in the egress proxy (src/crux/proxy/outbound_prose.go). It lives in +// the crux module so both sides call the same function (hivecommons/hive#19103). +// +// Sanitize rewrites only prose. Fenced code blocks, inline code spans and +// Markdown blockquote lines are left byte-for-byte alone, because they carry +// quoted or pasted content (a diff, a log line, a file excerpt) that must not +// be altered. +package prosefilter + +import ( + "regexp" + "strings" +) + +const ( + // EmDash and EnDash are the two characters the filter rewrites. + EmDash = "\u2014" + EnDash = "\u2013" + + emDash = EmDash + enDash = EnDash + + // plainDash is what a dash used as punctuation becomes. + plainDash = "-" + // spacedDash is the replacement between two words. + spacedDash = " - " + + // maxFenceIndent is the Markdown limit on leading spaces for a code fence. + maxFenceIndent = 3 + // minFenceRun is the shortest run of fence characters that opens a fence. + minFenceRun = 3 +) + +var ( + // emDashRE matches an em-dash with the horizontal whitespace around it. + emDashRE = regexp.MustCompile(`[ \t]*` + emDash + `[ \t]*`) + // spacedEnDashRE matches an en-dash used as a dash (whitespace on both + // sides). An en-dash in a range such as 3-5 has no spaces and is left alone + // here; a hard-coded range is fixed at its source. + spacedEnDashRE = regexp.MustCompile(`[ \t]+` + enDash + `[ \t]+`) +) + +// HasDash reports whether s contains an em-dash or an en-dash. +func HasDash(s string) bool { + return strings.Contains(s, emDash) || strings.Contains(s, enDash) +} + +// Sanitize replaces every em-dash (U+2014), and every en-dash (U+2013) used +// as a spaced dash, in the prose of s with a plain hyphen. Text inside fenced +// code blocks, inline code spans and blockquote lines is untouched. A string +// with no such dash is returned unchanged. +func Sanitize(s string) string { + if !HasDash(s) { + return s + } + lines := strings.Split(s, "\n") + var fenceChar byte + var fenceLen int + for i, line := range lines { + if fenceChar != 0 { + if closesFence(line, fenceChar, fenceLen) { + fenceChar, fenceLen = 0, 0 + } + continue + } + if c, n := opensFence(line); c != 0 { + fenceChar, fenceLen = c, n + continue + } + if isBlockquote(line) { + continue + } + lines[i] = sanitizeLine(line) + } + return strings.Join(lines, "\n") +} + +// SanitizeTitle is Sanitize for a single-line title. A title has no code +// fence or blockquote, so only inline code spans are protected. +func SanitizeTitle(s string) string { + if !HasDash(s) { + return s + } + return sanitizeLine(s) +} + +func opensFence(line string) (byte, int) { + trimmed := strings.TrimLeft(line, " ") + if len(line)-len(trimmed) > maxFenceIndent || trimmed == "" { + return 0, 0 + } + c := trimmed[0] + if c != '`' && c != '~' { + return 0, 0 + } + n := runLen(trimmed, 0, c) + if n < minFenceRun { + return 0, 0 + } + // A backtick fence's info string may not contain a backtick; such a line + // is an inline code span, not a fence. + if c == '`' && strings.Contains(trimmed[n:], "`") { + return 0, 0 + } + return c, n +} + +func closesFence(line string, c byte, openLen int) bool { + trimmed := strings.TrimLeft(line, " ") + if len(line)-len(trimmed) > maxFenceIndent || trimmed == "" || trimmed[0] != c { + return false + } + n := runLen(trimmed, 0, c) + return n >= openLen && strings.TrimSpace(trimmed[n:]) == "" +} + +func isBlockquote(line string) bool { + return strings.HasPrefix(strings.TrimLeft(line, " "), ">") +} + +func runLen(s string, from int, c byte) int { + n := 0 + for from+n < len(s) && s[from+n] == c { + n++ + } + return n +} + +// sanitizeLine rewrites one prose line, skipping inline code spans. +func sanitizeLine(line string) string { + if !HasDash(line) { + return line + } + indentLen := len(line) - len(strings.TrimLeft(line, " \t")) + indent, rest := line[:indentLen], line[indentLen:] + + var b strings.Builder + b.WriteString(indent) + segStart := 0 + flush := func(end int) { + if end > segStart { + b.WriteString(rewriteSegment(rest[segStart:end], segStart == 0, end == len(rest))) + } + } + for i := 0; i < len(rest); { + if rest[i] != '`' { + i++ + continue + } + n := runLen(rest, i, '`') + closeAt := findClosingRun(rest, i+n, n) + if closeAt < 0 { + i += n + continue + } + flush(i) + spanEnd := closeAt + n + b.WriteString(rest[i:spanEnd]) + segStart = spanEnd + i = spanEnd + } + flush(len(rest)) + return b.String() +} + +// findClosingRun returns the index of the next run of exactly n backticks at +// or after from, or -1. +func findClosingRun(s string, from, n int) int { + for i := from; i < len(s); { + if s[i] != '`' { + i++ + continue + } + m := runLen(s, i, '`') + if m == n { + return i + } + i += m + } + return -1 +} + +// rewriteSegment replaces dashes in one non-code stretch of a line. atStart +// and atEnd say whether the segment touches the start or end of the line's +// prose, so a leading or trailing dash does not leave a stray space. +func rewriteSegment(seg string, atStart, atEnd bool) string { + seg = spacedEnDashRE.ReplaceAllString(seg, spacedDash) + locs := emDashRE.FindAllStringIndex(seg, -1) + if len(locs) == 0 { + return seg + } + var b strings.Builder + prev := 0 + for _, loc := range locs { + b.WriteString(seg[prev:loc[0]]) + switch { + case loc[0] == 0 && atStart: + b.WriteString(plainDash + " ") + case loc[1] == len(seg) && atEnd: + b.WriteString(" " + plainDash) + default: + b.WriteString(spacedDash) + } + prev = loc[1] + } + b.WriteString(seg[prev:]) + return b.String() +} diff --git a/src/crux/prosefilter/prosefilter_test.go b/src/crux/prosefilter/prosefilter_test.go new file mode 100644 index 0000000000..3932825796 --- /dev/null +++ b/src/crux/prosefilter/prosefilter_test.go @@ -0,0 +1,62 @@ +package prosefilter + +import "testing" + +const ( + em = "\u2014" + en = "\u2013" +) + +func TestSanitize(t *testing.T) { + cases := []struct { + name, in, want string + }{ + {"no dash is unchanged", "plain - text", "plain - text"}, + {"spaced em-dash", "a " + em + " b", "a - b"}, + {"unspaced em-dash", "a" + em + "b", "a - b"}, + {"several em-dashes", "a " + em + " b " + em + " c", "a - b - c"}, + {"leading em-dash", em + " hive: agent=x", "- hive: agent=x"}, + {"leading em-dash keeps indent", " " + em + " note", " - note"}, + {"trailing em-dash", "wait " + em, "wait -"}, + {"spaced en-dash", "a " + en + " b", "a - b"}, + {"en-dash range untouched", "levels 3" + en + "5", "levels 3" + en + "5"}, + {"inline code untouched", "see `a " + em + " b` and c " + em + " d", "see `a " + em + " b` and c - d"}, + {"double-backtick span untouched", "``a ` " + em + " b`` " + em + " c", "``a ` " + em + " b`` - c"}, + {"unclosed backtick is prose", "a ` b " + em + " c", "a ` b - c"}, + {"fenced block untouched", "x " + em + " y\n```\nfile " + em + " list\n```\nz " + em + " w", "x - y\n```\nfile " + em + " list\n```\nz - w"}, + {"tilde fence untouched", "~~~go\na " + em + " b\n~~~\nc " + em + " d", "~~~go\na " + em + " b\n~~~\nc - d"}, + {"longer closing fence", "````\na " + em + " b\n`````\nc " + em + " d", "````\na " + em + " b\n`````\nc - d"}, + {"short closing fence does not close", "````\na\n```\nb " + em + " c\n````", "````\na\n```\nb " + em + " c\n````"}, + {"blockquote untouched", "> quoted " + em + " text\nmine " + em + " here", "> quoted " + em + " text\nmine - here"}, + {"markdown list", "- file.go " + em + " added", "- file.go - added"}, + {"CRLF preserved", "a " + em + " b\r\nc", "a - b\r\nc"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := Sanitize(tc.in); got != tc.want { + t.Fatalf("Sanitize(%q) = %q, want %q", tc.in, got, tc.want) + } + if got := Sanitize(Sanitize(tc.in)); got != tc.want { + t.Fatalf("Sanitize is not idempotent for %q: %q", tc.in, got) + } + }) + } +} + +func TestSanitizeTitle(t *testing.T) { + if got, want := SanitizeTitle("fix: a "+em+" b `c "+em+" d`"), "fix: a - b `c "+em+" d`"; got != want { + t.Fatalf("SanitizeTitle = %q, want %q", got, want) + } + if got := SanitizeTitle("fix: plain"); got != "fix: plain" { + t.Fatalf("clean title changed: %q", got) + } +} + +func TestHasDash(t *testing.T) { + if HasDash("a - b") { + t.Fatal("hyphen reported as dash") + } + if !HasDash("a "+em+" b") || !HasDash("a "+en+" b") { + t.Fatal("em or en dash not detected") + } +} diff --git a/src/crux/proxy/github_proxy.go b/src/crux/proxy/github_proxy.go index 5875dedfc4..61a61bf438 100644 --- a/src/crux/proxy/github_proxy.go +++ b/src/crux/proxy/github_proxy.go @@ -20,6 +20,7 @@ import ( "github.com/hivecommons/hive/crux/config" "github.com/hivecommons/hive/crux/egressmark" "github.com/hivecommons/hive/crux/inferencehealth" + "github.com/hivecommons/hive/crux/prosefilter" "github.com/hivecommons/hive/crux/upstreamproxy" ) @@ -85,6 +86,10 @@ type GitHubProxy struct { caCert tls.Certificate caX509 *x509.Certificate logger *slog.Logger + // proseFilterOff turns off the agent-write dash filter + // (outbound_prose.go). Zero means on, so every proxy, test literals + // included, filters unless the operator set HIVE_AGENT_PROSE_FILTER=false. + proseFilterOff bool // uidMap is read through loadedUIDMap and swapped only by refreshUIDMap // under uidMapMu (#18184): agents added at runtime land in uid-map.json // after construction. uidMapPath/uidMapInfo are what the refresh compares @@ -343,6 +348,7 @@ func newGitHubProxyWithCA(caCert tls.Certificate, caX509 *x509.Certificate, logg proxyAdvisoryOK: advisoryOK, allowPrivateForward: allowPrivateForward, injectGHAuth: injectGHAuth, + proseFilterOff: !prosefilter.AgentWritesEnabled(), allowedRepos: allowed, violations: make(map[string]int), certCache: make(map[string]cachedCert), diff --git a/src/crux/proxy/github_proxy_mitm.go b/src/crux/proxy/github_proxy_mitm.go index c170eb5180..c154a0bf39 100644 --- a/src/crux/proxy/github_proxy_mitm.go +++ b/src/crux/proxy/github_proxy_mitm.go @@ -359,6 +359,14 @@ func (p *GitHubProxy) proxyHTTPHost(client net.Conn, upstream net.Conn, host str if isLinear && agentName != internalCallerName { p.injectLinearCredential(req, agentName, mode) } + // #19103: an allowed agent write's text fields get the same dash + // rewrite hive's own writes get, after every gate has read the body + // as the agent sent it. Git smart-HTTP traffic is never touched. + if err := p.sanitizeAgentWriteProse(req, agentName, isLinear); err != nil { + _ = client.SetReadDeadline(time.Time{}) + p.logTimeout("proxy prose-filter body read timed out", err, "agent", agentName, "path", req.URL.Path) + return + } // Git smart HTTP uses chunked streaming that http.ReadResponse // can't handle reliably. After the ACMM check passes, forward // the request and switch to raw bidirectional streaming. diff --git a/src/crux/proxy/outbound_prose.go b/src/crux/proxy/outbound_prose.go new file mode 100644 index 0000000000..5af65ee246 --- /dev/null +++ b/src/crux/proxy/outbound_prose.go @@ -0,0 +1,196 @@ +package proxy + +import ( + "bytes" + "encoding/json" + "io" + "net/http" + "regexp" + "sort" + "strconv" + "strings" + + "github.com/hivecommons/hive/crux/prosefilter" +) + +// Outbound prose filter for agent-originated writes (hivecommons/hive#19103). +// +// Hive's own forge writes pass their text through prosefilter before they are +// sent (#18970), but an agent that posts with `gh api`, `gh pr comment` or +// `gh pr create` talks to GitHub through this proxy, and the relay forwarded +// those bodies verbatim. This is the boundary for that traffic: an allowed +// agent write's text fields get the same rewrite hive's own writes get, and +// nothing else in the request changes. +// +// Scope, deliberately narrow: +// +// - Only JSON bodies. A body that is not a JSON document, or that declares a +// non-JSON Content-Type (a release asset upload, a form), is forwarded +// byte-for-byte. +// - Only the text fields: REST `body`, `title` and `message` at any depth +// (a review's `comments[].body` included); GraphQL `body`/`title` (and +// Linear `description`) values inside the variables of a mutation +// document. Every other field, and the GraphQL query text itself, is left +// alone. A literal string inlined in the query text is not rewritten. +// - Never git smart-HTTP or LFS traffic: commit messages inside pushed +// objects cannot be rewritten in transit without changing object ids. +// The push broker covers the HEAD commit message instead +// (src/crux/pushbroker/commit_message.go). +// - Never the hive's own control-plane traffic, whose text is already +// filtered (or deliberately left as rendered) before it reaches here. +// +// The switch is prosefilter.AgentWritesEnv (default on). + +// outboundProseBodyLimit bounds the body the filter buffers. A larger body is +// forwarded unchanged rather than truncated: GitHub caps a comment body at +// 65536 characters, so a legitimate text write is far below this. +const outboundProseBodyLimit = 1 << 20 + +// restProseKeys are the REST JSON fields that carry prose. `message` is the +// commit message of the contents and git-data APIs. +var restProseKeys = prosefilter.Keys{"body": prosefilter.Prose, "title": prosefilter.Title, "message": prosefilter.Prose} + +// graphQLProseKeys are the variable keys whose values are prose in a GitHub +// mutation. +var graphQLProseKeys = prosefilter.Keys{"body": prosefilter.Prose, "title": prosefilter.Title} + +// linearProseKeys are the same for Linear, where an issue's text is +// `description`. +var linearProseKeys = prosefilter.Keys{"body": prosefilter.Prose, "title": prosefilter.Title, "description": prosefilter.Prose} + +// graphQLProseMutations are the GitHub mutations whose inputs carry comment, +// review, PR or issue text. A mutation document selecting at least one of +// them has its `body` and `title` variables filtered. +var graphQLProseMutations = map[string]bool{ + "addComment": true, + "addDiscussionComment": true, + "addPullRequestReview": true, + "addPullRequestReviewComment": true, + "addPullRequestReviewThread": true, + "addPullRequestReviewThreadReply": true, + "submitPullRequestReview": true, + "updateIssueComment": true, + "updatePullRequestReviewComment": true, + "createPullRequest": true, + "updatePullRequest": true, + "createIssue": true, + "updateIssue": true, +} + +// sanitizeAgentWriteProse applies the shared prose filter to an allowed agent +// write just before it is forwarded. It replaces req.Body (and the length) +// only when a text field changed. A non-nil error means the body could not be +// read; the caller drops the exchange, as the other body gates do. +func (p *GitHubProxy) sanitizeAgentWriteProse(req *http.Request, agentName string, isLinear bool) error { + if p.proseFilterOff || agentName == internalCallerName || req.Body == nil || req.Body == http.NoBody { + return nil + } + if !proseFilterApplies(req) { + return nil + } + body, err := io.ReadAll(io.LimitReader(req.Body, outboundProseBodyLimit+1)) + if err != nil { + _ = req.Body.Close() + return err + } + if int64(len(body)) > outboundProseBodyLimit { + req.Body = bodyReadCloser{Reader: io.MultiReader(bytes.NewReader(body), req.Body), Closer: req.Body} + return nil + } + _ = req.Body.Close() + out, st := sanitizeProseBody(req.URL.Path, body, isLinear) + req.Body = io.NopCloser(bytes.NewReader(out)) + if st.Fields == 0 { + return nil + } + req.ContentLength = int64(len(out)) + req.Header.Set("Content-Length", strconv.Itoa(len(out))) + // A chunked request is now a fixed-length one. + req.TransferEncoding = nil + if p.logger != nil { + p.logger.Debug("proxy: rewrote dashes in agent write text", "agent", agentName, + "method", req.Method, "path", req.URL.Path, "fields", st.Fields, "dashes", st.Dashes) + } + return nil +} + +// proseFilterApplies is the cheap pre-check: a write method, not git or LFS +// traffic, and a Content-Type that is JSON or absent. +func proseFilterApplies(req *http.Request) bool { + switch req.Method { + case http.MethodPost, http.MethodPatch, http.MethodPut: + default: + return false + } + if isGitPath(req.URL.Path) || strings.Contains(req.URL.Path, ".git/") { + return false + } + ct := strings.ToLower(req.Header.Get("Content-Type")) + return ct == "" || strings.Contains(ct, "json") +} + +// sanitizeProseBody returns body with its text fields filtered, and what +// changed. A body it does not recognise is returned as is. +func sanitizeProseBody(path string, body []byte, isLinear bool) ([]byte, prosefilter.Stats) { + if isLinear || IsGraphQLPath(path) { + return sanitizeGraphQLProse(body, isLinear) + } + return prosefilter.SanitizeJSON(body, restProseKeys) +} + +// sanitizeGraphQLProse filters only the variables of a mutation document, +// never its query text. +func sanitizeGraphQLProse(body []byte, isLinear bool) ([]byte, prosefilter.Stats) { + var obj map[string]json.RawMessage + if err := json.Unmarshal(body, &obj); err != nil || obj == nil { + return body, prosefilter.Stats{} + } + var query string + if json.Unmarshal(obj["query"], &query) != nil || !isGitHubGraphQLMutation(query) { + return body, prosefilter.Stats{} + } + keys := linearProseKeys + if !isLinear { + if !selectsProseMutation(query) { + return body, prosefilter.Stats{} + } + keys = graphQLProseKeys + } + rawVars, ok := obj["variables"] + if !ok { + return body, prosefilter.Stats{} + } + vars, st := prosefilter.SanitizeJSON(rawVars, keys) + if st.Fields == 0 { + return body, prosefilter.Stats{} + } + obj["variables"] = vars + var buf bytes.Buffer + enc := json.NewEncoder(&buf) + enc.SetEscapeHTML(false) + if err := enc.Encode(obj); err != nil { + return body, prosefilter.Stats{} + } + return bytes.TrimRight(buf.Bytes(), "\n"), st +} + +// graphQLProseMutationRe matches any name in graphQLProseMutations as a whole +// word. A substring test is the right posture here, unlike for a permission +// grant: a batched document that also does something else still carries the +// text, and rewriting a dash is never a widening. +var graphQLProseMutationRe = regexp.MustCompile(`\b(` + strings.Join(sortedKeys(graphQLProseMutations), "|") + `)\b`) + +// selectsProseMutation reports whether a GitHub mutation document names a +// mutation in graphQLProseMutations. +func selectsProseMutation(query string) bool { + return graphQLProseMutationRe.MatchString(query) +} + +func sortedKeys(m map[string]bool) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out +} diff --git a/src/crux/proxy/outbound_prose_test.go b/src/crux/proxy/outbound_prose_test.go new file mode 100644 index 0000000000..104f6dcf37 --- /dev/null +++ b/src/crux/proxy/outbound_prose_test.go @@ -0,0 +1,253 @@ +package proxy + +import ( + "bufio" + "bytes" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "strconv" + "strings" + "testing" + + "github.com/hivecommons/hive/crux/agent" + "github.com/hivecommons/hive/crux/prosefilter" +) + +const ( + em = prosefilter.EmDash + en = prosefilter.EnDash +) + +func mustJSON(t *testing.T, v any) string { + t.Helper() + b, err := json.Marshal(v) + if err != nil { + t.Fatal(err) + } + return string(b) +} + +// decodeField returns the string at a dotted path of a JSON document. +func decodeField(t *testing.T, body []byte, path ...string) string { + t.Helper() + var cur any + if err := json.Unmarshal(body, &cur); err != nil { + t.Fatalf("body is not JSON: %v: %s", err, body) + } + for _, p := range path { + switch c := cur.(type) { + case map[string]any: + cur = c[p] + case []any: + i, _ := strconv.Atoi(p) + cur = c[i] + } + } + s, _ := cur.(string) + return s +} + +func TestSanitizeAgentWriteProse(t *testing.T) { + fence := "```\nfile " + em + " list\n```" + cases := []struct { + name string + method string + path string + contentType string + body string + linear bool + agent string + off bool + // field is the JSON path checked; want is its value after the filter. + field []string + want string + // unchanged means the forwarded body must equal the input byte-for-byte. + unchanged bool + }{ + {name: "REST issue comment", method: "POST", path: "/repos/o/r/issues/1/comments", + body: mustJSON(t, map[string]any{"body": "looks good " + em + " merging"}), field: []string{"body"}, want: "looks good - merging"}, + {name: "PR body edit", method: "PATCH", path: "/repos/o/r/pulls/2", + body: mustJSON(t, map[string]any{"body": "Fixes x " + em + " and y", "state": "open"}), field: []string{"body"}, want: "Fixes x - and y"}, + {name: "review thread reply", method: "POST", path: "/repos/o/r/pulls/2/comments/9/replies", + body: mustJSON(t, map[string]any{"body": "done " + em + " see abc123"}), field: []string{"body"}, want: "done - see abc123"}, + {name: "PR title", method: "POST", path: "/repos/o/r/pulls", + body: mustJSON(t, map[string]any{"title": "fix: a " + em + " b", "head": "x" + em + "y", "base": "main"}), field: []string{"title"}, want: "fix: a - b"}, + {name: "review with inline comments", method: "POST", path: "/repos/o/r/pulls/2/reviews", + body: mustJSON(t, map[string]any{"event": "COMMENT", "comments": []any{map[string]any{"path": "a.go", "line": 3, "body": "nit " + em + " rename"}}}), + field: []string{"comments", "0", "body"}, want: "nit - rename"}, + {name: "contents API commit message", method: "PUT", path: "/repos/o/r/contents/a.md", + body: mustJSON(t, map[string]any{"message": "docs: a " + em + " b", "content": "YQ=="}), field: []string{"message"}, want: "docs: a - b"}, + {name: "fenced code block kept", method: "POST", path: "/repos/o/r/issues/1/comments", + body: mustJSON(t, map[string]any{"body": "x " + em + " y\n" + fence}), field: []string{"body"}, want: "x - y\n" + fence}, + {name: "en-dash range kept", method: "POST", path: "/repos/o/r/issues/1/comments", + body: mustJSON(t, map[string]any{"body": "levels 3" + en + "5"}), unchanged: true}, + {name: "non-text field untouched", method: "POST", path: "/repos/o/r/git/refs", + body: mustJSON(t, map[string]any{"ref": "refs/heads/a" + em + "b", "sha": "abc"}), unchanged: true}, + {name: "non-JSON content type untouched", method: "POST", path: "/repos/o/r/releases/1/assets", + contentType: "application/octet-stream", body: "raw " + em + " bytes", unchanged: true}, + {name: "body that is not JSON untouched", method: "POST", path: "/repos/o/r/issues/1/comments", + body: "plain " + em + " text", unchanged: true}, + {name: "git push untouched", method: "POST", path: "/o/r.git/git-receive-pack", + contentType: "application/x-git-receive-pack-request", body: "0000 " + em, unchanged: true}, + {name: "LFS batch untouched", method: "POST", path: "/o/r.git/info/lfs/objects/batch", + body: mustJSON(t, map[string]any{"message": "a " + em + " b"}), unchanged: true}, + {name: "GET untouched", method: "GET", path: "/repos/o/r/issues", + body: mustJSON(t, map[string]any{"body": "a " + em + " b"}), unchanged: true}, + {name: "switch off", method: "POST", path: "/repos/o/r/issues/1/comments", off: true, + body: mustJSON(t, map[string]any{"body": "a " + em + " b"}), unchanged: true}, + {name: "hive control plane untouched", method: "POST", path: "/repos/o/r/issues/1/comments", agent: internalCallerName, + body: mustJSON(t, map[string]any{"body": "a " + em + " b"}), unchanged: true}, + {name: "GraphQL review thread reply", method: "POST", path: "/graphql", + body: mustJSON(t, map[string]any{ + "query": "mutation($input: AddPullRequestReviewThreadReplyInput!) { addPullRequestReviewThreadReply(input: $input) { comment { id } } }", + "variables": map[string]any{"input": map[string]any{"pullRequestReviewThreadId": "T_1", "body": "fixed " + em + " thanks"}}, + }), field: []string{"variables", "input", "body"}, want: "fixed - thanks"}, + {name: "GraphQL createPullRequest title", method: "POST", path: "/graphql", + body: mustJSON(t, map[string]any{ + "query": "mutation PullRequestCreate($input: CreatePullRequestInput!) { createPullRequest(input: $input) { pullRequest { url } } }", + "variables": map[string]any{"input": map[string]any{"title": "feat: x " + em + " y", "body": "b", "baseRefName": "main", "headRefName": "f"}}, + }), field: []string{"variables", "input", "title"}, want: "feat: x - y"}, + {name: "GraphQL query untouched", method: "POST", path: "/graphql", + body: mustJSON(t, map[string]any{"query": "query { viewer { login } }", "variables": map[string]any{"body": "a " + em + " b"}}), unchanged: true}, + {name: "GraphQL unrelated mutation untouched", method: "POST", path: "/graphql", + body: mustJSON(t, map[string]any{"query": "mutation($id: ID!) { addLabelsToLabelable(input: {labelableId: $id}) { clientMutationId } }", "variables": map[string]any{"title": "a " + em + " b"}}), unchanged: true}, + {name: "Linear issue description", method: "POST", path: "/graphql", linear: true, + body: mustJSON(t, map[string]any{ + "query": "mutation($input: IssueCreateInput!) { issueCreate(input: $input) { success } }", + "variables": map[string]any{"input": map[string]any{"title": "t", "description": "a " + em + " b"}}, + }), field: []string{"variables", "input", "description"}, want: "a - b"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + p := newTestProxy() + p.proseFilterOff = tc.off + agentName := tc.agent + if agentName == "" { + agentName = "quality" + } + req, err := http.NewRequest(tc.method, "https://api.github.com"+tc.path, strings.NewReader(tc.body)) + if err != nil { + t.Fatal(err) + } + if tc.contentType != "" { + req.Header.Set("Content-Type", tc.contentType) + } + if err := p.sanitizeAgentWriteProse(req, agentName, tc.linear); err != nil { + t.Fatalf("sanitizeAgentWriteProse: %v", err) + } + got, err := io.ReadAll(req.Body) + if err != nil { + t.Fatal(err) + } + if tc.unchanged { + if string(got) != tc.body { + t.Fatalf("body changed:\n got %s\nwant %s", got, tc.body) + } + return + } + if req.ContentLength != int64(len(got)) { + t.Errorf("ContentLength = %d, body is %d bytes", req.ContentLength, len(got)) + } + if h := req.Header.Get("Content-Length"); h != strconv.Itoa(len(got)) { + t.Errorf("Content-Length header = %q, body is %d bytes", h, len(got)) + } + if v := decodeField(t, got, tc.field...); v != tc.want { + t.Errorf("%v = %q, want %q", tc.field, v, tc.want) + } + }) + } +} + +// A body bigger than the buffer bound is forwarded whole and unchanged. +func TestSanitizeAgentWriteProseOversizedBodyForwardedIntact(t *testing.T) { + p := newTestProxy() + big := mustJSON(t, map[string]any{"body": em + strings.Repeat("x", outboundProseBodyLimit)}) + req, _ := http.NewRequest(http.MethodPost, "https://api.github.com/repos/o/r/issues/1/comments", strings.NewReader(big)) + if err := p.sanitizeAgentWriteProse(req, "quality", false); err != nil { + t.Fatal(err) + } + got, _ := io.ReadAll(req.Body) + if string(got) != big { + t.Fatal("oversized body was altered or truncated") + } +} + +// A read error is returned so the relay can drop the exchange. +func TestSanitizeAgentWriteProseReadError(t *testing.T) { + p := newTestProxy() + req, _ := http.NewRequest(http.MethodPost, "https://api.github.com/repos/o/r/issues/1/comments", io.NopCloser(errReader{})) + if err := p.sanitizeAgentWriteProse(req, "quality", false); err == nil { + t.Fatal("want a read error") + } +} + +type errReader struct{} + +func (errReader) Read([]byte) (int, error) { return 0, fmt.Errorf("boom") } + +// End to end through the MITM relay: what reaches GitHub has a plain hyphen +// and a Content-Length that matches the rewritten body. +func TestProxyHTTPRewritesAgentCommentDashes(t *testing.T) { + p := newTestProxy() + clientConn, proxyClient := net.Pipe() + upstreamConn, proxyUpstream := net.Pipe() + defer clientConn.Close() + defer upstreamConn.Close() + + go p.proxyHTTP(proxyClient, proxyUpstream, "quality", agent.ModeIssuesPRsMerge, agent.AgentCapabilities{}) + + body := mustJSON(t, map[string]any{"body": "Reviewed the change " + em + " it looks right.\n\n```\nkeep " + em + " this\n```"}) + go func() { + fmt.Fprintf(clientConn, "POST /repos/org/repo/issues/1/comments HTTP/1.1\r\nHost: api.github.com\r\nContent-Type: application/json\r\nContent-Length: %d\r\n\r\n%s", len(body), body) + }() + + type seen struct { + body []byte + cl int64 + } + got := make(chan seen, 1) + go func() { + req, err := http.ReadRequest(bufio.NewReader(upstreamConn)) + if err != nil { + got <- seen{} + return + } + b, _ := io.ReadAll(req.Body) + got <- seen{body: b, cl: req.ContentLength} + resp := &http.Response{StatusCode: http.StatusCreated, Proto: "HTTP/1.1", ProtoMajor: 1, ProtoMinor: 1, Header: make(http.Header), Body: http.NoBody, Request: req} + _ = resp.Write(upstreamConn) + }() + + resp, err := http.ReadResponse(bufio.NewReader(clientConn), nil) + if err != nil { + t.Fatalf("read response: %v", err) + } + if resp.StatusCode != http.StatusCreated { + t.Fatalf("status = %d, want 201", resp.StatusCode) + } + s := <-got + if s.cl != int64(len(s.body)) { + t.Errorf("upstream Content-Length = %d, body is %d bytes", s.cl, len(s.body)) + } + text := decodeField(t, s.body, "body") + if want := "Reviewed the change - it looks right.\n\n```\nkeep " + em + " this\n```"; text != want { + t.Errorf("forwarded body = %q, want %q", text, want) + } + if bytes.Contains(s.body, []byte("change "+em)) { + t.Error("prose em-dash reached upstream") + } +} + +func TestAgentWritesSwitchReadsEnv(t *testing.T) { + t.Setenv(prosefilter.AgentWritesEnv, "false") + if prosefilter.AgentWritesEnabled() { + t.Fatal("switch off not honoured") + } + t.Setenv(prosefilter.AgentWritesEnv, "") + if !prosefilter.AgentWritesEnabled() { + t.Fatal("switch must default on") + } +} diff --git a/src/crux/pushbroker/commit_message.go b/src/crux/pushbroker/commit_message.go new file mode 100644 index 0000000000..6a7f7dd630 --- /dev/null +++ b/src/crux/pushbroker/commit_message.go @@ -0,0 +1,82 @@ +package pushbroker + +import ( + "context" + "fmt" + "os" + "strings" + + "github.com/hivecommons/hive/crux/prosefilter" +) + +// sanitizeHeadCommitMessage applies the shared prose filter +// (hivecommons/hive#19103) to the outgoing HEAD commit's message and amends +// it when a dash was rewritten. It reports whether it amended HEAD. +// +// A commit message travels inside the pushed git objects, so the egress proxy +// cannot rewrite it in transit without changing object ids. The broker is +// the one place hive touches an agent's commits before they leave, and it +// already amends HEAD for the trailing-newline rule, so it is where the +// message is fixed. Only HEAD is rewritten: an older outgoing commit would +// need a rebase of the agent's branch, which the broker never does. Hive +// squash-merges, and the squash commit message hive authors is filtered, so +// what lands on the default branch is clean either way. +// +// The amend keeps the tree, the author and every trailer (Signed-off-by +// included); --cleanup=verbatim keeps the message byte-for-byte apart from +// the rewritten dashes. The switch is prosefilter.AgentWritesEnv. +func (b *Broker) sanitizeHeadCommitMessage(ctx context.Context) (amended bool, err error) { + if !prosefilter.AgentWritesEnabled() { + return false, nil + } + // cat-file gives the message exactly as stored; `git log --format=%B` + // appends a newline, which --cleanup=verbatim would then keep. + raw, err := b.git(ctx, "cat-file", "commit", "HEAD") + if err != nil { + return false, fmt.Errorf("reading HEAD commit message: %w", err) + } + _, msg, found := strings.Cut(string(raw), "\n\n") + if !found { + return false, nil + } + if !prosefilter.HasDash(msg) { + return false, nil + } + clean := sanitizeCommitMessage(msg) + if clean == msg { + return false, nil + } + f, err := os.CreateTemp("", "hive-commit-msg-*") + if err != nil { + return false, fmt.Errorf("staging commit message: %w", err) + } + defer func() { _ = os.Remove(f.Name()) }() + if _, err := f.WriteString(clean); err != nil { + _ = f.Close() + return false, fmt.Errorf("staging commit message: %w", err) + } + if err := f.Close(); err != nil { + return false, fmt.Errorf("staging commit message: %w", err) + } + // --only with no paths amends the message alone; the agent's staged + // changes stay staged and out of the commit. + if _, err := b.git(ctx, "commit", "--amend", "--only", "--cleanup=verbatim", "-F", f.Name()); err != nil { + return false, err + } + if b.Logger != nil { + b.Logger.Info("pushbroker rewrote dashes in the HEAD commit message", "repo", b.Repo, "branch", b.Branch, + "dashes", prosefilter.DashCount(msg)-prosefilter.DashCount(clean)) + } + return true, nil +} + +// sanitizeCommitMessage filters a commit message: the subject line as a +// title, the rest as prose. +func sanitizeCommitMessage(msg string) string { + subject, rest, hasRest := strings.Cut(msg, "\n") + subject = prosefilter.SanitizeTitle(subject) + if !hasRest { + return subject + } + return subject + "\n" + prosefilter.Sanitize(rest) +} diff --git a/src/crux/pushbroker/commit_message_test.go b/src/crux/pushbroker/commit_message_test.go new file mode 100644 index 0000000000..fd73bc26a7 --- /dev/null +++ b/src/crux/pushbroker/commit_message_test.go @@ -0,0 +1,113 @@ +package pushbroker + +import ( + "context" + "errors" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/hivecommons/hive/crux/prosefilter" +) + +const emDashForTest = prosefilter.EmDash + +// commitWithMessage commits a file with an exact commit message. +func commitWithMessage(t *testing.T, dir, msg string) { + t.Helper() + if err := os.WriteFile(filepath.Join(dir, "a.txt"), []byte("a\n"), 0o644); err != nil { + t.Fatal(err) + } + msgFile := filepath.Join(t.TempDir(), "msg") + if err := os.WriteFile(msgFile, []byte(msg), 0o644); err != nil { + t.Fatal(err) + } + runGit(t, dir, "add", "a.txt") + runGit(t, dir, "commit", "--cleanup=verbatim", "-F", msgFile) +} + +func headMessage(t *testing.T, dir string) string { + t.Helper() + raw := runGitOutput(t, dir, "cat-file", "commit", "HEAD") + _, msg, _ := strings.Cut(raw, "\n\n") + return msg +} + +func TestBrokerRewritesHeadCommitMessageDashes(t *testing.T) { + dir := initRepo(t) + msg := "fix: a " + emDashForTest + " b\n\nWhy " + emDashForTest + " because.\n\n```\nkeep " + emDashForTest + " this\n```\n\nSigned-off-by: Hive Test \n" + commitWithMessage(t, dir, msg) + before := strings.TrimSpace(runGitOutput(t, dir, "rev-parse", "HEAD")) + res, err := (&Broker{Workspace: dir, Branch: "work", Repo: "hivecommons/hive", Minter: fakeMinter{"ghs_pushbroker"}, Runner: &recordingRunner{}}).Run(context.Background()) + if err != nil { + t.Fatalf("Run: %v (res=%+v)", err, res) + } + want := "fix: a - b\n\nWhy - because.\n\n```\nkeep " + emDashForTest + " this\n```\n\nSigned-off-by: Hive Test \n" + if got := headMessage(t, dir); got != want { + t.Fatalf("HEAD message = %q, want %q", got, want) + } + after := strings.TrimSpace(runGitOutput(t, dir, "rev-parse", "HEAD")) + if after == before || res.Commit != after { + t.Fatalf("Result.Commit = %s, HEAD %s (was %s): the amend must be what is pushed", res.Commit, after, before) + } +} + +func TestBrokerLeavesCleanCommitMessageAlone(t *testing.T) { + dir := initRepo(t) + commitWithMessage(t, dir, "fix: a - b\n\nlevels 3"+prosefilter.EnDash+"5\n") + before := strings.TrimSpace(runGitOutput(t, dir, "rev-parse", "HEAD")) + res, err := (&Broker{Workspace: dir, Branch: "work", Repo: "hivecommons/hive", Minter: fakeMinter{"ghs_pushbroker"}, Runner: &recordingRunner{}}).Run(context.Background()) + if err != nil { + t.Fatalf("Run: %v", err) + } + if res.Commit != before { + t.Fatalf("a commit with nothing to rewrite was amended: %s -> %s", before, res.Commit) + } +} + +func TestBrokerCommitMessageSwitchOff(t *testing.T) { + t.Setenv(prosefilter.AgentWritesEnv, "false") + dir := initRepo(t) + msg := "fix: a " + emDashForTest + " b\n" + commitWithMessage(t, dir, msg) + if _, err := (&Broker{Workspace: dir, Branch: "work", Repo: "hivecommons/hive", Minter: fakeMinter{"ghs_pushbroker"}, Runner: &recordingRunner{}}).Run(context.Background()); err != nil { + t.Fatalf("Run: %v", err) + } + if got := headMessage(t, dir); got != msg { + t.Fatalf("switch off still rewrote the message: %q", got) + } +} + +func TestBrokerSurfacesCommitMessageAmendFailure(t *testing.T) { + dir := initRepo(t) + commitWithMessage(t, dir, "fix: a "+emDashForTest+" b\n") + r := &failingRunner{failSubstr: "--cleanup=verbatim", failErr: errors.New("amend refused")} + _, err := (&Broker{Workspace: dir, Branch: "work", Repo: "hivecommons/hive", Minter: fakeMinter{"ghs_pushbroker"}, Runner: r}).Run(context.Background()) + if err == nil || !strings.Contains(err.Error(), "normalising commit message dashes") { + t.Fatalf("Run error = %v, want it to mention normalising commit message dashes", err) + } +} + +func TestBrokerSurfacesCommitMessageReadFailure(t *testing.T) { + dir := initRepo(t) + commitWithMessage(t, dir, "fix: plain\n") + r := &failingRunner{failSubstr: "cat-file commit HEAD", failErr: errors.New("object missing")} + _, err := (&Broker{Workspace: dir, Branch: "work", Repo: "hivecommons/hive", Minter: fakeMinter{"ghs_pushbroker"}, Runner: r}).Run(context.Background()) + if err == nil || !strings.Contains(err.Error(), "reading HEAD commit message") { + t.Fatalf("Run error = %v, want it to mention reading HEAD commit message", err) + } +} + +func TestSanitizeCommitMessage(t *testing.T) { + cases := map[string]string{ + "subject only " + emDashForTest + " x": "subject only - x", + "s " + emDashForTest + " t\n\nbody " + emDashForTest: "s - t\n\nbody -", + "clean\n\nbody\n": "clean\n\nbody\n", + } + for in, want := range cases { + if got := sanitizeCommitMessage(in); got != want { + t.Errorf("sanitizeCommitMessage(%q) = %q, want %q", in, got, want) + } + } +} diff --git a/src/crux/pushbroker/pushbroker.go b/src/crux/pushbroker/pushbroker.go index fe80b39f9b..961294d4b7 100644 --- a/src/crux/pushbroker/pushbroker.go +++ b/src/crux/pushbroker/pushbroker.go @@ -221,9 +221,18 @@ func (b *Broker) Run(ctx context.Context) (res Result, err error) { // the final newline (kubestellar/hive#5116). Fixing it here, once, covers // every backend and every target-repo language instead of teaching each // coding CLI's own formatter to run first. - if amended, err := b.stripTrailingBlankLines(ctx, res.ChangedFiles); err != nil { + amended, err := b.stripTrailingBlankLines(ctx, res.ChangedFiles) + if err != nil { return b.fail(res, fmt.Errorf("normalising trailing newlines: %w", err)) - } else if amended { + } + // The commit message travels inside the pushed objects, out of the + // egress proxy's reach, so the broker applies the outbound dash rule to + // it here (hivecommons/hive#19103). + msgAmended, err := b.sanitizeHeadCommitMessage(ctx) + if err != nil { + return b.fail(res, fmt.Errorf("normalising commit message dashes: %w", err)) + } + if amended || msgAmended { commit, err = b.git(ctx, "rev-parse", "HEAD") if err != nil { return b.fail(res, fmt.Errorf("reading HEAD after newline normalisation: %w", err)) diff --git a/src/docs/documentation-map.md b/src/docs/documentation-map.md index 8a848e1614..319e641544 100644 --- a/src/docs/documentation-map.md +++ b/src/docs/documentation-map.md @@ -197,6 +197,7 @@ Documentation for the current Hive line (branch `v5`; the code and docs live und - [Experiment Lab (Nous)](strategy-lab.md) — experiment lifecycle, dashboard/API configuration, fast-fail bounds, and the gate-decision flow. No `nous:` block in `hive.yaml`. - [Hive-self background experiments (Nous)](nous-hive-self-experiments.md) — background-only self-improvement experiment mode, outcome questions (including user satisfaction), cohorts, guardrails, and auto-revert. Hidden from spoke dashboards and Experiment Lab. - [GitHub write surface](github-write-surface.md) — inventory agent and system writes, their audit coverage, and opt-in per-lane restrictions. +- [Outbound text: no em-dashes](outbound-prose.md) - the shared dash filter, every outbound path it covers (proxy agent writes, notifications, meetings, hub posts, commit messages), the `HIVE_AGENT_PROSE_FILTER` switch and the CI inventory guard. - [Compact GitHub CLI reads (`hive-gh`) and one-line denials](github-cli-compact.md) — **Agent/developer reference:** compact output fields, full-read escape hatch, and denial codes. - [GitHub App setup](github-app-setup.md) — the Forge App on GitHub and GitHub Enterprise: app creation, permissions, Setup URL, and `/gh-setup`. - [Forge setup: GitLab, Gitea, and Forgejo](forge-app-setup.md) — **Adapter/reference only:** explains configured adapter surfaces and why a complete hive still requires GitHub. diff --git a/src/docs/env-vars.md b/src/docs/env-vars.md index 1a3def9f04..0fac604357 100644 --- a/src/docs/env-vars.md +++ b/src/docs/env-vars.md @@ -273,6 +273,7 @@ new value at the same time. | `HIVE_PROXY_EGRESS_MARK` | No | `0x1112` | Packet mark exempted from the MITM egress redirect. | | `HIVE_PROXY_ADVISORY_OK` | No | `false` | Allows the spoke to start when the forced-proxy egress redirect cannot be installed (no `CAP_NET_ADMIN`/iptables). Enforcement becomes advisory-only — agents can bypass the proxy. Also gates whether the Go proxy trusts a self-asserted `Proxy-Authorization` header as agent identity when its UID map is unavailable (N7, #3841) — off by default, an unidentified caller is treated as `ADVISORY` (writes blocked) rather than whatever name it claims. See [security-model.md](security-model.md#5-network-and-credentials). | | `HIVE_PROXY_ALLOW_PRIVATE_FORWARD` | No | `false` | Lifts the proxy's destination guard ([#14082](https://github.com/hivecommons/hive/issues/14082), [#16785](https://github.com/hivecommons/hive/pull/16785)). Off by default: a plain-HTTP (non-CONNECT) forward, a CONNECT to a host the proxy does not inspect, and a transparent-TLS passthrough are each dialled only when the resolved address is globally routable, so an agent cannot use the proxy to reach loopback services beside it, the link-local cloud metadata endpoint (`169.254.169.254`), RFC 1918/CGNAT cluster addresses or multicast, and the refusal is a `403` naming the class. Only the exact string `true` turns it on, and it lifts the guard for all three legs at once, for deployments that genuinely egress to a private network through the proxy (a LAN mirror on a native install); the proxy logs a boot-time warning when it is set. Loopback refusals stay at `DEBUG` and are not counted as violations, because agents are already handed `NO_PROXY` for loopback (#14076). With the guard lifted, the inference translator also stops trusting the proxy UID (`dev`, 1001 on v5 pods): agents that run as the hive user without a per-agent UID are refused with `reason=proxy_uid_private_forward` ([#18061](https://github.com/hivecommons/hive/issues/18061), [#14448](https://github.com/hivecommons/hive/issues/14448)). | +| `HIVE_AGENT_PROSE_FILTER` | No | on | Rewrites em-dashes (and spaced en-dashes) in text an agent wrote itself: its GitHub and Linear writes through the egress proxy (JSON text fields only) and its HEAD commit message in the push broker. `false`, `0`, `off` or `no` turns it off. Text hive composes is always filtered. See [outbound-prose.md](outbound-prose.md) ([#19103](https://github.com/hivecommons/hive/issues/19103)). | | `HIVE_PROXY_INJECT_GH_AUTH` | No | unset (off on every hive, including hosted) | Exact `true` enables proxy-side App-token injection. **Requires GitHub App auth. Refuses startup (exit 19) with `HIVE_PROXY_ADVISORY_OK=true`.** Other spellings do not enable it. Read [behavior and exceptions](#proxy-credential-injection). | | `HIVE_AUTHZ_SHADOW` | No | unset (off) | Parseable true value (`true`, `1`) turns on the authorization decision seam in shadow mode ([design](design/authorization-decision-point.md), RFC #18988 phase 0/1): the push broker's changed-files guard is re-evaluated by `authz.Decide` after it decides, one decision event per call is kept on a bounded in-memory stream and a difference from the guard is logged. It enforces nothing and changes no behaviour. Read once at first use, so a change needs a restart. | | `HIVE_AUTHZ_SHADOW_BUFFER` | No | `1000` | Number of decision events the shadow stream keeps (oldest evicted first, evictions counted). Positive integer, capped at `10000`; anything else uses the default. Only read when `HIVE_AUTHZ_SHADOW` is on. | diff --git a/src/docs/github-write-surface.md b/src/docs/github-write-surface.md index a563f2deb6..b5710b5844 100644 --- a/src/docs/github-write-surface.md +++ b/src/docs/github-write-surface.md @@ -25,6 +25,11 @@ By default agents still have their direct write paths (`gh`, the API through the proxy, `git push`). An operator removes them lane by lane; see [Enforcing the write surface](#enforcing-the-write-surface). +Whichever path a write takes, its text gets the same outbound dash rule: +relay bodies are filtered by the relay, and direct agent writes through the +GitHub proxy have their JSON text fields filtered before they are forwarded. +See [Outbound text: no em-dashes](outbound-prose.md). + ## Relay operations Each relay authorizes a request in this order. It stops at the first refusal. diff --git a/src/docs/outbound-prose.md b/src/docs/outbound-prose.md new file mode 100644 index 0000000000..305d6870a9 --- /dev/null +++ b/src/docs/outbound-prose.md @@ -0,0 +1,119 @@ +# Outbound text: no em-dashes + +Nothing hive or its agents send out of the hive may contain an em-dash +(U+2014), or an en-dash (U+2013) used as a dash. This page lists where that +rule is enforced and how CI keeps new send paths from skipping it +([#18970](https://github.com/hivecommons/hive/pull/18970), +[#19103](https://github.com/hivecommons/hive/issues/19103)). + +## The filter + +There is one filter, `src/crux/prosefilter`. The root module reaches it +through `src/pkg/prosefilter`, which only delegates. The filter lives in the +`crux` module so the egress proxy, which cannot import `pkg/`, uses the same +function. + +- `Sanitize` rewrites multi-line Markdown prose. An em-dash becomes ` - ` + (or `- ` / ` -` at the start or end of a line). An en-dash with spaces on + both sides becomes ` - `. An en-dash in a range such as `3-5` written with + U+2013 is left alone. +- `SanitizeTitle` does the same for a single line. +- `SanitizeJSON` applies the same rule to the string values under named keys + of a JSON document, at any depth, and leaves every other value unchanged. +- Fenced code blocks, inline code spans and blockquote lines are never + changed, because they carry quoted or pasted content. + +Agents are also told the rule in every kick (the plain-dash rule in the kick +header), so most text never needs rewriting. + +## Where it applies + +| Path | Boundary | Filtered | +| --- | --- | --- | +| Agent relay requests (PR, issue, comment, review bodies and titles) | `relayBody`, `CreatePR`, the issue-request watcher (`src/pkg/github`) | yes | +| Agent GitHub writes through the egress proxy (`gh api`, `gh pr comment`, `gh pr create`, REST and GraphQL) | `sanitizeAgentWriteProse` in `src/crux/proxy/outbound_prose.go` | yes | +| Agent Linear writes through the egress proxy | same, Linear mutation variables | yes | +| Agent commit messages | HEAD commit message in the push broker (`src/crux/pushbroker/commit_message.go`) | yes, HEAD only (see below) | +| Hive-signed commits (`createCommitOnBranch`) | `src/pkg/github/pr_request_signed.go` | yes | +| Squash and merge commit messages hive writes | `MergePRVia` in `src/pkg/github/pullrequest.go` | yes | +| Hive-rendered GitHub comments, issues and status notes | source templates | enforced at the source by the emit ratchet | +| ntfy, Slack and Discord webhook alerts | `notify.(*Notifier).Send` | yes | +| Slack DMs from the hub | `hub.sendSlackDM` | yes | +| Discord bot messages, embeds, edits, threads, topics | `discord.(*Bot).do`, `setChannelTopic` | yes | +| Hub Discord webhooks (requests, GitHub activity feed) | `hub.postDiscordRequestWebhook`, `hub.(*GitHubActivityFeed).postDiscord` | yes | +| Hub ntfy alert | `hub.(*HubServer).pushAuthAuditAlert` | yes | +| Meeting chat and spoken text (Quorum `chat.post`, `say`, `say.prewarm`) | `meetings.(*Client).post` | yes | +| Persona voice sample | `dashboard.meetingsSpeak` | yes | +| Hub-filed GitHub issues and comments (self-reports, feedback) | `hub.postHubGitHubIssue`, `hub.postHubGitHubIssueComment` | yes | +| Hub seen-log and affected-hives edits | `pkg/fleetreport` templates | emit ratchet | +| Linear and Jira work-source writes | `worksource.linearGraphQL`, `worksource.(*jiraSource).doJSON` | yes | +| Linear agent activity | `linearagent.doGraphQL` | yes | +| DIBS inbox ideas | `inbox.(*DibsDestination).Deliver` | yes | +| GitLab, Gitea and Forgejo writes | `pkg/forge` | emit ratchet | +| Email replies | no sender is wired yet | not applicable | + +## The egress proxy + +An agent that writes to GitHub itself talks to GitHub through the hive's MITM +egress proxy. Before #19103 the relay forwarded an allowed write verbatim, so +an agent's `gh api` comment kept its em-dashes even though hive's own writes +were filtered. The proxy now rewrites, just before forwarding: + +- REST JSON bodies: `body`, `title` and `message` at any depth, which covers + issue and PR comments, PR and issue titles and bodies, review bodies and + their inline `comments[].body`, review-thread replies and contents-API + commit messages. +- GraphQL mutation variables: `body` and `title` when the document names a + comment, review, review-thread reply, PR or issue mutation (`addComment`, + `addPullRequestReviewComment`, `addPullRequestReviewThreadReply`, + `createPullRequest`, `updatePullRequest`, `createIssue`, `updateIssue` and + the related review and comment-edit mutations). The query text is never + changed, so a string literal written inline in the query is not rewritten. +- Linear mutation variables: `body`, `title` and `description`. + +It never changes a body that is not JSON, a body that declares a non-JSON +`Content-Type`, any other field, git smart-HTTP or LFS traffic, or the hive's +own control-plane requests. A body over 1 MiB is forwarded unchanged rather +than truncated. When a body changes, its `Content-Length` is recomputed. One +debug line records the agent, method, path and the number of fields and dashes +changed, never the text. + +## Commit messages + +A commit message travels inside the pushed git objects, so it cannot be +rewritten in transit without changing object ids. The push broker, which +already amends an agent's HEAD commit for the trailing-newline rule, rewrites +the HEAD commit message with the same filter before pushing. Only HEAD is +rewritten: an older commit in the same push keeps its message, because the +broker never rebases an agent's branch. Hive squash-merges, and the squash +commit message it writes is filtered, so the commit that lands on the default +branch is clean either way. Hive installs no git hooks in agent workspaces +(hooks are disabled with `core.hooksPath=/dev/null`), so this is done by the +broker rather than a `commit-msg` hook. + +## Switch + +`HIVE_AGENT_PROSE_FILTER` turns off the rewriting of text an agent wrote +itself (its writes through the proxy and its HEAD commit message). It is on by +default; `false`, `0`, `off` or `no` turns it off. Text hive composes is always +filtered. + +## CI guards + +- `TestOutboundSitesInventory` (`src/pkg/prosefilter/outbound_sites_test.go`) + scans every non-test Go file under `src/pkg`, `src/cmd` and `src/crux` for + functions that send an HTTP request with a body. Each one must be listed in + its inventory as filtered (it calls the filter), filtered upstream by a + named function that calls the filter, a hive template under the emit + ratchet, or machine traffic with a reason. A new sender that is not listed + fails CI, as does a filtered site that stops calling the filter. The test + also pins the proxy, push broker, relay, PR, merge and notification + boundaries. +- `TestNoNewDashesInEmittedText` (`src/pkg/prosefilter/emit_guard_test.go`) + is a ratchet on hive's own templates: the dash count of each emitter file + may only go down. + +## Follow-ups + +- GraphQL string literals inlined in the query text are not rewritten. +- Commits other than HEAD in a multi-commit push keep their messages. diff --git a/src/pkg/dashboard/api_meetings_persona.go b/src/pkg/dashboard/api_meetings_persona.go index 41ed28a210..71d5c19d00 100644 --- a/src/pkg/dashboard/api_meetings_persona.go +++ b/src/pkg/dashboard/api_meetings_persona.go @@ -17,6 +17,7 @@ import ( "time" "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/prosefilter" ) // Meeting attendee settings (hivecommons/hive#18396): Settings, Meeting @@ -626,6 +627,7 @@ func (s *Server) handleMeetingsPersonaSample(w http.ResponseWriter, r *http.Requ // meetingsSpeak calls the TTS service. On failure it returns the HTTP // status to answer with. func meetingsSpeak(ctx context.Context, client *http.Client, cfg MeetingsTTSConfig, voice, text string) ([]byte, string, int, error) { + text = prosefilter.Sanitize(text) body, _ := json.Marshal(meetingsTTSSpeechRequest{Model: meetingsTTSModel, Input: text, Voice: voice, ResponseFormat: meetingsTTSFormat}) ctx, cancel := context.WithTimeout(ctx, meetingsSampleTimeout) defer cancel() diff --git a/src/pkg/discord/bot.go b/src/pkg/discord/bot.go index edfd4189eb..100211f870 100644 --- a/src/pkg/discord/bot.go +++ b/src/pkg/discord/bot.go @@ -16,6 +16,7 @@ import ( "unicode/utf8" "github.com/hivecommons/hive/crux/paneclass" + "github.com/hivecommons/hive/pkg/prosefilter" ) const ( @@ -488,6 +489,9 @@ func (b *Bot) do(method, url string, payload, out any) error { if data, err = json.Marshal(payload); err != nil { return err } + // Every bot message, embed, edit, thread and channel name goes + // through here: apply the shared outbound dash rule (#19103). + data, _ = prosefilter.SanitizeJSON(data, discordProseKeys) } for attempt := 0; ; attempt++ { var body io.Reader @@ -1015,7 +1019,18 @@ func (b *Bot) updateTopic(snap *statusSnapshot) { } } +// discordProseKeys are the Discord payload fields people read. +var discordProseKeys = prosefilter.Keys{ + "content": prosefilter.Prose, + "description": prosefilter.Prose, + "value": prosefilter.Prose, + "title": prosefilter.Title, + "name": prosefilter.Title, + "topic": prosefilter.Title, +} + func (b *Bot) setChannelTopic(topic string) error { + topic = prosefilter.SanitizeTitle(topic) payload, err := json.Marshal(map[string]string{"topic": topic}) if err != nil { return fmt.Errorf("marshal topic payload: %w", err) diff --git a/src/pkg/discord/bot_prose_test.go b/src/pkg/discord/bot_prose_test.go new file mode 100644 index 0000000000..098132ea1f --- /dev/null +++ b/src/pkg/discord/bot_prose_test.go @@ -0,0 +1,38 @@ +package discord + +import ( + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +// Every bot write goes through Bot.do, which applies the shared outbound dash +// rule (hivecommons/hive#19103) without touching the other payload fields. +func TestBotDoFiltersDashes(t *testing.T) { + var body []byte + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, _ = io.ReadAll(r.Body) + w.WriteHeader(http.StatusOK) + })) + defer ts.Close() + + b := newTestBot(ts, "chan") + if err := b.SendMessageTo("rfc-channel", "RFC opened — please review"); err != nil { + t.Fatalf("SendMessageTo: %v", err) + } + var got struct { + Content string `json:"content"` + } + if err := json.Unmarshal(body, &got); err != nil { + t.Fatalf("unmarshal: %v", err) + } + if got.Content != "RFC opened - please review" { + t.Errorf("content = %q", got.Content) + } + if !strings.Contains(string(body), `"allowed_mentions":{"parse":[]}`) { + t.Errorf("allowed_mentions changed: %s", body) + } +} diff --git a/src/pkg/github/pullrequest.go b/src/pkg/github/pullrequest.go index 1acc2d9b80..669dff3f2e 100644 --- a/src/pkg/github/pullrequest.go +++ b/src/pkg/github/pullrequest.go @@ -474,6 +474,8 @@ func (c *Client) MergePRVia(ctx context.Context, repo string, number int, mergeM if err != nil { return MergePRResult{}, fmt.Errorf("merging PR %s/%s#%d (%s): %w", owner, repo, number, mergeMethod, err) } + // The landing commit message is outbound text like the PR body (#19103). + commitMessage = prosefilter.Sanitize(commitMessage) var res *gh.PullRequestMergeResult boundary := c.mutationBoundary() claim := effects.Claim{ diff --git a/src/pkg/hub/auth_audit.go b/src/pkg/hub/auth_audit.go index c33e9a7a33..e06f9db3dc 100644 --- a/src/pkg/hub/auth_audit.go +++ b/src/pkg/hub/auth_audit.go @@ -8,6 +8,8 @@ import ( "os" "strings" "time" + + "github.com/hivecommons/hive/pkg/prosefilter" ) const ( @@ -179,6 +181,7 @@ func (s *HubServer) reportURLHealth(hives []RegistryEntry, clusterProbed, cluste // best-effort: failure to alert is logged but never fatal. The log Warn above // is always emitted regardless, so an external log-based alert still fires. func (s *HubServer) pushAuthAuditAlert(message string) { + message = prosefilter.Sanitize(message) server := strings.TrimRight(os.Getenv("HIVE_NTFY_SERVER"), "/") topic := os.Getenv("HIVE_NTFY_TOPIC") if server == "" || topic == "" { diff --git a/src/pkg/hub/discord_request_notify.go b/src/pkg/hub/discord_request_notify.go index f079a6eb60..287d6a8896 100644 --- a/src/pkg/hub/discord_request_notify.go +++ b/src/pkg/hub/discord_request_notify.go @@ -14,6 +14,7 @@ import ( "time" "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/prosefilter" ) const ( @@ -129,11 +130,21 @@ func (s *HubServer) deliverDiscordRequestNotification(webhookURL, requestID stri "request_id", requestID, "attempts", discordRequestWebhookMaxAttempts, "error", lastErr) } +// discordWebhookProseKeys are the Discord webhook payload fields people read. +var discordWebhookProseKeys = prosefilter.Keys{ + "content": prosefilter.Prose, + "description": prosefilter.Prose, + "value": prosefilter.Prose, + "title": prosefilter.Title, + "name": prosefilter.Title, +} + func postDiscordRequestWebhook(ctx context.Context, webhookURL string, payload discordWebhookPayload) error { body, err := json.Marshal(payload) if err != nil { return err } + body, _ = prosefilter.SanitizeJSON(body, discordWebhookProseKeys) req, err := http.NewRequestWithContext(ctx, http.MethodPost, webhookURL, bytes.NewReader(body)) if err != nil { return err diff --git a/src/pkg/hub/feedback.go b/src/pkg/hub/feedback.go index 96f8cf7c24..35a5e084af 100644 --- a/src/pkg/hub/feedback.go +++ b/src/pkg/hub/feedback.go @@ -18,6 +18,7 @@ import ( "github.com/hivecommons/hive/pkg/config" "github.com/hivecommons/hive/pkg/feedbackbundle" ghauth "github.com/hivecommons/hive/pkg/github" + "github.com/hivecommons/hive/pkg/prosefilter" ) const ( @@ -685,6 +686,7 @@ func writeHubFeedbackDiagnostics(b *strings.Builder, d *feedbackDiagnostics, sub } } func postHubGitHubIssue(ctx context.Context, client *http.Client, apiBase, token, owner, repo, title, body string, labels []string) (feedbackIssueResult, int, error) { + title, body = prosefilter.SanitizeTitle(title), prosefilter.Sanitize(body) ctx = hubGitHubCallerContext(ctx, "feedback_issue") payload := map[string]any{"title": title, "body": body} if labels != nil { diff --git a/src/pkg/hub/github_activity.go b/src/pkg/hub/github_activity.go index 396218e1a2..3f7af8dc3d 100644 --- a/src/pkg/hub/github_activity.go +++ b/src/pkg/hub/github_activity.go @@ -16,6 +16,8 @@ import ( "strings" "sync" "time" + + "github.com/hivecommons/hive/pkg/prosefilter" ) const ( @@ -676,6 +678,7 @@ func (f *GitHubActivityFeed) authorSuppressed(author string, forwardMerge bool, const discordSuppressEmbeds = 1 << 2 func (f *GitHubActivityFeed) postDiscord(ctx context.Context, content string) error { + content = prosefilter.Sanitize(content) payload, err := json.Marshal(map[string]any{"content": content, "flags": discordSuppressEmbeds}) if err != nil { return err diff --git a/src/pkg/hub/selfreport_relay.go b/src/pkg/hub/selfreport_relay.go index 845cef2a70..8e7f08a842 100644 --- a/src/pkg/hub/selfreport_relay.go +++ b/src/pkg/hub/selfreport_relay.go @@ -14,6 +14,7 @@ import ( "github.com/hivecommons/hive/crux/logscrub" "github.com/hivecommons/hive/pkg/config" "github.com/hivecommons/hive/pkg/fleetreport" + "github.com/hivecommons/hive/pkg/prosefilter" ) // Self-report relay (hivecommons/hive#11868). Spokes whose GitHub credential @@ -654,6 +655,7 @@ func findSelfReportIssueByFingerprintIn(ctx context.Context, client *http.Client } func postHubGitHubIssueComment(ctx context.Context, client *http.Client, apiBase, token, owner, repo string, number int, body string) (int, error) { + body = prosefilter.Sanitize(body) ctx = hubGitHubCallerContext(ctx, "selfreport_comment") data, _ := json.Marshal(map[string]string{"body": body}) u := fmt.Sprintf("%s/repos/%s/%s/issues/%d/comments", strings.TrimRight(apiBase, "/"), owner, repo, number) diff --git a/src/pkg/hub/slack.go b/src/pkg/hub/slack.go index 97592cc4c2..7840ceabb8 100644 --- a/src/pkg/hub/slack.go +++ b/src/pkg/hub/slack.go @@ -9,6 +9,8 @@ import ( "strings" "sync" "time" + + "github.com/hivecommons/hive/pkg/prosefilter" ) // slackPostMessageURL is Slack's chat.postMessage endpoint. A bot token @@ -134,6 +136,7 @@ var slackClient = &http.Client{Timeout: slackHTTPTimeout} // body is decoded and ok is checked. The token is never logged; the returned // error carries only Slack's error code. func sendSlackDM(token, slackID, message string) error { + message = prosefilter.Sanitize(message) payload, err := json.Marshal(map[string]string{ "channel": slackID, "text": message, diff --git a/src/pkg/inbox/dibs.go b/src/pkg/inbox/dibs.go index 334c2c203b..6f79bf53f1 100644 --- a/src/pkg/inbox/dibs.go +++ b/src/pkg/inbox/dibs.go @@ -10,6 +10,8 @@ import ( "net/http" "strings" "time" + + "github.com/hivecommons/hive/pkg/prosefilter" ) // dibsTimeout bounds one idea submission. @@ -51,8 +53,8 @@ func (d *DibsDestination) Deliver(ctx context.Context, it Item, c Content, _ Req } body := DibsBody(it, c) payload, _ := json.Marshal(dibsIdea{ - Title: firstNonEmpty(c.Title, it.Title), - Body: body, + Title: prosefilter.SanitizeTitle(firstNonEmpty(c.Title, it.Title)), + Body: prosefilter.Sanitize(body), Visibility: "private", Status: "draft", }) diff --git a/src/pkg/linearagent/client.go b/src/pkg/linearagent/client.go index b74032895c..4c6317755d 100644 --- a/src/pkg/linearagent/client.go +++ b/src/pkg/linearagent/client.go @@ -9,6 +9,8 @@ import ( "log/slog" "net/http" "time" + + "github.com/hivecommons/hive/pkg/prosefilter" ) // Client is the CONTROL-PLANE Linear GraphQL client: it acknowledges sessions @@ -266,6 +268,9 @@ type graphQLEnvelope struct { } `json:"errors"` } +// linearAgentProseKeys are the Linear agent-session fields people read. +var linearAgentProseKeys = prosefilter.Keys{"body": prosefilter.Prose, "title": prosefilter.Title, "description": prosefilter.Prose} + // doGraphQL runs one authenticated GraphQL POST. Error strings carry only the // HTTP status and GraphQL error message — never the token or variables (which // hold issue content). @@ -274,6 +279,8 @@ func doGraphQL(ctx context.Context, hc *http.Client, graphqlURL, accessToken, qu if err != nil { return fmt.Errorf("marshal request: %w", err) } + // Agent activity text shown in Linear (#19103). + body, _ = prosefilter.SanitizeJSON(body, linearAgentProseKeys) req, err := http.NewRequestWithContext(ctx, http.MethodPost, graphqlURL, bytes.NewReader(body)) if err != nil { return fmt.Errorf("build request: %w", err) diff --git a/src/pkg/meetings/client.go b/src/pkg/meetings/client.go index 5712b62da6..3aa68f9213 100644 --- a/src/pkg/meetings/client.go +++ b/src/pkg/meetings/client.go @@ -15,6 +15,8 @@ import ( "strings" "sync" "sync/atomic" + + "github.com/hivecommons/hive/pkg/prosefilter" ) // MCP over streamable HTTP (POST /mcp), the transport Quorum serves. The @@ -104,6 +106,9 @@ func (c *Client) post(ctx context.Context, body any, wantID int64) (json.RawMess if err != nil { return nil, err } + // Every chat line and spoken line (chat.post, say, say.prewarm) leaves + // through here: apply the shared outbound dash rule (#19103). + data, _ = prosefilter.SanitizeJSON(data, meetingProseKeys) req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.base+"/mcp", bytes.NewReader(data)) if err != nil { return nil, err @@ -359,6 +364,10 @@ func (c *Client) TranscriptRead(ctx context.Context, nativeID string, afterSeq i return r, err } +// meetingProseKeys are the tool-argument fields that become meeting chat or +// speech. +var meetingProseKeys = prosefilter.Keys{"text": prosefilter.Prose, "texts": prosefilter.Prose} + // ChatPost calls chat.post. func (c *Client) ChatPost(ctx context.Context, nativeID, text string) error { return c.Call(ctx, "chat.post", textArgs{nativeID, text}, nil) diff --git a/src/pkg/meetings/client_prose_test.go b/src/pkg/meetings/client_prose_test.go new file mode 100644 index 0000000000..2450df48b3 --- /dev/null +++ b/src/pkg/meetings/client_prose_test.go @@ -0,0 +1,42 @@ +package meetings + +import ( + "context" + "testing" +) + +// Chat lines and spoken lines leave hive through Client.post, which applies +// the shared outbound dash rule (hivecommons/hive#19103). +func TestClientFiltersDashesInChatAndSpeech(t *testing.T) { + f, srv := newFakeQuorumServer(t) + c := NewClient(srv.URL, "tok", nil) + ctx := context.Background() + f.results["chat.post"] = `{"ok":true}` + f.results["say"] = `{"ok":true}` + f.results["say.prewarm"] = `{"ok":true}` + if err := c.ChatPost(ctx, "q1", "noted — next item"); err != nil { + t.Fatal(err) + } + if err := c.Say(ctx, "q1", "one — two"); err != nil { + t.Fatal(err) + } + if err := c.SayPrewarm(ctx, "q1", []string{"three — four"}); err != nil { + t.Fatal(err) + } + f.mu.Lock() + defer f.mu.Unlock() + n := len(f.args) + if n < 3 { + t.Fatalf("recorded %d tool calls, want at least 3", n) + } + if got := f.args[n-3]["text"]; got != "noted - next item" { + t.Errorf("chat.post text = %v", got) + } + if got := f.args[n-2]["text"]; got != "one - two" { + t.Errorf("say text = %v", got) + } + texts, _ := f.args[n-1]["texts"].([]any) + if len(texts) != 1 || texts[0] != "three - four" { + t.Errorf("say.prewarm texts = %v", f.args[n-1]["texts"]) + } +} diff --git a/src/pkg/notify/notify.go b/src/pkg/notify/notify.go index b553728975..3f01e6d711 100644 --- a/src/pkg/notify/notify.go +++ b/src/pkg/notify/notify.go @@ -12,6 +12,7 @@ import ( "time" "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/prosefilter" ) const ( @@ -59,6 +60,9 @@ const ( ) func (n *Notifier) Send(title, message string, priority Priority) { + // The one funnel for ntfy, Slack and Discord webhook alerts: the shared + // outbound dash rule applies here (hivecommons/hive#19103). + title, message = prosefilter.SanitizeTitle(title), prosefilter.Sanitize(message) if n.hiveID != "" { title = fmt.Sprintf("[%s] %s", n.hiveID, title) } diff --git a/src/pkg/prosefilter/emit_guard_test.go b/src/pkg/prosefilter/emit_guard_test.go index ab0882ecb7..02241770dd 100644 --- a/src/pkg/prosefilter/emit_guard_test.go +++ b/src/pkg/prosefilter/emit_guard_test.go @@ -25,6 +25,10 @@ var emitDirs = []string{ "pkg/prfollowup", "pkg/scheduler", "pkg/review", + // Outbound send sites classed as templates (outbound_sites_test.go). + "pkg/forge", + "pkg/feedbackbundle", + "pkg/fleetreport", } // TestNoNewDashesInEmittedText is a ratchet: the em-dash and en-dash count of diff --git a/src/pkg/prosefilter/outbound_sites_test.go b/src/pkg/prosefilter/outbound_sites_test.go new file mode 100644 index 0000000000..e354ece22b --- /dev/null +++ b/src/pkg/prosefilter/outbound_sites_test.go @@ -0,0 +1,432 @@ +package prosefilter + +import ( + "go/ast" + "go/parser" + "go/token" + "io/fs" + "os" + "path/filepath" + "runtime" + "slices" + "sort" + "strconv" + "strings" + "testing" +) + +// Outbound send-site inventory (hivecommons/hive#19103). +// +// Every non-test function under src/pkg, src/cmd and src/crux that sends an +// HTTP request with a body (http.NewRequest / NewRequestWithContext with a +// method other than GET, HEAD, DELETE or OPTIONS, http.Post, http.PostForm, +// or a client's Post(url, "content/type", body)) must be listed in +// outboundSites with a class. TestOutboundSitesInventory fails when a new +// sender is added without an entry, when an entry no longer names a sender, +// and when a site classed as filtered stops calling the shared filter. That +// keeps the em-dash rule from regressing as new channels are added: a new +// Slack, Discord, meeting, email or forge sender cannot land without someone +// deciding, in review, how its text is filtered. +// +// Hive's own go-github writes do not appear here (they are not raw HTTP +// sends): their text is either agent text passed through relayBody or a +// hive-rendered template guarded by TestNoNewDashesInEmittedText. The +// egress proxy's relay of agent writes and the push broker's commit message +// rule are pinned by requiredCalls below. + +type outboundClass string + +const ( + // classFiltered: the function itself calls prosefilter before sending. + classFiltered outboundClass = "filtered" + // classVia: the text was filtered by the named function before it got + // here (via must itself call prosefilter). + classVia outboundClass = "via" + // classTemplate: hive-rendered text whose source package (source) is + // under the emit ratchet, so no dash can be written into it. Used where a + // send compares the rendered text with what is already on the forge, and + // rewriting after the compare would edit on every pass. + classTemplate outboundClass = "template" + // classMachine: the body carries no text a person reads on a third-party + // surface (tokens, ids, state, labels, heartbeats, model prompts, calls + // to hive's own services). + classMachine outboundClass = "machine" +) + +type outboundSite struct { + class outboundClass + via string + source string + reason string +} + +func filtered() outboundSite { return outboundSite{class: classFiltered} } +func via(fn, reason string) outboundSite { + return outboundSite{class: classVia, via: fn, reason: reason} +} +func template(source, reason string) outboundSite { + return outboundSite{class: classTemplate, source: source, reason: reason} +} +func machine(reason string) outboundSite { return outboundSite{class: classMachine, reason: reason} } + +const ( + reasonSpokeToHub = "hive-to-hub or hub-to-spoke control traffic; hub-side delivery is inventoried separately" + reasonModelPrompt = "model or agent-runtime request; nothing here is published" + reasonAuth = "OAuth, token or credential exchange" + reasonInternalAPI = "call to hive's own API or service" + reasonPlatform = "platform API (Kubernetes, object storage, secrets store)" + reasonStateOnly = "state, labels or ids only; no text" + reasonTranscribe = "audio upload for transcription" + reasonTestFixture = "test fixture server" + reasonFleetReport = "seen-log and affected-hives bookkeeping, compared with the forge copy before an edit" + reasonForgeWrite = "forge comment and issue writes; text is agent text already filtered or hive templates" + reasonFixedMessage = "fixed ASCII commit message" +) + +var outboundSites = map[string]outboundSite{ + // Filtered at the send boundary. + "dashboard.meetingsSpeak": filtered(), + "discord.(*Bot).do": filtered(), + "discord.(*Bot).setChannelTopic": filtered(), + "github.(*Client).createCommitOnBranch": filtered(), + "hub.(*GitHubActivityFeed).postDiscord": filtered(), + "hub.(*HubServer).pushAuthAuditAlert": filtered(), + "hub.postDiscordRequestWebhook": filtered(), + "hub.postHubGitHubIssue": filtered(), + "hub.postHubGitHubIssueComment": filtered(), + "hub.sendSlackDM": filtered(), + "inbox.(*DibsDestination).Deliver": filtered(), + "linearagent.doGraphQL": filtered(), + "meetings.(*Client).post": filtered(), + "worksource.(*jiraSource).doJSON": filtered(), + "worksource.linearGraphQL": filtered(), + "notify.(*Notifier).sendNtfy": via("notify.(*Notifier).Send", "ntfy alert"), + "notify.(*Notifier).sendSlack": via("notify.(*Notifier).Send", "Slack webhook alert"), + "notify.(*Notifier).sendDiscordWebhook": via("notify.(*Notifier).Send", "Discord webhook alert"), + "main.hubSelfReportRelay": via("hub.postHubGitHubIssue", "spoke self-report; the hub files it"), + "dashboard.(*Server).forwardFeedbackToHub": via("hub.postHubGitHubIssue", "operator feedback; the hub files it"), + + // Hive-rendered text under the emit ratchet. + "hub.patchHubGitHubIssueBody": template("pkg/fleetreport", reasonFleetReport), + "hub.patchHubGitHubIssueComment": template("pkg/fleetreport", reasonFleetReport), + "forge.(*gitLabForge).doWrite": template("pkg/forge", reasonForgeWrite), + "forge.(*giteaForge).doWrite": template("pkg/forge", reasonForgeWrite), + "forge.(*giteaForge).Merge": template("pkg/forge", "merge request on a non-GitHub forge"), + "forge.(*gitLabForge).doAcceptMerge": template("pkg/forge", "merge request on a non-GitHub forge"), + "feedbackbundle.githubJSON": template("pkg/feedbackbundle", "feedback bundle commit message"), + + // No human-read text. + "agent.(*sharedSessionPoolClient).request": machine(reasonInternalAPI), + "api.(*Client).request": machine(reasonInternalAPI), + "auth.(*Provider).fetchIDToken": machine(reasonAuth), + "callback.(*Client).do": machine(reasonInternalAPI), + "client.(*Client).doJSON": machine(reasonInternalAPI), + "connector.(*HTTPClient).once": machine(reasonInternalAPI), + "dashboard.(*Server).forwardNPS": machine(reasonSpokeToHub), + "dashboard.(*Server).handleCopilotAuthStart": machine(reasonAuth), + "dashboard.(*Server).handleReleaseChannelSwitch": machine(reasonSpokeToHub), + "dashboard.(*Server).handleSelfUpgrade": machine(reasonSpokeToHub), + "dashboard.(*Server).hubRequest": machine(reasonSpokeToHub), + "dashboard.(*Server).pollCopilotToken": machine(reasonAuth), + "dashboard.(*Server).postBenchmarksBatch": machine(reasonSpokeToHub), + "dashboard.kubernetesSelfUpgradeRequest": machine(reasonPlatform), + "dashboard.npsRelayPost": machine(reasonSpokeToHub), + "dashboard.patchKeyIntoHiveSecrets": machine(reasonPlatform), + "dashboard.probeLiteLLMInference": machine(reasonModelPrompt), + "discord.(*Bot).dashboardPost": machine(reasonInternalAPI), + "fixture.(*Server).attemptEffect": machine(reasonTestFixture), + "flue.(*Adapter).do": machine(reasonModelPrompt), + "forge.(*gitLabForge).doRebase": machine(reasonStateOnly), + "github.PollDeviceFlowWithScope": machine(reasonAuth), + "github.StartDeviceFlowWithScope": machine(reasonAuth), + "hivectl.(*Client).request": machine(reasonInternalAPI), + "hivectl.Register": machine(reasonInternalAPI), + "hivectl.Reissue": machine(reasonInternalAPI), + "hub.(*HubServer).exchangeGitHubLogin": machine(reasonAuth), + "hub.(*HubServer).pushGitHubConfigToSpoke": machine(reasonSpokeToHub), + "hub.(*HubServer).pushVisibilityToSpoke": machine(reasonSpokeToHub), + "hub.(*HubServer).relayWebhookToSpoke": machine(reasonSpokeToHub), + "hub.closeHubGitHubIssue": machine(reasonStateOnly), + "hub.commitBenchmarkFile": machine(reasonFixedMessage), + "hub.createOCIExport": machine(reasonPlatform), + "hub.createOCIFileSystem": machine(reasonPlatform), + "hub.npsRelayAck": machine(reasonSpokeToHub), + "hub.postHubGitHubIssueLabels": machine(reasonStateOnly), + "hub.relayMailOnce": machine(reasonSpokeToHub), + "hub.topRepoRequest": machine(reasonStateOnly), + "hubbackup.(*ObjectStore).do": machine(reasonPlatform), + "intent.(*AlignmentReviewer).reviewUncached": machine(reasonModelPrompt), + "ioscan.(*openAIChatClient).completeOnce": machine(reasonModelPrompt), + "jev.(*Client).Decide": machine(reasonModelPrompt), + "jev.callEndpoint": machine(reasonModelPrompt), + "knowledge.(*Client).postJSON": machine(reasonInternalAPI), + "knowledge.(*Promoter).ingestToLayer": machine(reasonInternalAPI), + "kubejob.(*Launcher).do": machine(reasonPlatform), + "linearagent.postTokenForm": machine(reasonAuth), + "mailin.(*HubClient).do": machine(reasonSpokeToHub), + "main.(*delegateDeciderRunner).RunDeciderTurn": machine(reasonModelPrompt), + "main.(*knowledgeAdviserLLM).complete": machine(reasonModelPrompt), + "main.kbPost": machine(reasonInternalAPI), + "managedagents.(*Adapter).doJSON": machine(reasonModelPrompt), + "meetings.(*Client).CompanionAudio": machine(reasonTranscribe), + "meetllm.complete": machine(reasonModelPrompt), + "meetllm.completeAnthropic": machine(reasonModelPrompt), + "openbao.(*Provider).raw": machine(reasonPlatform), + "openrouter.ExchangeCode": machine(reasonAuth), + "proxy.(*GitHubProxy).handleInferenceRequest": machine(reasonModelPrompt), + "proxy.(*GitHubProxy).inferenceTranslatorHandler": machine(reasonModelPrompt), + "proxy.(*GitHubProxy).retryInferenceOnAuthFailure": machine(reasonModelPrompt), + "proxy.forwardToInference": machine(reasonModelPrompt), + "retro.(*openAIChatClient).Complete": machine(reasonModelPrompt), + "reviewpreflight.(*Client).do": machine(reasonInternalAPI), + "rotation.KiroProber.Probe": machine(reasonModelPrompt), + "sandbox.(*routerBridge).do": machine(reasonPlatform), + "spoke.SendUpgradingHeartbeat": machine(reasonSpokeToHub), + "spoke.StartTaskStatusPush": machine(reasonSpokeToHub), + "spoke.k8sAPIPatchOnceWithContentType": machine(reasonPlatform), + "spoke.postActivityUpdateAck": machine(reasonSpokeToHub), + "spoke.postHeartbeatToHub": machine(reasonSpokeToHub), + "spoke.postHeartbeatToMirror": machine(reasonSpokeToHub), + "spoke.reportUpgradeOutcome": machine(reasonSpokeToHub), + "trajectory.(*Reviewer).Review": machine(reasonModelPrompt), + "transcribe.(*Deepgram).Transcribe": machine(reasonTranscribe), + "transcribe.(*Gemini).do": machine(reasonTranscribe), + "watsonx.(*TokenMinter).mint": machine(reasonAuth), + "worksource.(*githubProjectsSource).graphql": machine(reasonStateOnly), +} + +// requiredCalls pins the boundaries that are not raw HTTP sends: file +// (relative to src/) -> the function in it that must call the named helper. +var requiredCalls = []struct{ file, fn, call string }{ + {"crux/proxy/github_proxy_mitm.go", "proxy.(*GitHubProxy).proxyHTTPHost", "sanitizeAgentWriteProse"}, + {"crux/pushbroker/pushbroker.go", "pushbroker.(*Broker).Run", "sanitizeHeadCommitMessage"}, + {"pkg/github/write_surface.go", "github.(*Client).relayBody", "Sanitize"}, + {"pkg/github/pullrequest.go", "github.(*Client).CreatePR", "Sanitize"}, + {"pkg/github/pullrequest.go", "github.(*Client).MergePRVia", "Sanitize"}, + {"pkg/notify/notify.go", "notify.(*Notifier).Send", "Sanitize"}, +} + +// safeMethods are the request methods that carry no body to filter. +var safeMethods = map[string]bool{ + "MethodGet": true, "MethodHead": true, "MethodDelete": true, "MethodOptions": true, + `"GET"`: true, `"HEAD"`: true, `"DELETE"`: true, `"OPTIONS"`: true, +} + +// scannedFunc is what the scan records for one function declaration. +type scannedFunc struct { + pos string + sends bool + callsFilter bool + calledIdents map[string]bool +} + +func TestOutboundSitesInventory(t *testing.T) { + srcRoot := srcRootDir(t) + funcs := scanFuncs(t, srcRoot) + + var missing []string + for name, f := range funcs { + if !f.sends { + continue + } + if _, ok := outboundSites[name]; !ok { + missing = append(missing, name+" ("+f.pos+")") + } + } + sort.Strings(missing) + for _, m := range missing { + t.Errorf("outbound send site %s is not in outboundSites: route its text through prosefilter (Sanitize, SanitizeTitle or SanitizeJSON) and add it as filtered(), or classify it with a reason", m) + } + + for name, site := range outboundSites { + f, ok := funcs[name] + if !ok || !f.sends { + t.Errorf("outboundSites entry %s no longer names an outbound send site; remove it", name) + continue + } + switch site.class { + case classFiltered: + if !f.callsFilter { + t.Errorf("%s (%s) is classed filtered but does not call prosefilter", name, f.pos) + } + case classVia: + v, ok := funcs[site.via] + if !ok || !v.callsFilter { + t.Errorf("%s says its text is filtered by %s, which does not exist or does not call prosefilter", name, site.via) + } + case classTemplate: + if !slices.Contains(emitDirs, site.source) { + t.Errorf("%s names template source %s, which is not under the emit ratchet (emitDirs)", name, site.source) + } + case classMachine: + default: + t.Errorf("%s has unknown class %q", name, site.class) + } + if site.class != classFiltered && strings.TrimSpace(site.reason) == "" { + t.Errorf("%s is classed %s without a reason", name, site.class) + } + } + + for _, rc := range requiredCalls { + f, ok := funcs[rc.fn] + if !ok { + t.Errorf("%s: function %s not found", rc.file, rc.fn) + continue + } + if !f.calledIdents[rc.call] { + t.Errorf("%s (%s) must call %s: it is the outbound dash boundary for its path", rc.fn, f.pos, rc.call) + } + } +} + +func srcRootDir(t *testing.T) string { + t.Helper() + _, thisFile, _, ok := runtime.Caller(0) + if !ok { + t.Fatal("runtime.Caller failed") + } + return filepath.Join(filepath.Dir(thisFile), "..", "..") +} + +// scanFuncs parses every non-test Go file under src/pkg, src/cmd and +// src/crux and records, per function, whether it sends a body over HTTP and +// whether it calls the shared filter. +func scanFuncs(t *testing.T, srcRoot string) map[string]*scannedFunc { + t.Helper() + funcs := map[string]*scannedFunc{} + fset := token.NewFileSet() + for _, dir := range []string{"pkg", "cmd", "crux"} { + root := filepath.Join(srcRoot, dir) + err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + switch d.Name() { + case "testdata", "vendor", "node_modules": + return filepath.SkipDir + } + return nil + } + if !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") { + return nil + } + src, rerr := os.ReadFile(path) + if rerr != nil { + return rerr + } + f, perr := parser.ParseFile(fset, path, src, parser.SkipObjectResolution) + if perr != nil { + return perr + } + rel, _ := filepath.Rel(srcRoot, path) + rel = filepath.ToSlash(rel) + for _, decl := range f.Decls { + fd, ok := decl.(*ast.FuncDecl) + if !ok || fd.Body == nil { + continue + } + name := f.Name.Name + "." + funcName(fd) + sf := funcs[name] + if sf == nil { + sf = &scannedFunc{pos: rel + ":" + strconv.Itoa(fset.Position(fd.Pos()).Line), calledIdents: map[string]bool{}} + funcs[name] = sf + } + ast.Inspect(fd.Body, func(n ast.Node) bool { + call, ok := n.(*ast.CallExpr) + if !ok { + return true + } + switch fn := call.Fun.(type) { + case *ast.Ident: + sf.calledIdents[fn.Name] = true + case *ast.SelectorExpr: + sf.calledIdents[fn.Sel.Name] = true + if x, ok := fn.X.(*ast.Ident); ok && x.Name == "prosefilter" { + sf.callsFilter = true + } + if isBodySend(fn, call) { + sf.sends = true + } + } + return true + }) + } + return nil + }) + if err != nil { + t.Fatalf("walk %s: %v", dir, err) + } + } + return funcs +} + +// isBodySend reports whether call sends an HTTP request that can carry a +// body. +func isBodySend(fn *ast.SelectorExpr, call *ast.CallExpr) bool { + x, isIdent := fn.X.(*ast.Ident) + fromHTTP := isIdent && x.Name == "http" + switch fn.Sel.Name { + case "NewRequest", "NewRequestWithContext": + if !fromHTTP { + return false + } + i := 0 + if fn.Sel.Name == "NewRequestWithContext" { + i = 1 + } + if len(call.Args) <= i { + return true + } + return !safeMethods[exprString(call.Args[i])] + case "PostForm": + return fromHTTP + case "Post": + if fromHTTP { + return true + } + if len(call.Args) != 3 { + return false + } + lit, ok := call.Args[1].(*ast.BasicLit) + return ok && lit.Kind == token.STRING + } + return false +} + +func exprString(e ast.Expr) string { + switch v := e.(type) { + case *ast.BasicLit: + return v.Value + case *ast.SelectorExpr: + if x, ok := v.X.(*ast.Ident); ok && x.Name == "http" { + return v.Sel.Name + } + } + return "" +} + +func funcName(d *ast.FuncDecl) string { + if d.Recv == nil || len(d.Recv.List) == 0 { + return d.Name.Name + } + typ := d.Recv.List[0].Type + star := false + if s, ok := typ.(*ast.StarExpr); ok { + star, typ = true, s.X + } + switch g := typ.(type) { + case *ast.IndexExpr: + typ = g.X + case *ast.IndexListExpr: + typ = g.X + } + recv := "?" + if id, ok := typ.(*ast.Ident); ok { + recv = id.Name + } + if star { + return "(*" + recv + ")." + d.Name.Name + } + return recv + "." + d.Name.Name +} diff --git a/src/pkg/prosefilter/prosefilter.go b/src/pkg/prosefilter/prosefilter.go index 5a3b008fec..9d054b5382 100644 --- a/src/pkg/prosefilter/prosefilter.go +++ b/src/pkg/prosefilter/prosefilter.go @@ -1,210 +1,43 @@ -// Package prosefilter is the one place hive rewrites dashes in text that -// leaves the hive for a forge (PR and issue titles and bodies, comments, -// review bodies, commit messages). -// -// The owner's rule is that em-dashes are never acceptable in anything hive or -// its agents write. Agents are told so in the shared base prompt, but a model -// can still emit one, so every outward write path also passes its prose -// through Sanitize before it is sent. -// -// Sanitize rewrites only prose. Fenced code blocks, inline code spans and -// Markdown blockquote lines are left byte-for-byte alone, because they carry -// quoted or pasted content (a diff, a log line, a file excerpt) that must not -// be altered. +// Package prosefilter is the root-module entry point to hive's one outbound +// dash filter. The implementation lives in the crux module +// (github.com/hivecommons/hive/crux/prosefilter) so the GitHub egress proxy, +// which cannot import pkg/, applies the very same rule to agent writes +// (hivecommons/hive#19103). Every function here delegates; do not add a +// second implementation. package prosefilter -import ( - "regexp" - "strings" -) +import cruxprose "github.com/hivecommons/hive/crux/prosefilter" const ( - emDash = "\u2014" - enDash = "\u2013" - - // plainDash is what a dash used as punctuation becomes. - plainDash = "-" - // spacedDash is the replacement between two words. - spacedDash = " - " - - // maxFenceIndent is the Markdown limit on leading spaces for a code fence. - maxFenceIndent = 3 - // minFenceRun is the shortest run of fence characters that opens a fence. - minFenceRun = 3 -) - -var ( - // emDashRE matches an em-dash with the horizontal whitespace around it. - emDashRE = regexp.MustCompile(`[ \t]*` + emDash + `[ \t]*`) - // spacedEnDashRE matches an en-dash used as a dash (whitespace on both - // sides). An en-dash in a range such as 3-5 has no spaces and is left alone - // here; a hard-coded range is fixed at its source. - spacedEnDashRE = regexp.MustCompile(`[ \t]+` + enDash + `[ \t]+`) + emDash = cruxprose.EmDash + enDash = cruxprose.EnDash ) // HasDash reports whether s contains an em-dash or an en-dash. -func HasDash(s string) bool { - return strings.Contains(s, emDash) || strings.Contains(s, enDash) -} - -// Sanitize replaces every em-dash (U+2014), and every en-dash (U+2013) used -// as a spaced dash, in the prose of s with a plain hyphen. Text inside fenced -// code blocks, inline code spans and blockquote lines is untouched. A string -// with no such dash is returned unchanged. -func Sanitize(s string) string { - if !HasDash(s) { - return s - } - lines := strings.Split(s, "\n") - var fenceChar byte - var fenceLen int - for i, line := range lines { - if fenceChar != 0 { - if closesFence(line, fenceChar, fenceLen) { - fenceChar, fenceLen = 0, 0 - } - continue - } - if c, n := opensFence(line); c != 0 { - fenceChar, fenceLen = c, n - continue - } - if isBlockquote(line) { - continue - } - lines[i] = sanitizeLine(line) - } - return strings.Join(lines, "\n") -} - -// SanitizeTitle is Sanitize for a single-line title. A title has no code -// fence or blockquote, so only inline code spans are protected. -func SanitizeTitle(s string) string { - if !HasDash(s) { - return s - } - return sanitizeLine(s) -} - -func opensFence(line string) (byte, int) { - trimmed := strings.TrimLeft(line, " ") - if len(line)-len(trimmed) > maxFenceIndent || trimmed == "" { - return 0, 0 - } - c := trimmed[0] - if c != '`' && c != '~' { - return 0, 0 - } - n := runLen(trimmed, 0, c) - if n < minFenceRun { - return 0, 0 - } - // A backtick fence's info string may not contain a backtick; such a line - // is an inline code span, not a fence. - if c == '`' && strings.Contains(trimmed[n:], "`") { - return 0, 0 - } - return c, n -} - -func closesFence(line string, c byte, openLen int) bool { - trimmed := strings.TrimLeft(line, " ") - if len(line)-len(trimmed) > maxFenceIndent || trimmed == "" || trimmed[0] != c { - return false - } - n := runLen(trimmed, 0, c) - return n >= openLen && strings.TrimSpace(trimmed[n:]) == "" -} - -func isBlockquote(line string) bool { - return strings.HasPrefix(strings.TrimLeft(line, " "), ">") -} - -func runLen(s string, from int, c byte) int { - n := 0 - for from+n < len(s) && s[from+n] == c { - n++ - } - return n -} - -// sanitizeLine rewrites one prose line, skipping inline code spans. -func sanitizeLine(line string) string { - if !HasDash(line) { - return line - } - indentLen := len(line) - len(strings.TrimLeft(line, " \t")) - indent, rest := line[:indentLen], line[indentLen:] - - var b strings.Builder - b.WriteString(indent) - segStart := 0 - flush := func(end int) { - if end > segStart { - b.WriteString(rewriteSegment(rest[segStart:end], segStart == 0, end == len(rest))) - } - } - for i := 0; i < len(rest); { - if rest[i] != '`' { - i++ - continue - } - n := runLen(rest, i, '`') - closeAt := findClosingRun(rest, i+n, n) - if closeAt < 0 { - i += n - continue - } - flush(i) - spanEnd := closeAt + n - b.WriteString(rest[i:spanEnd]) - segStart = spanEnd - i = spanEnd - } - flush(len(rest)) - return b.String() -} +func HasDash(s string) bool { return cruxprose.HasDash(s) } + +// Sanitize replaces em-dashes, and en-dashes used as spaced dashes, in the +// prose of s with a plain hyphen. Code fences, inline code and blockquote +// lines are untouched. +func Sanitize(s string) string { return cruxprose.Sanitize(s) } + +// SanitizeTitle is Sanitize for a single-line title. +func SanitizeTitle(s string) string { return cruxprose.SanitizeTitle(s) } + +// Keys, Kind, Prose, Title and Stats are the shared filter's JSON field +// selectors and change counts. +type ( + Keys = cruxprose.Keys + Kind = cruxprose.Kind + Stats = cruxprose.Stats +) -// findClosingRun returns the index of the next run of exactly n backticks at -// or after from, or -1. -func findClosingRun(s string, from, n int) int { - for i := from; i < len(s); { - if s[i] != '`' { - i++ - continue - } - m := runLen(s, i, '`') - if m == n { - return i - } - i += m - } - return -1 -} +const ( + Prose = cruxprose.Prose + Title = cruxprose.Title +) -// rewriteSegment replaces dashes in one non-code stretch of a line. atStart -// and atEnd say whether the segment touches the start or end of the line's -// prose, so a leading or trailing dash does not leave a stray space. -func rewriteSegment(seg string, atStart, atEnd bool) string { - seg = spacedEnDashRE.ReplaceAllString(seg, spacedDash) - locs := emDashRE.FindAllStringIndex(seg, -1) - if len(locs) == 0 { - return seg - } - var b strings.Builder - prev := 0 - for _, loc := range locs { - b.WriteString(seg[prev:loc[0]]) - switch { - case loc[0] == 0 && atStart: - b.WriteString(plainDash + " ") - case loc[1] == len(seg) && atEnd: - b.WriteString(" " + plainDash) - default: - b.WriteString(spacedDash) - } - prev = loc[1] - } - b.WriteString(seg[prev:]) - return b.String() -} +// SanitizeJSON rewrites the string values under keys, at any depth of a JSON +// document, with the shared rule. Anything that is not JSON, or has nothing +// to change, is returned as is. +func SanitizeJSON(body []byte, keys Keys) ([]byte, Stats) { return cruxprose.SanitizeJSON(body, keys) } diff --git a/src/pkg/prosefilter/prosefilter_test.go b/src/pkg/prosefilter/prosefilter_test.go index 3932825796..f48f3aa179 100644 --- a/src/pkg/prosefilter/prosefilter_test.go +++ b/src/pkg/prosefilter/prosefilter_test.go @@ -2,61 +2,21 @@ package prosefilter import "testing" -const ( - em = "\u2014" - en = "\u2013" -) - -func TestSanitize(t *testing.T) { - cases := []struct { - name, in, want string - }{ - {"no dash is unchanged", "plain - text", "plain - text"}, - {"spaced em-dash", "a " + em + " b", "a - b"}, - {"unspaced em-dash", "a" + em + "b", "a - b"}, - {"several em-dashes", "a " + em + " b " + em + " c", "a - b - c"}, - {"leading em-dash", em + " hive: agent=x", "- hive: agent=x"}, - {"leading em-dash keeps indent", " " + em + " note", " - note"}, - {"trailing em-dash", "wait " + em, "wait -"}, - {"spaced en-dash", "a " + en + " b", "a - b"}, - {"en-dash range untouched", "levels 3" + en + "5", "levels 3" + en + "5"}, - {"inline code untouched", "see `a " + em + " b` and c " + em + " d", "see `a " + em + " b` and c - d"}, - {"double-backtick span untouched", "``a ` " + em + " b`` " + em + " c", "``a ` " + em + " b`` - c"}, - {"unclosed backtick is prose", "a ` b " + em + " c", "a ` b - c"}, - {"fenced block untouched", "x " + em + " y\n```\nfile " + em + " list\n```\nz " + em + " w", "x - y\n```\nfile " + em + " list\n```\nz - w"}, - {"tilde fence untouched", "~~~go\na " + em + " b\n~~~\nc " + em + " d", "~~~go\na " + em + " b\n~~~\nc - d"}, - {"longer closing fence", "````\na " + em + " b\n`````\nc " + em + " d", "````\na " + em + " b\n`````\nc - d"}, - {"short closing fence does not close", "````\na\n```\nb " + em + " c\n````", "````\na\n```\nb " + em + " c\n````"}, - {"blockquote untouched", "> quoted " + em + " text\nmine " + em + " here", "> quoted " + em + " text\nmine - here"}, - {"markdown list", "- file.go " + em + " added", "- file.go - added"}, - {"CRLF preserved", "a " + em + " b\r\nc", "a - b\r\nc"}, - } - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - if got := Sanitize(tc.in); got != tc.want { - t.Fatalf("Sanitize(%q) = %q, want %q", tc.in, got, tc.want) - } - if got := Sanitize(Sanitize(tc.in)); got != tc.want { - t.Fatalf("Sanitize is not idempotent for %q: %q", tc.in, got) - } - }) +// TestDelegatesToCrux pins that the root-module entry point applies the +// shared crux filter rather than a copy of it. +func TestDelegatesToCrux(t *testing.T) { + in := "a " + emDash + " b `c " + emDash + " d`" + if got, want := Sanitize(in), "a - b `c "+emDash+" d`"; got != want { + t.Fatalf("Sanitize = %q, want %q", got, want) } -} - -func TestSanitizeTitle(t *testing.T) { - if got, want := SanitizeTitle("fix: a "+em+" b `c "+em+" d`"), "fix: a - b `c "+em+" d`"; got != want { + if got, want := SanitizeTitle("x "+enDash+" y"), "x - y"; got != want { t.Fatalf("SanitizeTitle = %q, want %q", got, want) } - if got := SanitizeTitle("fix: plain"); got != "fix: plain" { - t.Fatalf("clean title changed: %q", got) - } -} - -func TestHasDash(t *testing.T) { - if HasDash("a - b") { - t.Fatal("hyphen reported as dash") + if !HasDash(emDash) || HasDash("-") { + t.Fatal("HasDash wrong") } - if !HasDash("a "+em+" b") || !HasDash("a "+en+" b") { - t.Fatal("em or en dash not detected") + out, st := SanitizeJSON([]byte(`{"body":"a `+emDash+` b"}`), Keys{"body": Prose, "title": Title}) + if string(out) != `{"body":"a - b"}` || st.Fields != 1 { + t.Fatalf("SanitizeJSON = %s (%+v)", out, st) } } diff --git a/src/pkg/prosefilter/testdata/emit_dash_baseline.json b/src/pkg/prosefilter/testdata/emit_dash_baseline.json index 1c4bdeabf2..66913553bf 100644 --- a/src/pkg/prosefilter/testdata/emit_dash_baseline.json +++ b/src/pkg/prosefilter/testdata/emit_dash_baseline.json @@ -3,6 +3,12 @@ "pkg/advisory/policy.go": 1, "pkg/dupsweep/render.go": 6, "pkg/escalation/escalation.go": 10, + "pkg/feedbackbundle/bundle.go": 1, + "pkg/fleetreport/affected.go": 5, + "pkg/fleetreport/fleetreport.go": 4, + "pkg/fleetreport/identity.go": 3, + "pkg/fleetreport/runtime.go": 5, + "pkg/fleetreport/slogfeed/slogfeed.go": 3, "pkg/github/advisory.go": 10, "pkg/github/agent_repo_scope.go": 1, "pkg/github/app.go": 6, diff --git a/src/pkg/worksource/jira.go b/src/pkg/worksource/jira.go index 827b99ffce..2f4253dc25 100644 --- a/src/pkg/worksource/jira.go +++ b/src/pkg/worksource/jira.go @@ -13,6 +13,8 @@ import ( "net/url" "strings" "time" + + "github.com/hivecommons/hive/pkg/prosefilter" ) const ( @@ -549,6 +551,9 @@ func jiraADFDocument(text string) map[string]any { } } +// jiraProseKeys are the Jira payload fields people read. +var jiraProseKeys = prosefilter.Keys{"summary": prosefilter.Title, "description": prosefilter.Prose, "body": prosefilter.Prose, "text": prosefilter.Prose} + func (s *jiraSource) doJSON(ctx context.Context, method, reqURL string, payload any, wantStatus int, out any) error { if s.tlsErr != nil { return s.tlsErr @@ -561,6 +566,12 @@ func (s *jiraSource) doJSON(ctx context.Context, method, reqURL string, payload } body = &buf } + if buf, ok := body.(*bytes.Buffer); ok { + // Issue summaries, descriptions and comments, plain or ADF text + // nodes (#19103). + clean, _ := prosefilter.SanitizeJSON(buf.Bytes(), jiraProseKeys) + body = bytes.NewReader(clean) + } req, err := http.NewRequestWithContext(ctx, method, reqURL, body) if err != nil { return fmt.Errorf("build request: %w", err) diff --git a/src/pkg/worksource/linear.go b/src/pkg/worksource/linear.go index f05c1ecbfc..2ac58d3da3 100644 --- a/src/pkg/worksource/linear.go +++ b/src/pkg/worksource/linear.go @@ -24,6 +24,7 @@ import ( "time" "github.com/hivecommons/hive/pkg/github" + "github.com/hivecommons/hive/pkg/prosefilter" ) // defaultLinearBaseURL is Linear's single GraphQL endpoint. @@ -489,6 +490,9 @@ func (s *LinearSource) doQuery(ctx context.Context, query string, vars map[strin return &resp, nil } +// linearProseKeys are the Linear variables people read. +var linearProseKeys = prosefilter.Keys{"title": prosefilter.Title, "description": prosefilter.Prose, "body": prosefilter.Prose} + // linearGraphQL posts one GraphQL request to Linear with the bare API-key // Authorization header Linear expects and returns the raw response body once // it has passed the HTTP-status and top-level `errors` checks. Shared by the @@ -505,6 +509,9 @@ func linearGraphQL(ctx context.Context, client *http.Client, baseURL, apiKey, qu if err != nil { return nil, fmt.Errorf("marshal request: %w", err) } + // Issue titles, descriptions and comments (#19103); the query text + // itself is under no prose key and is never touched. + body, _ = prosefilter.SanitizeJSON(body, linearProseKeys) req, err := http.NewRequestWithContext(ctx, http.MethodPost, baseURL, bytes.NewReader(body)) if err != nil { return nil, fmt.Errorf("build request: %w", err) From 6c35e9317b2d1736c73faaa32678d74996567b9a Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Wed, 18 Nov 2026 14:42:16 -0500 Subject: [PATCH 2/5] =?UTF-8?q?=F0=9F=93=9D=20Refresh=20api-reference=20ci?= =?UTF-8?q?tations=20shifted=20by=20the=20prosefilter=20import?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Andy Anderson --- src/docs/api-reference/knowledge.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/docs/api-reference/knowledge.md b/src/docs/api-reference/knowledge.md index af6a86de32..c72ea9e714 100644 --- a/src/docs/api-reference/knowledge.md +++ b/src/docs/api-reference/knowledge.md @@ -123,11 +123,11 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `POST` | `/api/meetings/actions/{aid}/edit` | Owner or read-write | Change a pending or failed draft: `{type?, title?, body?, repo?}`; the repo must be a monitored one. 404 unknown, 409 already decided. Audited as `meeting_action_edit` | `pkg/dashboard/api_meetings_actions.go:72` | | `POST` | `/api/meetings/actions/{aid}/approve` | Owner or read-write | Create the draft (issue or RFC on GitHub, Spek session through the Inbox `spek_rfc` destination, knowledge note in the `meetings` vault) and report the result to the meeting. Returns the draft, `done` with `result_url` or `failed` with `error`. 409 when it is not pending or failed, or action requests are off. Audited as `meeting_action_approve` | `pkg/dashboard/api_meetings_actions.go:73` | | `POST` | `/api/meetings/actions/{aid}/reject` | Owner or read-write | Reject a pending or failed draft; nothing is created. Audited as `meeting_action_reject` | `pkg/dashboard/api_meetings_actions.go:74` | -| `GET` | `/api/meetings/persona` | Owner or read-write | Settings, Meeting attendee: `{effective, presets, overridden, voices, voice_types, restart_required, running, tts_configured, email, research}`. `research` is the research agent pool Betty's deep dives go to: `{wired, agents, available, base_agent, backend, model, pool_size, warning}`, with `warning` set when no agent can take a request. `effective` is the resolved persona (fields resolved as if on, `enabled` separately) with the rendered `intro` and `voice_on_address`; `presets` lists the built-in personas, the default first; `overridden` maps each `meetings.persona` key not taken from the preset or defaults to `config` or `env`; `voices` are the Kokoro voices with language, accent and voice type; `restart_required` is true when the saved persona differs from the one the hive booted with; `email` is the attendee address (`hive@`), the access list with registered calendar emails and proof status, the approval queue and the email-in switches (`available: false` until email-in is enabled). See [Meeting attendee settings](../meetings.md#meeting-attendee-settings) | `pkg/dashboard/api_meetings_persona.go:169` | -| `PUT` | `/api/meetings/persona` | Owner or read-write | Save `meetings.persona` to `hive.yaml`: `{enabled?, preset?, clear_overrides?, name?, display_name?, aliases?, pronouns?, language?, voice?, voice_type?, accent?, intro_template?, leave_phrases?, humor?, unprompted?: {per_10m?, min_gap_s?, per_meeting?, answer_wait_s?, min_confidence?, queue_depth?, stale_s?}, voice_on_address?}`. An absent field is unchanged, an empty one (or 0 for a cap) clears the override, a value equal to the preset's is stored as no override, and `clear_overrides` drops every field override first. Validated as if the persona were on; returns the GET body. 400 invalid. Audited as `meetings_persona`. An optional `email: {calendar_invites?, mail_in_files?}` sets `meetings.email.invites_enabled` / `files_enabled` (owner only, else 403; [#18395](https://github.com/hivecommons/hive/issues/18395)) | `pkg/dashboard/api_meetings_persona.go:170` | -| `POST` | `/api/meetings/persona/sample` | Owner or read-write | Speak a short line through the meetings TTS service (Kokoro `POST /v1/audio/speech`) and return the audio (`audio/mpeg`, at most 2 MiB, `Cache-Control: no-store`). Optional `{voice?, language?, display_name?, text?}` (unsaved form values; `text` at most 200 characters, default the persona's sample line in its language). 503 "TTS not configured" until `meetings.tts_url` is set, 400 invalid, 502 the TTS service failed | `pkg/dashboard/api_meetings_persona.go:171` | -| `GET` | `/api/meetings/persona/sample` | Owner or read-write | The same sample with `voice`, `language`, `display_name` and `text` as query parameters, for the dashboard's `