diff --git a/changelog.d/fixed-19103-agent-writes-no-em-dash.md b/changelog.d/fixed-19103-agent-writes-no-em-dash.md new file mode 100644 index 0000000000..bf065fc955 --- /dev/null +++ b/changelog.d/fixed-19103-agent-writes-no-em-dash.md @@ -0,0 +1 @@ +- Em-dashes no longer reach GitHub or any other outbound channel through paths the first filter missed ([#19103](https://github.com/hivecommons/hive/issues/19103)). The shared dash filter moved into the `crux` module so the GitHub egress proxy now applies it to agent writes it relays (`gh api` comments, review replies, PR and issue titles and bodies, contents-API commit messages, and the text inputs of comment, review, PR and issue GraphQL mutations, plus Linear mutation variables), touching only JSON text fields and never git traffic or non-JSON bodies. The push broker rewrites the agent's HEAD commit message, and notifications (ntfy, Slack, Discord), Discord bot messages, meeting chat and spoken text, hub-filed issues and comments, Linear, Jira and DIBS writes, and squash commit messages hive writes all pass through the same filter. A new CI inventory test fails when an outbound HTTP send is added without the filter or a stated reason. Set `HIVE_AGENT_PROSE_FILTER=false` to stop rewriting agent-authored text. diff --git a/src/crux/prosefilter/json.go b/src/crux/prosefilter/json.go new file mode 100644 index 0000000000..1517f8ffcd --- /dev/null +++ b/src/crux/prosefilter/json.go @@ -0,0 +1,154 @@ +package prosefilter + +import ( + "bytes" + "encoding/json" + "os" + "strings" +) + +// Kind says how a JSON field's text is filtered. +type Kind int + +const ( + // Prose is multi-line Markdown text: Sanitize. + Prose Kind = iota + 1 + // Title is single-line text: SanitizeTitle. + Title +) + +// Keys maps a JSON object key to the filter its string values get. +type Keys map[string]Kind + +// Stats counts what SanitizeJSON changed: text fields rewritten, and dashes +// removed. It never carries the text itself, so it is safe to log. +type Stats struct { + Fields int + Dashes int +} + +// AgentWritesEnv is the operator switch for rewriting text that an agent +// wrote itself (its GitHub writes through the egress proxy, its HEAD commit +// message in the push broker). It is on unless set to one of +// agentWritesOffValues. Text hive itself composes is always filtered. +const AgentWritesEnv = "HIVE_AGENT_PROSE_FILTER" + +var agentWritesOffValues = map[string]bool{"false": true, "0": true, "off": true, "no": true} + +// AgentWritesEnabled reports whether AgentWritesEnv leaves the agent-write +// rewrite on. +func AgentWritesEnabled() bool { + return !agentWritesOffValues[strings.ToLower(strings.TrimSpace(os.Getenv(AgentWritesEnv)))] +} + +// SanitizeJSON rewrites, at any depth of a JSON document, the string values +// (and the strings of a string array) stored under a key in keys, using the +// same rule as Sanitize. Every other value, number formatting included, is +// preserved. A body that is not JSON, holds no dash, or has nothing to change +// is returned unchanged with zero Stats; only a changed body is re-encoded +// (keys sorted, no HTML escaping). +func SanitizeJSON(body []byte, keys Keys) ([]byte, Stats) { + if len(keys) == 0 || !mayHoldDash(body) { + return body, Stats{} + } + dec := json.NewDecoder(bytes.NewReader(body)) + dec.UseNumber() + var doc any + if dec.Decode(&doc) != nil || dec.More() { + return body, Stats{} + } + var st Stats + doc = sanitizeTree(doc, keys, &st) + if st.Fields == 0 { + return body, Stats{} + } + out, err := marshalNoEscape(doc) + if err != nil { + return body, Stats{} + } + return out, st +} + +// mayHoldDash reports whether body could contain an em- or en-dash, raw or as +// a JSON \u escape. +func mayHoldDash(body []byte) bool { + if bytes.Contains(body, []byte(emDash)) || bytes.Contains(body, []byte(enDash)) { + return true + } + lower := bytes.ToLower(body) + return bytes.Contains(lower, []byte(`—`)) || bytes.Contains(lower, []byte(`–`)) +} + +func sanitizeTree(v any, keys Keys, st *Stats) any { + switch t := v.(type) { + case map[string]any: + for k, child := range t { + if kind, ok := keys[k]; ok { + switch c := child.(type) { + case string: + t[k] = sanitizeCounted(c, kind, st) + continue + case []any: + if allStrings(c) { + for i, s := range c { + c[i] = sanitizeCounted(s.(string), kind, st) + } + continue + } + } + } + t[k] = sanitizeTree(child, keys, st) + } + return t + case []any: + for i, child := range t { + t[i] = sanitizeTree(child, keys, st) + } + return t + default: + return v + } +} + +func allStrings(items []any) bool { + for _, it := range items { + if _, ok := it.(string); !ok { + return false + } + } + return len(items) > 0 +} + +func sanitizeCounted(s string, kind Kind, st *Stats) string { + if !HasDash(s) { + return s + } + var clean string + if kind == Title { + clean = SanitizeTitle(s) + } else { + clean = Sanitize(s) + } + if clean != s { + st.Fields++ + st.Dashes += DashCount(s) - DashCount(clean) + } + return clean +} + +// DashCount is the number of em-dashes and en-dashes in s. +func DashCount(s string) int { + return strings.Count(s, emDash) + strings.Count(s, enDash) +} + +// marshalNoEscape is json.Marshal without HTML escaping, so a body that only +// had its dashes rewritten does not also gain < escapes. +func marshalNoEscape(v any) ([]byte, error) { + var buf bytes.Buffer + enc := json.NewEncoder(&buf) + enc.SetEscapeHTML(false) + if err := enc.Encode(v); err != nil { + return nil, err + } + return bytes.TrimRight(buf.Bytes(), "\n"), nil +} diff --git a/src/crux/prosefilter/json_test.go b/src/crux/prosefilter/json_test.go new file mode 100644 index 0000000000..c6e3c1ae54 --- /dev/null +++ b/src/crux/prosefilter/json_test.go @@ -0,0 +1,68 @@ +package prosefilter + +import ( + "encoding/json" + "testing" +) + +func TestSanitizeJSON(t *testing.T) { + keys := Keys{"body": Prose, "title": Title, "texts": Prose} + cases := []struct { + name, in, want string + fields int + }{ + {"top-level body", `{"body":"a ` + em + ` b"}`, `{"body":"a - b"}`, 1}, + {"escaped dash", `{"body":"a — b"}`, `{"body":"a - b"}`, 1}, + {"title", `{"title":"x ` + em + ` y"}`, `{"title":"x - y"}`, 1}, + {"nested", `{"input":{"body":"a ` + em + ` b","id":"T_1"}}`, `{"input":{"body":"a - b","id":"T_1"}}`, 1}, + {"array items", `{"comments":[{"body":"a ` + em + ` b","line":3}]}`, `{"comments":[{"body":"a - b","line":3}]}`, 1}, + {"string array under prose key", `{"texts":["a ` + em + ` b","c"]}`, `{"texts":["a - b","c"]}`, 1}, + {"numbers kept exactly", `{"body":"a ` + em + ` b","n":12345678901234567890,"f":1.50}`, `{"body":"a - b","f":1.50,"n":12345678901234567890}`, 1}, + {"no HTML escaping", `{"body":" & ` + em + `"}`, `{"body":" & -"}`, 1}, + {"other key untouched", `{"ref":"a ` + em + ` b"}`, `{"ref":"a ` + em + ` b"}`, 0}, + {"non-string under key untouched", `{"body":{"ref":"a ` + em + ` b"}}`, `{"body":{"ref":"a ` + em + ` b"}}`, 0}, + {"no dash", `{"body":"plain"}`, `{"body":"plain"}`, 0}, + {"not JSON", `plain ` + em + ` text`, `plain ` + em + ` text`, 0}, + {"trailing data", `{"body":"a ` + em + ` b"} {}`, `{"body":"a ` + em + ` b"} {}`, 0}, + {"code span kept", `{"body":"` + "`a " + em + " b`" + `"}`, `{"body":"` + "`a " + em + " b`" + `"}`, 0}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, st := SanitizeJSON([]byte(tc.in), keys) + if string(got) != tc.want { + t.Fatalf("SanitizeJSON = %s, want %s", got, tc.want) + } + if st.Fields != tc.fields { + t.Fatalf("Fields = %d, want %d", st.Fields, tc.fields) + } + if tc.fields > 0 && st.Dashes == 0 { + t.Fatal("Dashes not counted") + } + if tc.fields > 0 && !json.Valid(got) { + t.Fatalf("output is not JSON: %s", got) + } + }) + } +} + +func TestSanitizeJSONNoKeys(t *testing.T) { + in := []byte(`{"body":"a ` + em + ` b"}`) + if got, st := SanitizeJSON(in, nil); string(got) != string(in) || st.Fields != 0 { + t.Fatalf("nil keys changed the body: %s", got) + } +} + +func TestDashCount(t *testing.T) { + if got := DashCount("a " + em + " b " + en + " c"); got != 2 { + t.Fatalf("DashCount = %d, want 2", got) + } +} + +func TestAgentWritesEnabled(t *testing.T) { + for v, want := range map[string]bool{"": true, "true": true, "false": false, "0": false, "OFF": false, "no": false} { + t.Setenv(AgentWritesEnv, v) + if got := AgentWritesEnabled(); got != want { + t.Errorf("%s=%q: AgentWritesEnabled = %v, want %v", AgentWritesEnv, v, got, want) + } + } +} diff --git a/src/crux/prosefilter/prosefilter.go b/src/crux/prosefilter/prosefilter.go new file mode 100644 index 0000000000..9d9ce0faae --- /dev/null +++ b/src/crux/prosefilter/prosefilter.go @@ -0,0 +1,217 @@ +// Package prosefilter is the one place hive rewrites dashes in text that +// leaves the hive (PR and issue titles and bodies, comments, review bodies, +// commit messages, notifications, meeting chat and spoken text). +// +// The owner's rule is that em-dashes are never acceptable in anything hive or +// its agents write. Agents are told so in the shared base prompt, but a model +// can still emit one, so every outward write path also passes its prose +// through Sanitize before it is sent: hive's own sends in the root module +// (via src/pkg/prosefilter, which delegates here) and agent-originated GitHub +// writes in the egress proxy (src/crux/proxy/outbound_prose.go). It lives in +// the crux module so both sides call the same function (hivecommons/hive#19103). +// +// Sanitize rewrites only prose. Fenced code blocks, inline code spans and +// Markdown blockquote lines are left byte-for-byte alone, because they carry +// quoted or pasted content (a diff, a log line, a file excerpt) that must not +// be altered. +package prosefilter + +import ( + "regexp" + "strings" +) + +const ( + // EmDash and EnDash are the two characters the filter rewrites. + EmDash = "\u2014" + EnDash = "\u2013" + + emDash = EmDash + enDash = EnDash + + // plainDash is what a dash used as punctuation becomes. + plainDash = "-" + // spacedDash is the replacement between two words. + spacedDash = " - " + + // maxFenceIndent is the Markdown limit on leading spaces for a code fence. + maxFenceIndent = 3 + // minFenceRun is the shortest run of fence characters that opens a fence. + minFenceRun = 3 +) + +var ( + // emDashRE matches an em-dash with the horizontal whitespace around it. + emDashRE = regexp.MustCompile(`[ \t]*` + emDash + `[ \t]*`) + // spacedEnDashRE matches an en-dash used as a dash (whitespace on both + // sides). An en-dash in a range such as 3-5 has no spaces and is left alone + // here; a hard-coded range is fixed at its source. + spacedEnDashRE = regexp.MustCompile(`[ \t]+` + enDash + `[ \t]+`) +) + +// HasDash reports whether s contains an em-dash or an en-dash. +func HasDash(s string) bool { + return strings.Contains(s, emDash) || strings.Contains(s, enDash) +} + +// Sanitize replaces every em-dash (U+2014), and every en-dash (U+2013) used +// as a spaced dash, in the prose of s with a plain hyphen. Text inside fenced +// code blocks, inline code spans and blockquote lines is untouched. A string +// with no such dash is returned unchanged. +func Sanitize(s string) string { + if !HasDash(s) { + return s + } + lines := strings.Split(s, "\n") + var fenceChar byte + var fenceLen int + for i, line := range lines { + if fenceChar != 0 { + if closesFence(line, fenceChar, fenceLen) { + fenceChar, fenceLen = 0, 0 + } + continue + } + if c, n := opensFence(line); c != 0 { + fenceChar, fenceLen = c, n + continue + } + if isBlockquote(line) { + continue + } + lines[i] = sanitizeLine(line) + } + return strings.Join(lines, "\n") +} + +// SanitizeTitle is Sanitize for a single-line title. A title has no code +// fence or blockquote, so only inline code spans are protected. +func SanitizeTitle(s string) string { + if !HasDash(s) { + return s + } + return sanitizeLine(s) +} + +func opensFence(line string) (byte, int) { + trimmed := strings.TrimLeft(line, " ") + if len(line)-len(trimmed) > maxFenceIndent || trimmed == "" { + return 0, 0 + } + c := trimmed[0] + if c != '`' && c != '~' { + return 0, 0 + } + n := runLen(trimmed, 0, c) + if n < minFenceRun { + return 0, 0 + } + // A backtick fence's info string may not contain a backtick; such a line + // is an inline code span, not a fence. + if c == '`' && strings.Contains(trimmed[n:], "`") { + return 0, 0 + } + return c, n +} + +func closesFence(line string, c byte, openLen int) bool { + trimmed := strings.TrimLeft(line, " ") + if len(line)-len(trimmed) > maxFenceIndent || trimmed == "" || trimmed[0] != c { + return false + } + n := runLen(trimmed, 0, c) + return n >= openLen && strings.TrimSpace(trimmed[n:]) == "" +} + +func isBlockquote(line string) bool { + return strings.HasPrefix(strings.TrimLeft(line, " "), ">") +} + +func runLen(s string, from int, c byte) int { + n := 0 + for from+n < len(s) && s[from+n] == c { + n++ + } + return n +} + +// sanitizeLine rewrites one prose line, skipping inline code spans. +func sanitizeLine(line string) string { + if !HasDash(line) { + return line + } + indentLen := len(line) - len(strings.TrimLeft(line, " \t")) + indent, rest := line[:indentLen], line[indentLen:] + + var b strings.Builder + b.WriteString(indent) + segStart := 0 + flush := func(end int) { + if end > segStart { + b.WriteString(rewriteSegment(rest[segStart:end], segStart == 0, end == len(rest))) + } + } + for i := 0; i < len(rest); { + if rest[i] != '`' { + i++ + continue + } + n := runLen(rest, i, '`') + closeAt := findClosingRun(rest, i+n, n) + if closeAt < 0 { + i += n + continue + } + flush(i) + spanEnd := closeAt + n + b.WriteString(rest[i:spanEnd]) + segStart = spanEnd + i = spanEnd + } + flush(len(rest)) + return b.String() +} + +// findClosingRun returns the index of the next run of exactly n backticks at +// or after from, or -1. +func findClosingRun(s string, from, n int) int { + for i := from; i < len(s); { + if s[i] != '`' { + i++ + continue + } + m := runLen(s, i, '`') + if m == n { + return i + } + i += m + } + return -1 +} + +// rewriteSegment replaces dashes in one non-code stretch of a line. atStart +// and atEnd say whether the segment touches the start or end of the line's +// prose, so a leading or trailing dash does not leave a stray space. +func rewriteSegment(seg string, atStart, atEnd bool) string { + seg = spacedEnDashRE.ReplaceAllString(seg, spacedDash) + locs := emDashRE.FindAllStringIndex(seg, -1) + if len(locs) == 0 { + return seg + } + var b strings.Builder + prev := 0 + for _, loc := range locs { + b.WriteString(seg[prev:loc[0]]) + switch { + case loc[0] == 0 && atStart: + b.WriteString(plainDash + " ") + case loc[1] == len(seg) && atEnd: + b.WriteString(" " + plainDash) + default: + b.WriteString(spacedDash) + } + prev = loc[1] + } + b.WriteString(seg[prev:]) + return b.String() +} diff --git a/src/crux/prosefilter/prosefilter_test.go b/src/crux/prosefilter/prosefilter_test.go new file mode 100644 index 0000000000..3932825796 --- /dev/null +++ b/src/crux/prosefilter/prosefilter_test.go @@ -0,0 +1,62 @@ +package prosefilter + +import "testing" + +const ( + em = "\u2014" + en = "\u2013" +) + +func TestSanitize(t *testing.T) { + cases := []struct { + name, in, want string + }{ + {"no dash is unchanged", "plain - text", "plain - text"}, + {"spaced em-dash", "a " + em + " b", "a - b"}, + {"unspaced em-dash", "a" + em + "b", "a - b"}, + {"several em-dashes", "a " + em + " b " + em + " c", "a - b - c"}, + {"leading em-dash", em + " hive: agent=x", "- hive: agent=x"}, + {"leading em-dash keeps indent", " " + em + " note", " - note"}, + {"trailing em-dash", "wait " + em, "wait -"}, + {"spaced en-dash", "a " + en + " b", "a - b"}, + {"en-dash range untouched", "levels 3" + en + "5", "levels 3" + en + "5"}, + {"inline code untouched", "see `a " + em + " b` and c " + em + " d", "see `a " + em + " b` and c - d"}, + {"double-backtick span untouched", "``a ` " + em + " b`` " + em + " c", "``a ` " + em + " b`` - c"}, + {"unclosed backtick is prose", "a ` b " + em + " c", "a ` b - c"}, + {"fenced block untouched", "x " + em + " y\n```\nfile " + em + " list\n```\nz " + em + " w", "x - y\n```\nfile " + em + " list\n```\nz - w"}, + {"tilde fence untouched", "~~~go\na " + em + " b\n~~~\nc " + em + " d", "~~~go\na " + em + " b\n~~~\nc - d"}, + {"longer closing fence", "````\na " + em + " b\n`````\nc " + em + " d", "````\na " + em + " b\n`````\nc - d"}, + {"short closing fence does not close", "````\na\n```\nb " + em + " c\n````", "````\na\n```\nb " + em + " c\n````"}, + {"blockquote untouched", "> quoted " + em + " text\nmine " + em + " here", "> quoted " + em + " text\nmine - here"}, + {"markdown list", "- file.go " + em + " added", "- file.go - added"}, + {"CRLF preserved", "a " + em + " b\r\nc", "a - b\r\nc"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := Sanitize(tc.in); got != tc.want { + t.Fatalf("Sanitize(%q) = %q, want %q", tc.in, got, tc.want) + } + if got := Sanitize(Sanitize(tc.in)); got != tc.want { + t.Fatalf("Sanitize is not idempotent for %q: %q", tc.in, got) + } + }) + } +} + +func TestSanitizeTitle(t *testing.T) { + if got, want := SanitizeTitle("fix: a "+em+" b `c "+em+" d`"), "fix: a - b `c "+em+" d`"; got != want { + t.Fatalf("SanitizeTitle = %q, want %q", got, want) + } + if got := SanitizeTitle("fix: plain"); got != "fix: plain" { + t.Fatalf("clean title changed: %q", got) + } +} + +func TestHasDash(t *testing.T) { + if HasDash("a - b") { + t.Fatal("hyphen reported as dash") + } + if !HasDash("a "+em+" b") || !HasDash("a "+en+" b") { + t.Fatal("em or en dash not detected") + } +} diff --git a/src/crux/proxy/github_proxy.go b/src/crux/proxy/github_proxy.go index 5875dedfc4..61a61bf438 100644 --- a/src/crux/proxy/github_proxy.go +++ b/src/crux/proxy/github_proxy.go @@ -20,6 +20,7 @@ import ( "github.com/hivecommons/hive/crux/config" "github.com/hivecommons/hive/crux/egressmark" "github.com/hivecommons/hive/crux/inferencehealth" + "github.com/hivecommons/hive/crux/prosefilter" "github.com/hivecommons/hive/crux/upstreamproxy" ) @@ -85,6 +86,10 @@ type GitHubProxy struct { caCert tls.Certificate caX509 *x509.Certificate logger *slog.Logger + // proseFilterOff turns off the agent-write dash filter + // (outbound_prose.go). Zero means on, so every proxy, test literals + // included, filters unless the operator set HIVE_AGENT_PROSE_FILTER=false. + proseFilterOff bool // uidMap is read through loadedUIDMap and swapped only by refreshUIDMap // under uidMapMu (#18184): agents added at runtime land in uid-map.json // after construction. uidMapPath/uidMapInfo are what the refresh compares @@ -343,6 +348,7 @@ func newGitHubProxyWithCA(caCert tls.Certificate, caX509 *x509.Certificate, logg proxyAdvisoryOK: advisoryOK, allowPrivateForward: allowPrivateForward, injectGHAuth: injectGHAuth, + proseFilterOff: !prosefilter.AgentWritesEnabled(), allowedRepos: allowed, violations: make(map[string]int), certCache: make(map[string]cachedCert), diff --git a/src/crux/proxy/github_proxy_mitm.go b/src/crux/proxy/github_proxy_mitm.go index c170eb5180..c154a0bf39 100644 --- a/src/crux/proxy/github_proxy_mitm.go +++ b/src/crux/proxy/github_proxy_mitm.go @@ -359,6 +359,14 @@ func (p *GitHubProxy) proxyHTTPHost(client net.Conn, upstream net.Conn, host str if isLinear && agentName != internalCallerName { p.injectLinearCredential(req, agentName, mode) } + // #19103: an allowed agent write's text fields get the same dash + // rewrite hive's own writes get, after every gate has read the body + // as the agent sent it. Git smart-HTTP traffic is never touched. + if err := p.sanitizeAgentWriteProse(req, agentName, isLinear); err != nil { + _ = client.SetReadDeadline(time.Time{}) + p.logTimeout("proxy prose-filter body read timed out", err, "agent", agentName, "path", req.URL.Path) + return + } // Git smart HTTP uses chunked streaming that http.ReadResponse // can't handle reliably. After the ACMM check passes, forward // the request and switch to raw bidirectional streaming. diff --git a/src/crux/proxy/outbound_prose.go b/src/crux/proxy/outbound_prose.go new file mode 100644 index 0000000000..5af65ee246 --- /dev/null +++ b/src/crux/proxy/outbound_prose.go @@ -0,0 +1,196 @@ +package proxy + +import ( + "bytes" + "encoding/json" + "io" + "net/http" + "regexp" + "sort" + "strconv" + "strings" + + "github.com/hivecommons/hive/crux/prosefilter" +) + +// Outbound prose filter for agent-originated writes (hivecommons/hive#19103). +// +// Hive's own forge writes pass their text through prosefilter before they are +// sent (#18970), but an agent that posts with `gh api`, `gh pr comment` or +// `gh pr create` talks to GitHub through this proxy, and the relay forwarded +// those bodies verbatim. This is the boundary for that traffic: an allowed +// agent write's text fields get the same rewrite hive's own writes get, and +// nothing else in the request changes. +// +// Scope, deliberately narrow: +// +// - Only JSON bodies. A body that is not a JSON document, or that declares a +// non-JSON Content-Type (a release asset upload, a form), is forwarded +// byte-for-byte. +// - Only the text fields: REST `body`, `title` and `message` at any depth +// (a review's `comments[].body` included); GraphQL `body`/`title` (and +// Linear `description`) values inside the variables of a mutation +// document. Every other field, and the GraphQL query text itself, is left +// alone. A literal string inlined in the query text is not rewritten. +// - Never git smart-HTTP or LFS traffic: commit messages inside pushed +// objects cannot be rewritten in transit without changing object ids. +// The push broker covers the HEAD commit message instead +// (src/crux/pushbroker/commit_message.go). +// - Never the hive's own control-plane traffic, whose text is already +// filtered (or deliberately left as rendered) before it reaches here. +// +// The switch is prosefilter.AgentWritesEnv (default on). + +// outboundProseBodyLimit bounds the body the filter buffers. A larger body is +// forwarded unchanged rather than truncated: GitHub caps a comment body at +// 65536 characters, so a legitimate text write is far below this. +const outboundProseBodyLimit = 1 << 20 + +// restProseKeys are the REST JSON fields that carry prose. `message` is the +// commit message of the contents and git-data APIs. +var restProseKeys = prosefilter.Keys{"body": prosefilter.Prose, "title": prosefilter.Title, "message": prosefilter.Prose} + +// graphQLProseKeys are the variable keys whose values are prose in a GitHub +// mutation. +var graphQLProseKeys = prosefilter.Keys{"body": prosefilter.Prose, "title": prosefilter.Title} + +// linearProseKeys are the same for Linear, where an issue's text is +// `description`. +var linearProseKeys = prosefilter.Keys{"body": prosefilter.Prose, "title": prosefilter.Title, "description": prosefilter.Prose} + +// graphQLProseMutations are the GitHub mutations whose inputs carry comment, +// review, PR or issue text. A mutation document selecting at least one of +// them has its `body` and `title` variables filtered. +var graphQLProseMutations = map[string]bool{ + "addComment": true, + "addDiscussionComment": true, + "addPullRequestReview": true, + "addPullRequestReviewComment": true, + "addPullRequestReviewThread": true, + "addPullRequestReviewThreadReply": true, + "submitPullRequestReview": true, + "updateIssueComment": true, + "updatePullRequestReviewComment": true, + "createPullRequest": true, + "updatePullRequest": true, + "createIssue": true, + "updateIssue": true, +} + +// sanitizeAgentWriteProse applies the shared prose filter to an allowed agent +// write just before it is forwarded. It replaces req.Body (and the length) +// only when a text field changed. A non-nil error means the body could not be +// read; the caller drops the exchange, as the other body gates do. +func (p *GitHubProxy) sanitizeAgentWriteProse(req *http.Request, agentName string, isLinear bool) error { + if p.proseFilterOff || agentName == internalCallerName || req.Body == nil || req.Body == http.NoBody { + return nil + } + if !proseFilterApplies(req) { + return nil + } + body, err := io.ReadAll(io.LimitReader(req.Body, outboundProseBodyLimit+1)) + if err != nil { + _ = req.Body.Close() + return err + } + if int64(len(body)) > outboundProseBodyLimit { + req.Body = bodyReadCloser{Reader: io.MultiReader(bytes.NewReader(body), req.Body), Closer: req.Body} + return nil + } + _ = req.Body.Close() + out, st := sanitizeProseBody(req.URL.Path, body, isLinear) + req.Body = io.NopCloser(bytes.NewReader(out)) + if st.Fields == 0 { + return nil + } + req.ContentLength = int64(len(out)) + req.Header.Set("Content-Length", strconv.Itoa(len(out))) + // A chunked request is now a fixed-length one. + req.TransferEncoding = nil + if p.logger != nil { + p.logger.Debug("proxy: rewrote dashes in agent write text", "agent", agentName, + "method", req.Method, "path", req.URL.Path, "fields", st.Fields, "dashes", st.Dashes) + } + return nil +} + +// proseFilterApplies is the cheap pre-check: a write method, not git or LFS +// traffic, and a Content-Type that is JSON or absent. +func proseFilterApplies(req *http.Request) bool { + switch req.Method { + case http.MethodPost, http.MethodPatch, http.MethodPut: + default: + return false + } + if isGitPath(req.URL.Path) || strings.Contains(req.URL.Path, ".git/") { + return false + } + ct := strings.ToLower(req.Header.Get("Content-Type")) + return ct == "" || strings.Contains(ct, "json") +} + +// sanitizeProseBody returns body with its text fields filtered, and what +// changed. A body it does not recognise is returned as is. +func sanitizeProseBody(path string, body []byte, isLinear bool) ([]byte, prosefilter.Stats) { + if isLinear || IsGraphQLPath(path) { + return sanitizeGraphQLProse(body, isLinear) + } + return prosefilter.SanitizeJSON(body, restProseKeys) +} + +// sanitizeGraphQLProse filters only the variables of a mutation document, +// never its query text. +func sanitizeGraphQLProse(body []byte, isLinear bool) ([]byte, prosefilter.Stats) { + var obj map[string]json.RawMessage + if err := json.Unmarshal(body, &obj); err != nil || obj == nil { + return body, prosefilter.Stats{} + } + var query string + if json.Unmarshal(obj["query"], &query) != nil || !isGitHubGraphQLMutation(query) { + return body, prosefilter.Stats{} + } + keys := linearProseKeys + if !isLinear { + if !selectsProseMutation(query) { + return body, prosefilter.Stats{} + } + keys = graphQLProseKeys + } + rawVars, ok := obj["variables"] + if !ok { + return body, prosefilter.Stats{} + } + vars, st := prosefilter.SanitizeJSON(rawVars, keys) + if st.Fields == 0 { + return body, prosefilter.Stats{} + } + obj["variables"] = vars + var buf bytes.Buffer + enc := json.NewEncoder(&buf) + enc.SetEscapeHTML(false) + if err := enc.Encode(obj); err != nil { + return body, prosefilter.Stats{} + } + return bytes.TrimRight(buf.Bytes(), "\n"), st +} + +// graphQLProseMutationRe matches any name in graphQLProseMutations as a whole +// word. A substring test is the right posture here, unlike for a permission +// grant: a batched document that also does something else still carries the +// text, and rewriting a dash is never a widening. +var graphQLProseMutationRe = regexp.MustCompile(`\b(` + strings.Join(sortedKeys(graphQLProseMutations), "|") + `)\b`) + +// selectsProseMutation reports whether a GitHub mutation document names a +// mutation in graphQLProseMutations. +func selectsProseMutation(query string) bool { + return graphQLProseMutationRe.MatchString(query) +} + +func sortedKeys(m map[string]bool) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out +} diff --git a/src/crux/proxy/outbound_prose_test.go b/src/crux/proxy/outbound_prose_test.go new file mode 100644 index 0000000000..104f6dcf37 --- /dev/null +++ b/src/crux/proxy/outbound_prose_test.go @@ -0,0 +1,253 @@ +package proxy + +import ( + "bufio" + "bytes" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "strconv" + "strings" + "testing" + + "github.com/hivecommons/hive/crux/agent" + "github.com/hivecommons/hive/crux/prosefilter" +) + +const ( + em = prosefilter.EmDash + en = prosefilter.EnDash +) + +func mustJSON(t *testing.T, v any) string { + t.Helper() + b, err := json.Marshal(v) + if err != nil { + t.Fatal(err) + } + return string(b) +} + +// decodeField returns the string at a dotted path of a JSON document. +func decodeField(t *testing.T, body []byte, path ...string) string { + t.Helper() + var cur any + if err := json.Unmarshal(body, &cur); err != nil { + t.Fatalf("body is not JSON: %v: %s", err, body) + } + for _, p := range path { + switch c := cur.(type) { + case map[string]any: + cur = c[p] + case []any: + i, _ := strconv.Atoi(p) + cur = c[i] + } + } + s, _ := cur.(string) + return s +} + +func TestSanitizeAgentWriteProse(t *testing.T) { + fence := "```\nfile " + em + " list\n```" + cases := []struct { + name string + method string + path string + contentType string + body string + linear bool + agent string + off bool + // field is the JSON path checked; want is its value after the filter. + field []string + want string + // unchanged means the forwarded body must equal the input byte-for-byte. + unchanged bool + }{ + {name: "REST issue comment", method: "POST", path: "/repos/o/r/issues/1/comments", + body: mustJSON(t, map[string]any{"body": "looks good " + em + " merging"}), field: []string{"body"}, want: "looks good - merging"}, + {name: "PR body edit", method: "PATCH", path: "/repos/o/r/pulls/2", + body: mustJSON(t, map[string]any{"body": "Fixes x " + em + " and y", "state": "open"}), field: []string{"body"}, want: "Fixes x - and y"}, + {name: "review thread reply", method: "POST", path: "/repos/o/r/pulls/2/comments/9/replies", + body: mustJSON(t, map[string]any{"body": "done " + em + " see abc123"}), field: []string{"body"}, want: "done - see abc123"}, + {name: "PR title", method: "POST", path: "/repos/o/r/pulls", + body: mustJSON(t, map[string]any{"title": "fix: a " + em + " b", "head": "x" + em + "y", "base": "main"}), field: []string{"title"}, want: "fix: a - b"}, + {name: "review with inline comments", method: "POST", path: "/repos/o/r/pulls/2/reviews", + body: mustJSON(t, map[string]any{"event": "COMMENT", "comments": []any{map[string]any{"path": "a.go", "line": 3, "body": "nit " + em + " rename"}}}), + field: []string{"comments", "0", "body"}, want: "nit - rename"}, + {name: "contents API commit message", method: "PUT", path: "/repos/o/r/contents/a.md", + body: mustJSON(t, map[string]any{"message": "docs: a " + em + " b", "content": "YQ=="}), field: []string{"message"}, want: "docs: a - b"}, + {name: "fenced code block kept", method: "POST", path: "/repos/o/r/issues/1/comments", + body: mustJSON(t, map[string]any{"body": "x " + em + " y\n" + fence}), field: []string{"body"}, want: "x - y\n" + fence}, + {name: "en-dash range kept", method: "POST", path: "/repos/o/r/issues/1/comments", + body: mustJSON(t, map[string]any{"body": "levels 3" + en + "5"}), unchanged: true}, + {name: "non-text field untouched", method: "POST", path: "/repos/o/r/git/refs", + body: mustJSON(t, map[string]any{"ref": "refs/heads/a" + em + "b", "sha": "abc"}), unchanged: true}, + {name: "non-JSON content type untouched", method: "POST", path: "/repos/o/r/releases/1/assets", + contentType: "application/octet-stream", body: "raw " + em + " bytes", unchanged: true}, + {name: "body that is not JSON untouched", method: "POST", path: "/repos/o/r/issues/1/comments", + body: "plain " + em + " text", unchanged: true}, + {name: "git push untouched", method: "POST", path: "/o/r.git/git-receive-pack", + contentType: "application/x-git-receive-pack-request", body: "0000 " + em, unchanged: true}, + {name: "LFS batch untouched", method: "POST", path: "/o/r.git/info/lfs/objects/batch", + body: mustJSON(t, map[string]any{"message": "a " + em + " b"}), unchanged: true}, + {name: "GET untouched", method: "GET", path: "/repos/o/r/issues", + body: mustJSON(t, map[string]any{"body": "a " + em + " b"}), unchanged: true}, + {name: "switch off", method: "POST", path: "/repos/o/r/issues/1/comments", off: true, + body: mustJSON(t, map[string]any{"body": "a " + em + " b"}), unchanged: true}, + {name: "hive control plane untouched", method: "POST", path: "/repos/o/r/issues/1/comments", agent: internalCallerName, + body: mustJSON(t, map[string]any{"body": "a " + em + " b"}), unchanged: true}, + {name: "GraphQL review thread reply", method: "POST", path: "/graphql", + body: mustJSON(t, map[string]any{ + "query": "mutation($input: AddPullRequestReviewThreadReplyInput!) { addPullRequestReviewThreadReply(input: $input) { comment { id } } }", + "variables": map[string]any{"input": map[string]any{"pullRequestReviewThreadId": "T_1", "body": "fixed " + em + " thanks"}}, + }), field: []string{"variables", "input", "body"}, want: "fixed - thanks"}, + {name: "GraphQL createPullRequest title", method: "POST", path: "/graphql", + body: mustJSON(t, map[string]any{ + "query": "mutation PullRequestCreate($input: CreatePullRequestInput!) { createPullRequest(input: $input) { pullRequest { url } } }", + "variables": map[string]any{"input": map[string]any{"title": "feat: x " + em + " y", "body": "b", "baseRefName": "main", "headRefName": "f"}}, + }), field: []string{"variables", "input", "title"}, want: "feat: x - y"}, + {name: "GraphQL query untouched", method: "POST", path: "/graphql", + body: mustJSON(t, map[string]any{"query": "query { viewer { login } }", "variables": map[string]any{"body": "a " + em + " b"}}), unchanged: true}, + {name: "GraphQL unrelated mutation untouched", method: "POST", path: "/graphql", + body: mustJSON(t, map[string]any{"query": "mutation($id: ID!) { addLabelsToLabelable(input: {labelableId: $id}) { clientMutationId } }", "variables": map[string]any{"title": "a " + em + " b"}}), unchanged: true}, + {name: "Linear issue description", method: "POST", path: "/graphql", linear: true, + body: mustJSON(t, map[string]any{ + "query": "mutation($input: IssueCreateInput!) { issueCreate(input: $input) { success } }", + "variables": map[string]any{"input": map[string]any{"title": "t", "description": "a " + em + " b"}}, + }), field: []string{"variables", "input", "description"}, want: "a - b"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + p := newTestProxy() + p.proseFilterOff = tc.off + agentName := tc.agent + if agentName == "" { + agentName = "quality" + } + req, err := http.NewRequest(tc.method, "https://api.github.com"+tc.path, strings.NewReader(tc.body)) + if err != nil { + t.Fatal(err) + } + if tc.contentType != "" { + req.Header.Set("Content-Type", tc.contentType) + } + if err := p.sanitizeAgentWriteProse(req, agentName, tc.linear); err != nil { + t.Fatalf("sanitizeAgentWriteProse: %v", err) + } + got, err := io.ReadAll(req.Body) + if err != nil { + t.Fatal(err) + } + if tc.unchanged { + if string(got) != tc.body { + t.Fatalf("body changed:\n got %s\nwant %s", got, tc.body) + } + return + } + if req.ContentLength != int64(len(got)) { + t.Errorf("ContentLength = %d, body is %d bytes", req.ContentLength, len(got)) + } + if h := req.Header.Get("Content-Length"); h != strconv.Itoa(len(got)) { + t.Errorf("Content-Length header = %q, body is %d bytes", h, len(got)) + } + if v := decodeField(t, got, tc.field...); v != tc.want { + t.Errorf("%v = %q, want %q", tc.field, v, tc.want) + } + }) + } +} + +// A body bigger than the buffer bound is forwarded whole and unchanged. +func TestSanitizeAgentWriteProseOversizedBodyForwardedIntact(t *testing.T) { + p := newTestProxy() + big := mustJSON(t, map[string]any{"body": em + strings.Repeat("x", outboundProseBodyLimit)}) + req, _ := http.NewRequest(http.MethodPost, "https://api.github.com/repos/o/r/issues/1/comments", strings.NewReader(big)) + if err := p.sanitizeAgentWriteProse(req, "quality", false); err != nil { + t.Fatal(err) + } + got, _ := io.ReadAll(req.Body) + if string(got) != big { + t.Fatal("oversized body was altered or truncated") + } +} + +// A read error is returned so the relay can drop the exchange. +func TestSanitizeAgentWriteProseReadError(t *testing.T) { + p := newTestProxy() + req, _ := http.NewRequest(http.MethodPost, "https://api.github.com/repos/o/r/issues/1/comments", io.NopCloser(errReader{})) + if err := p.sanitizeAgentWriteProse(req, "quality", false); err == nil { + t.Fatal("want a read error") + } +} + +type errReader struct{} + +func (errReader) Read([]byte) (int, error) { return 0, fmt.Errorf("boom") } + +// End to end through the MITM relay: what reaches GitHub has a plain hyphen +// and a Content-Length that matches the rewritten body. +func TestProxyHTTPRewritesAgentCommentDashes(t *testing.T) { + p := newTestProxy() + clientConn, proxyClient := net.Pipe() + upstreamConn, proxyUpstream := net.Pipe() + defer clientConn.Close() + defer upstreamConn.Close() + + go p.proxyHTTP(proxyClient, proxyUpstream, "quality", agent.ModeIssuesPRsMerge, agent.AgentCapabilities{}) + + body := mustJSON(t, map[string]any{"body": "Reviewed the change " + em + " it looks right.\n\n```\nkeep " + em + " this\n```"}) + go func() { + fmt.Fprintf(clientConn, "POST /repos/org/repo/issues/1/comments HTTP/1.1\r\nHost: api.github.com\r\nContent-Type: application/json\r\nContent-Length: %d\r\n\r\n%s", len(body), body) + }() + + type seen struct { + body []byte + cl int64 + } + got := make(chan seen, 1) + go func() { + req, err := http.ReadRequest(bufio.NewReader(upstreamConn)) + if err != nil { + got <- seen{} + return + } + b, _ := io.ReadAll(req.Body) + got <- seen{body: b, cl: req.ContentLength} + resp := &http.Response{StatusCode: http.StatusCreated, Proto: "HTTP/1.1", ProtoMajor: 1, ProtoMinor: 1, Header: make(http.Header), Body: http.NoBody, Request: req} + _ = resp.Write(upstreamConn) + }() + + resp, err := http.ReadResponse(bufio.NewReader(clientConn), nil) + if err != nil { + t.Fatalf("read response: %v", err) + } + if resp.StatusCode != http.StatusCreated { + t.Fatalf("status = %d, want 201", resp.StatusCode) + } + s := <-got + if s.cl != int64(len(s.body)) { + t.Errorf("upstream Content-Length = %d, body is %d bytes", s.cl, len(s.body)) + } + text := decodeField(t, s.body, "body") + if want := "Reviewed the change - it looks right.\n\n```\nkeep " + em + " this\n```"; text != want { + t.Errorf("forwarded body = %q, want %q", text, want) + } + if bytes.Contains(s.body, []byte("change "+em)) { + t.Error("prose em-dash reached upstream") + } +} + +func TestAgentWritesSwitchReadsEnv(t *testing.T) { + t.Setenv(prosefilter.AgentWritesEnv, "false") + if prosefilter.AgentWritesEnabled() { + t.Fatal("switch off not honoured") + } + t.Setenv(prosefilter.AgentWritesEnv, "") + if !prosefilter.AgentWritesEnabled() { + t.Fatal("switch must default on") + } +} diff --git a/src/crux/pushbroker/commit_message.go b/src/crux/pushbroker/commit_message.go new file mode 100644 index 0000000000..2c515a4a50 --- /dev/null +++ b/src/crux/pushbroker/commit_message.go @@ -0,0 +1,83 @@ +package pushbroker + +import ( + "context" + "fmt" + "strings" + + "github.com/hivecommons/hive/crux/prosefilter" +) + +// sanitizeHeadCommitMessage applies the shared prose filter +// (hivecommons/hive#19103) to the outgoing HEAD commit's message and amends +// it when a dash was rewritten. It reports whether it amended HEAD. +// +// A commit message travels inside the pushed git objects, so the egress proxy +// cannot rewrite it in transit without changing object ids. The broker is +// the one place hive touches an agent's commits before they leave, and it +// already amends HEAD for the trailing-newline rule, so it is where the +// message is fixed. Only HEAD is rewritten: an older outgoing commit would +// need a rebase of the agent's branch, which the broker never does. Hive +// squash-merges, and the squash commit message hive authors is filtered, so +// what lands on the default branch is clean either way. +// +// The amend keeps the tree, the author and every trailer (Signed-off-by +// included); --cleanup=verbatim keeps the message byte-for-byte apart from +// the rewritten dashes. The switch is prosefilter.AgentWritesEnv. +// +// It is best-effort: a dash rewrite never blocks a push. A failure to read or +// amend the message is logged and the commit leaves as the agent wrote it. +func (b *Broker) sanitizeHeadCommitMessage(ctx context.Context) bool { + amended, err := b.rewriteHeadCommitMessage(ctx) + if err != nil && b.Logger != nil { + b.Logger.Warn("pushbroker could not rewrite dashes in the HEAD commit message; pushing it unchanged", + "repo", b.Repo, "branch", b.Branch, "error", err) + } + return amended && err == nil +} + +func (b *Broker) rewriteHeadCommitMessage(ctx context.Context) (amended bool, err error) { + if !prosefilter.AgentWritesEnabled() { + return false, nil + } + // cat-file gives the message exactly as stored; `git log --format=%B` + // appends a newline, which --cleanup=verbatim would then keep. + raw, err := b.git(ctx, "cat-file", "commit", "HEAD") + if err != nil { + return false, fmt.Errorf("reading HEAD commit message: %w", err) + } + // A commit object always has a blank line before its message; without + // one msg is empty and HasDash below is false, so no separate check. + _, msg, _ := strings.Cut(string(raw), "\n\n") + if !prosefilter.HasDash(msg) { + return false, nil + } + clean := sanitizeCommitMessage(msg) + if clean == msg { + return false, nil + } + // The message goes in as one -m argument rather than through a temp file + // (git completes a missing final newline and, with --cleanup=verbatim, + // keeps everything else as given), so no staging file can fail. --only + // with no paths amends the message alone; the agent's staged changes + // stay staged and out of the commit. + if _, err := b.git(ctx, "commit", "--amend", "--only", "--cleanup=verbatim", "-m", clean); err != nil { + return false, err + } + if b.Logger != nil { + b.Logger.Info("pushbroker rewrote dashes in the HEAD commit message", "repo", b.Repo, "branch", b.Branch, + "dashes", prosefilter.DashCount(msg)-prosefilter.DashCount(clean)) + } + return true, nil +} + +// sanitizeCommitMessage filters a commit message: the subject line as a +// title, the rest as prose. +func sanitizeCommitMessage(msg string) string { + subject, rest, hasRest := strings.Cut(msg, "\n") + subject = prosefilter.SanitizeTitle(subject) + if !hasRest { + return subject + } + return subject + "\n" + prosefilter.Sanitize(rest) +} diff --git a/src/crux/pushbroker/commit_message_test.go b/src/crux/pushbroker/commit_message_test.go new file mode 100644 index 0000000000..d968018875 --- /dev/null +++ b/src/crux/pushbroker/commit_message_test.go @@ -0,0 +1,144 @@ +package pushbroker + +import ( + "context" + "errors" + "log/slog" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/hivecommons/hive/crux/prosefilter" +) + +const emDashForTest = prosefilter.EmDash + +const testSignoff = "Signed-off-by: Hive Test \n" + +// commitWithMessage commits a file with an exact commit message, adding the +// committer's Signed-off-by trailer the broker requires when it is missing, +// and returns the message as committed. +func commitWithMessage(t *testing.T, dir, msg string) string { + t.Helper() + if !strings.Contains(msg, testSignoff) { + msg += "\n" + testSignoff + } + if err := os.WriteFile(filepath.Join(dir, "a.txt"), []byte("a\n"), 0o644); err != nil { + t.Fatal(err) + } + msgFile := filepath.Join(t.TempDir(), "msg") + if err := os.WriteFile(msgFile, []byte(msg), 0o644); err != nil { + t.Fatal(err) + } + runGit(t, dir, "add", "a.txt") + runGit(t, dir, "commit", "--cleanup=verbatim", "-F", msgFile) + return msg +} + +func headMessage(t *testing.T, dir string) string { + t.Helper() + raw := runGitOutput(t, dir, "cat-file", "commit", "HEAD") + _, msg, _ := strings.Cut(raw, "\n\n") + return msg +} + +func TestBrokerRewritesHeadCommitMessageDashes(t *testing.T) { + dir := initRepo(t) + msg := "fix: a " + emDashForTest + " b\n\nWhy " + emDashForTest + " because.\n\n```\nkeep " + emDashForTest + " this\n```\n\nSigned-off-by: Hive Test \n" + commitWithMessage(t, dir, msg) + before := strings.TrimSpace(runGitOutput(t, dir, "rev-parse", "HEAD")) + res, err := (&Broker{Workspace: dir, Branch: "work", Repo: "hivecommons/hive", Minter: fakeMinter{"ghs_pushbroker"}, Runner: &recordingRunner{}}).Run(context.Background()) + if err != nil { + t.Fatalf("Run: %v (res=%+v)", err, res) + } + want := "fix: a - b\n\nWhy - because.\n\n```\nkeep " + emDashForTest + " this\n```\n\nSigned-off-by: Hive Test \n" + if got := headMessage(t, dir); got != want { + t.Fatalf("HEAD message = %q, want %q", got, want) + } + after := strings.TrimSpace(runGitOutput(t, dir, "rev-parse", "HEAD")) + if after == before || res.Commit != after { + t.Fatalf("Result.Commit = %s, HEAD %s (was %s): the amend must be what is pushed", res.Commit, after, before) + } +} + +// The success path logs how many dashes it rewrote, so an operator can tell +// the amend ran; the count is the drop between the original and the result. +func TestBrokerLogsHeadCommitMessageRewrite(t *testing.T) { + dir := initRepo(t) + commitWithMessage(t, dir, "fix: a "+emDashForTest+" b\n\nWhy "+emDashForTest+" because.\n") + var buf strings.Builder + res, err := (&Broker{Workspace: dir, Branch: "work", Repo: "hivecommons/hive", Minter: fakeMinter{"ghs_pushbroker"}, Runner: &recordingRunner{}, + Logger: slog.New(slog.NewTextHandler(&buf, nil))}).Run(context.Background()) + if err != nil || !res.Pushed { + t.Fatalf("Run = %+v, %v", res, err) + } + if strings.Contains(headMessage(t, dir), emDashForTest) { + t.Fatalf("dashes survived the amend: %q", headMessage(t, dir)) + } + if log := buf.String(); !strings.Contains(log, "rewrote dashes in the HEAD commit message") || !strings.Contains(log, "dashes=2") { + t.Fatalf("rewrite not logged with its dash count: %s", log) + } +} + +func TestBrokerLeavesCleanCommitMessageAlone(t *testing.T) { + dir := initRepo(t) + commitWithMessage(t, dir, "fix: a - b\n\nlevels 3"+prosefilter.EnDash+"5\n") + before := strings.TrimSpace(runGitOutput(t, dir, "rev-parse", "HEAD")) + res, err := (&Broker{Workspace: dir, Branch: "work", Repo: "hivecommons/hive", Minter: fakeMinter{"ghs_pushbroker"}, Runner: &recordingRunner{}}).Run(context.Background()) + if err != nil { + t.Fatalf("Run: %v", err) + } + if res.Commit != before { + t.Fatalf("a commit with nothing to rewrite was amended: %s -> %s", before, res.Commit) + } +} + +func TestBrokerCommitMessageSwitchOff(t *testing.T) { + t.Setenv(prosefilter.AgentWritesEnv, "false") + dir := initRepo(t) + msg := commitWithMessage(t, dir, "fix: a "+emDashForTest+" b\n") + if _, err := (&Broker{Workspace: dir, Branch: "work", Repo: "hivecommons/hive", Minter: fakeMinter{"ghs_pushbroker"}, Runner: &recordingRunner{}}).Run(context.Background()); err != nil { + t.Fatalf("Run: %v", err) + } + if got := headMessage(t, dir); got != msg { + t.Fatalf("switch off still rewrote the message: %q", got) + } +} + +// A dash rewrite never blocks a push: an amend or read failure is logged and +// the commit leaves as the agent wrote it. +func TestBrokerCommitMessageFailuresDoNotBlockPush(t *testing.T) { + for _, substr := range []string{"--cleanup=verbatim", "cat-file commit HEAD"} { + t.Run(substr, func(t *testing.T) { + dir := initRepo(t) + msg := commitWithMessage(t, dir, "fix: a "+emDashForTest+" b\n") + var buf strings.Builder + r := &nthCallFailingRunner{failSubstr: substr, failOnCall: 1, failErr: errors.New("injected")} + res, err := (&Broker{Workspace: dir, Branch: "work", Repo: "hivecommons/hive", Minter: fakeMinter{"ghs_pushbroker"}, Runner: r, + Logger: slog.New(slog.NewTextHandler(&buf, nil))}).Run(context.Background()) + if err != nil || !res.Pushed { + t.Fatalf("Run = %+v, %v; want the push to go ahead", res, err) + } + if got := headMessage(t, dir); got != msg { + t.Fatalf("message changed despite the failure: %q", got) + } + if !strings.Contains(buf.String(), "could not rewrite dashes") { + t.Fatalf("failure not logged: %s", buf.String()) + } + }) + } +} + +func TestSanitizeCommitMessage(t *testing.T) { + cases := map[string]string{ + "subject only " + emDashForTest + " x": "subject only - x", + "s " + emDashForTest + " t\n\nbody " + emDashForTest: "s - t\n\nbody -", + "clean\n\nbody\n": "clean\n\nbody\n", + } + for in, want := range cases { + if got := sanitizeCommitMessage(in); got != want { + t.Errorf("sanitizeCommitMessage(%q) = %q, want %q", in, got, want) + } + } +} diff --git a/src/crux/pushbroker/pushbroker.go b/src/crux/pushbroker/pushbroker.go index fe80b39f9b..707bceac1f 100644 --- a/src/crux/pushbroker/pushbroker.go +++ b/src/crux/pushbroker/pushbroker.go @@ -221,9 +221,17 @@ func (b *Broker) Run(ctx context.Context) (res Result, err error) { // the final newline (kubestellar/hive#5116). Fixing it here, once, covers // every backend and every target-repo language instead of teaching each // coding CLI's own formatter to run first. - if amended, err := b.stripTrailingBlankLines(ctx, res.ChangedFiles); err != nil { + amended, err := b.stripTrailingBlankLines(ctx, res.ChangedFiles) + if err != nil { return b.fail(res, fmt.Errorf("normalising trailing newlines: %w", err)) - } else if amended { + } + // The commit message travels inside the pushed objects, out of the + // egress proxy's reach, so the broker applies the outbound dash rule to + // it here (hivecommons/hive#19103). + if b.sanitizeHeadCommitMessage(ctx) { + amended = true + } + if amended { commit, err = b.git(ctx, "rev-parse", "HEAD") if err != nil { return b.fail(res, fmt.Errorf("reading HEAD after newline normalisation: %w", err)) diff --git a/src/docs/design/session-taint.md b/src/docs/design/session-taint.md index 519c51862b..4859849269 100644 --- a/src/docs/design/session-taint.md +++ b/src/docs/design/session-taint.md @@ -83,7 +83,7 @@ has strong per-message checks and no memory. | Canary at the proxy | `inspectCanaryEgress` (`src/crux/proxy/github_proxy_policy.go:266`), called at `src/crux/proxy/github_proxy_mitm.go:242` | Reads write bodies up to `maxGitHubWriteBodyScan` and denies with code `DenyReasonCanaryEgress` (`src/crux/proxy/denial_message.go:24`) in closed mode. Refuses `git-receive-pack` in closed mode because the pack cannot be scanned. | Only bodies on paths `githubWriteBodyMayLeak` accepts (`src/crux/proxy/github_proxy_policy.go:363`): issues, pulls, comments, reviews, git objects, GraphQL. | | Canary at the relays | `scanCanaryText` call sites at `src/pkg/github/issue_request_watcher.go:1596`, `src/pkg/github/pullrequest.go:233`, `src/pkg/github/review_request_watcher.go:598`; advisory findings at `src/cmd/hive/eval_cycle_seams.go:722` | Same scan on relay bodies and on agent advisory findings; leak recorded as `ioscan_canary_leak` with a critical bead by `handleCanaryLeak` (`src/cmd/hive/github_client_config.go:482`). | Same limit: marker only. | | Secret classifier | `FindSecret` (`src/crux/logscrub/handler.go:157`) over `secretPatterns` (`src/crux/logscrub/handler.go:37`) | Returns a category for the first exact-shape credential, never the value. Heuristic shapes are skipped on purpose "because refusing a push on one blocks legitimate work". | Exact shapes only. A token in an unknown format, or a reworded secret, is not found. | -| Push broker | The secret gate in `Broker.Run` (`src/crux/pushbroker/pushbroker.go:244`) | Rejects an outgoing diff containing a `FindSecret` hit, after fast-forward, protected-path and sign-off checks. The push itself is recorded through the effects boundary and the broker is built by the sandbox executor. | The scan covers the diff (`git diff`, or `git show` when no base resolves), not commit messages **(unverified beyond reading the two git invocations)**. It has no idea what the agent read. | +| Push broker | The secret gate in `Broker.Run` (`src/crux/pushbroker/pushbroker.go:252`) | Rejects an outgoing diff containing a `FindSecret` hit, after fast-forward, protected-path and sign-off checks. The push itself is recorded through the effects boundary and the broker is built by the sandbox executor. | The scan covers the diff (`git diff`, or `git show` when no base resolves), not commit messages **(unverified beyond reading the two git invocations)**. It has no idea what the agent read. | | Proxy modes and rules | `rules` (`src/crux/proxy/rules.go:135`), `AllowedByModeCaps` (`src/crux/proxy/rules.go:248`), `denyRules` (`src/crux/proxy/rules_deny.go:43`) | Method and path need a minimum ACMM mode; unknown writes are denied; direct PR create and merge are hard-denied. | Looks at the request, not at history. | | Write-surface gate | `WriteSurfaceEnforceRefusal` (`src/crux/proxy/write_surface_enforce.go:49`) | For a lane under `write_surface.enforce`, every direct write is refused so the relays are the only write path. | Says nothing about relay content. | | Which hosts are inspected | `NeedsMITM` (`src/crux/proxy/rules.go:89`), `IsCopilotAPIHost` (`src/crux/proxy/rules.go:124`) | TLS is terminated for the forge API host and the tracker host, and for completion hosts only to read usage. Everything else is an opaque tunnel (`tunnelDirect`, `src/crux/proxy/github_proxy_connect.go:605`) guarded by destination class (`plainForwardDenyReason`, `src/crux/proxy/plain_forward_guard.go:119`). | No payload visibility on other hosts, so no content-based egress rule is possible there. | @@ -174,7 +174,7 @@ agent, epoch)` and a pure function from `(labels, descriptor)` to a verdict. |---|---|---| | S-relay: `hive-open-issue`, `hive-open-pr`, `hive-review` and the spool-processed equivalents | beside `scanCanaryText` in `pkg/github` | Host-side, attribution comes from the manifest, so it works for sandboxed lanes without proxy identity. Highest value, lowest risk. | | S-proxy: direct REST and GraphQL writes by lanes not under `write_surface.enforce` | body-gate stage of the pipeline in [crux-tool-policy](crux-tool-policy.md), next to the canary gate | Requires UID attribution; observe-only when `HIVE_PROXY_ADVISORY_OK` is in effect. | -| S-push: the broker and any `git-receive-pack` | the broker, beside its secret gate and before the push token is minted (`src/crux/pushbroker/pushbroker.go:244`) | Destination visibility is the point: a push to a public fork with a `private-repo` label is the RFC flow 1 in code form. | +| S-push: the broker and any `git-receive-pack` | the broker, beside its secret gate and before the push token is minted (`src/crux/pushbroker/pushbroker.go:252`) | Destination visibility is the point: a push to a public fork with a `private-repo` label is the RFC flow 1 in code form. | | S-pr: the sandbox PR step (`CreateAgentPR` after a broker push) | `src/crux/agent/sandbox_executor.go` after `broker.Run` | PR title and body come from workspace files (`sandboxPRText`), so they are agent-authored text. | | S-advisory: agent advisory findings | the eval-cycle canary gate | Same text, same rule. | | S-egress: connections to hosts the proxy does not inspect | `tunnelDirect` | Destination-only: allow or hold by host, never content. Disabled by default even in enforce; see risks. | diff --git a/src/docs/documentation-map.md b/src/docs/documentation-map.md index 4bf1dd8ec8..fb797fd5ab 100644 --- a/src/docs/documentation-map.md +++ b/src/docs/documentation-map.md @@ -198,6 +198,7 @@ Documentation for the current Hive line (branch `v5`; the code and docs live und - [Experiment Lab (Nous)](strategy-lab.md) — experiment lifecycle, dashboard/API configuration, fast-fail bounds, and the gate-decision flow. No `nous:` block in `hive.yaml`. - [Hive-self background experiments (Nous)](nous-hive-self-experiments.md) — background-only self-improvement experiment mode, outcome questions (including user satisfaction), cohorts, guardrails, and auto-revert. Hidden from spoke dashboards and Experiment Lab. - [GitHub write surface](github-write-surface.md) — inventory agent and system writes, their audit coverage, and opt-in per-lane restrictions. +- [Outbound text: no em-dashes](outbound-prose.md) - the shared dash filter, every outbound path it covers (proxy agent writes, notifications, meetings, hub posts, commit messages), the `HIVE_AGENT_PROSE_FILTER` switch and the CI inventory guard. - [Compact GitHub CLI reads (`hive-gh`) and one-line denials](github-cli-compact.md) — **Agent/developer reference:** compact output fields, full-read escape hatch, and denial codes. - [GitHub App setup](github-app-setup.md) — the Forge App on GitHub and GitHub Enterprise: app creation, permissions, Setup URL, and `/gh-setup`. - [Forge setup: GitLab, Gitea, and Forgejo](forge-app-setup.md) — **Adapter/reference only:** explains configured adapter surfaces and why a complete hive still requires GitHub. diff --git a/src/docs/env-vars.md b/src/docs/env-vars.md index e28acd1b66..f51c12a553 100644 --- a/src/docs/env-vars.md +++ b/src/docs/env-vars.md @@ -274,6 +274,7 @@ new value at the same time. | `HIVE_PROXY_EGRESS_MARK` | No | `0x1112` | Packet mark exempted from the MITM egress redirect. | | `HIVE_PROXY_ADVISORY_OK` | No | `false` | Allows the spoke to start when the forced-proxy egress redirect cannot be installed (no `CAP_NET_ADMIN`/iptables). Enforcement becomes advisory-only — agents can bypass the proxy. Also gates whether the Go proxy trusts a self-asserted `Proxy-Authorization` header as agent identity when its UID map is unavailable (N7, #3841) — off by default, an unidentified caller is treated as `ADVISORY` (writes blocked) rather than whatever name it claims. See [security-model.md](security-model.md#5-network-and-credentials). | | `HIVE_PROXY_ALLOW_PRIVATE_FORWARD` | No | `false` | Lifts the proxy's destination guard ([#14082](https://github.com/hivecommons/hive/issues/14082), [#16785](https://github.com/hivecommons/hive/pull/16785)). Off by default: a plain-HTTP (non-CONNECT) forward, a CONNECT to a host the proxy does not inspect, and a transparent-TLS passthrough are each dialled only when the resolved address is globally routable, so an agent cannot use the proxy to reach loopback services beside it, the link-local cloud metadata endpoint (`169.254.169.254`), RFC 1918/CGNAT cluster addresses or multicast, and the refusal is a `403` naming the class. Only the exact string `true` turns it on, and it lifts the guard for all three legs at once, for deployments that genuinely egress to a private network through the proxy (a LAN mirror on a native install); the proxy logs a boot-time warning when it is set. Loopback refusals stay at `DEBUG` and are not counted as violations, because agents are already handed `NO_PROXY` for loopback (#14076). With the guard lifted, the inference translator also stops trusting the proxy UID (`dev`, 1001 on v5 pods): agents that run as the hive user without a per-agent UID are refused with `reason=proxy_uid_private_forward` ([#18061](https://github.com/hivecommons/hive/issues/18061), [#14448](https://github.com/hivecommons/hive/issues/14448)). | +| `HIVE_AGENT_PROSE_FILTER` | No | on | Rewrites em-dashes (and spaced en-dashes) in text an agent wrote itself: its GitHub and Linear writes through the egress proxy (JSON text fields only) and its HEAD commit message in the push broker. `false`, `0`, `off` or `no` turns it off. Text hive composes is always filtered. See [outbound-prose.md](outbound-prose.md) ([#19103](https://github.com/hivecommons/hive/issues/19103)). | | `HIVE_PROXY_INJECT_GH_AUTH` | No | unset (off on every hive, including hosted) | Exact `true` enables proxy-side App-token injection. **Requires GitHub App auth. Refuses startup (exit 19) with `HIVE_PROXY_ADVISORY_OK=true`.** Other spellings do not enable it. Read [behavior and exceptions](#proxy-credential-injection). | | `HIVE_AUTHZ_SHADOW` | No | unset (off) | Parseable true value (`true`, `1`) turns on the authorization decision seam in shadow mode ([design](design/authorization-decision-point.md), RFC #18988 phase 0/1): the push broker's changed-files guard is re-evaluated by `authz.Decide` after it decides, one decision event per call is kept on a bounded in-memory stream and a difference from the guard is logged. It enforces nothing and changes no behaviour. Read once at first use, so a change needs a restart. | | `HIVE_AUTHZ_SHADOW_BUFFER` | No | `1000` | Number of decision events the shadow stream keeps (oldest evicted first, evictions counted). Positive integer, capped at `10000`; anything else uses the default. Only read when `HIVE_AUTHZ_SHADOW` is on. | diff --git a/src/docs/github-write-surface.md b/src/docs/github-write-surface.md index a563f2deb6..b5710b5844 100644 --- a/src/docs/github-write-surface.md +++ b/src/docs/github-write-surface.md @@ -25,6 +25,11 @@ By default agents still have their direct write paths (`gh`, the API through the proxy, `git push`). An operator removes them lane by lane; see [Enforcing the write surface](#enforcing-the-write-surface). +Whichever path a write takes, its text gets the same outbound dash rule: +relay bodies are filtered by the relay, and direct agent writes through the +GitHub proxy have their JSON text fields filtered before they are forwarded. +See [Outbound text: no em-dashes](outbound-prose.md). + ## Relay operations Each relay authorizes a request in this order. It stops at the first refusal. diff --git a/src/docs/outbound-prose.md b/src/docs/outbound-prose.md new file mode 100644 index 0000000000..fee02e7c61 --- /dev/null +++ b/src/docs/outbound-prose.md @@ -0,0 +1,124 @@ +# Outbound text: no em-dashes + +Nothing hive or its agents send out of the hive may contain an em-dash +(U+2014), or an en-dash (U+2013) used as a dash. This page lists where that +rule is enforced and how CI keeps new send paths from skipping it +([#18970](https://github.com/hivecommons/hive/pull/18970), +[#19103](https://github.com/hivecommons/hive/issues/19103)). + +## The filter + +There is one filter, `src/crux/prosefilter`. The root module reaches it +through `src/pkg/prosefilter`, which only delegates. The filter lives in the +`crux` module so the egress proxy, which cannot import `pkg/`, uses the same +function. + +- `Sanitize` rewrites multi-line Markdown prose. An em-dash becomes ` - ` + (or `- ` / ` -` at the start or end of a line). An en-dash with spaces on + both sides becomes ` - `. An en-dash in a range such as `3-5` written with + U+2013 is left alone. +- `SanitizeTitle` does the same for a single line. +- `SanitizeJSON` applies the same rule to the string values under named keys + of a JSON document, at any depth, and leaves every other value unchanged. +- Fenced code blocks, inline code spans and blockquote lines are never + changed, because they carry quoted or pasted content. + +Agents are also told the rule in every kick (the plain-dash rule in the kick +header), so most text never needs rewriting. + +## Where it applies + +| Path | Boundary | Filtered | +| --- | --- | --- | +| Agent relay requests (PR, issue, comment, review bodies and titles) | `relayBody`, `CreatePR`, the issue-request watcher (`src/pkg/github`) | yes | +| Agent GitHub writes through the egress proxy (`gh api`, `gh pr comment`, `gh pr create`, REST and GraphQL) | `sanitizeAgentWriteProse` in `src/crux/proxy/outbound_prose.go` | yes | +| Agent Linear writes through the egress proxy | same, Linear mutation variables | yes | +| Agent commit messages | HEAD commit message in the push broker (`src/crux/pushbroker/commit_message.go`) | yes, HEAD only (see below) | +| Hive-signed commits (`createCommitOnBranch`) | `src/pkg/github/pr_request_signed.go` | yes | +| Squash and merge commit messages hive writes | `MergePRVia` in `src/pkg/github/pullrequest.go` | yes | +| Hive-rendered GitHub comments, issues and status notes | source templates | enforced at the source by the emit ratchet | +| ntfy, Slack and Discord webhook alerts | `notify.(*Notifier).Send` | yes | +| Slack DMs from the hub | `hub.sendSlackDM` | yes | +| Discord bot messages, embeds, edits, threads, topics | `discord.(*Bot).do`, `setChannelTopic` | yes | +| Hub Discord webhooks (requests, GitHub activity feed) | `hub.postDiscordRequestWebhook`, `hub.(*GitHubActivityFeed).postDiscord` | yes | +| Hub ntfy alert | `hub.(*HubServer).pushAuthAuditAlert` | yes | +| Meeting chat and spoken text (Quorum `chat.post`, `say`, `say.prewarm`) | `meetings.(*Client).post` | yes | +| Hub-filed GitHub issue titles (self-reports, feedback) | `hub.postHubGitHubIssue` | yes | +| Hub-filed issue bodies, seen-log comments and affected-hives edits | `pkg/fleetreport` templates | emit ratchet (the affected-hives block is parsed back from the issue by its separators, so it is not rewritten after rendering) | +| Linear advisory digest | `LinearAdvisoryPoster.PostDigest` filters before its skip-if-unchanged compare; the footer marker is now plain ASCII and the old em-dash marker is still recognised | yes | +| Linear and Jira work-source writes | `worksource.linearGraphQL`, `worksource.(*jiraSource).doJSON` | yes | +| Linear agent activity | `linearagent.doGraphQL` | yes | +| DIBS inbox ideas | `inbox.(*DibsDestination).Deliver` | yes | +| GitLab, Gitea and Forgejo writes | `pkg/forge` | emit ratchet | +| Email replies | no sender is wired yet | not applicable | + +## The egress proxy + +An agent that writes to GitHub itself talks to GitHub through the hive's MITM +egress proxy. Before #19103 the relay forwarded an allowed write verbatim, so +an agent's `gh api` comment kept its em-dashes even though hive's own writes +were filtered. The proxy now rewrites, just before forwarding: + +- REST JSON bodies: `body`, `title` and `message` at any depth, which covers + issue and PR comments, PR and issue titles and bodies, review bodies and + their inline `comments[].body`, review-thread replies and contents-API + commit messages. +- GraphQL mutation variables: `body` and `title` when the document names a + comment, review, review-thread reply, PR or issue mutation (`addComment`, + `addPullRequestReviewComment`, `addPullRequestReviewThreadReply`, + `createPullRequest`, `updatePullRequest`, `createIssue`, `updateIssue` and + the related review and comment-edit mutations). The query text is never + changed, so a string literal written inline in the query is not rewritten. +- Linear mutation variables: `body`, `title` and `description`. + +It never changes a body that is not JSON, a body that declares a non-JSON +`Content-Type`, any other field, git smart-HTTP or LFS traffic, or the hive's +own control-plane requests. A body over 1 MiB is forwarded unchanged rather +than truncated. When a body changes, its `Content-Length` is recomputed. One +debug line records the agent, method, path and the number of fields and dashes +changed, never the text. + +## Commit messages + +A commit message travels inside the pushed git objects, so it cannot be +rewritten in transit without changing object ids. The push broker, which +already amends an agent's HEAD commit for the trailing-newline rule, rewrites +the HEAD commit message with the same filter before pushing. Only HEAD is +rewritten: an older commit in the same push keeps its message, because the +broker never rebases an agent's branch. The rewrite is best-effort: if the message cannot be +read or amended, the broker logs a warning and pushes the commit unchanged. Hive squash-merges, and the squash +commit message it writes is filtered, so the commit that lands on the default +branch is clean either way. Hive installs no git hooks in agent workspaces +(hooks are disabled with `core.hooksPath=/dev/null`), so this is done by the +broker rather than a `commit-msg` hook. + +## Switch + +`HIVE_AGENT_PROSE_FILTER` turns off the rewriting of text an agent wrote +itself (its writes through the proxy and its HEAD commit message). It is on by +default; `false`, `0`, `off` or `no` turns it off. Text hive composes is always +filtered. + +## CI guards + +- `TestOutboundSitesInventory` (`src/pkg/prosefilter/outbound_sites_test.go`) + scans every non-test Go file under `src/pkg`, `src/cmd` and `src/crux` for + functions that send an HTTP request with a body. Each one must be listed in + its inventory as filtered (it calls the filter), filtered upstream by a + named function that calls the filter, a hive template under the emit + ratchet, or machine traffic with a reason. A new sender that is not listed + fails CI, as does a filtered site that stops calling the filter. The test + also pins the proxy, push broker, relay, PR, merge and notification + boundaries. +- `TestNoNewDashesInEmittedText` (`src/pkg/prosefilter/emit_guard_test.go`) + is a ratchet on hive's own templates: the dash count of each emitter file + may only go down. + +## Follow-ups + +- GraphQL string literals inlined in the query text are not rewritten. +- Commits other than HEAD in a multi-commit push keep their messages. +- `pkg/fleetreport` still renders ` - ` separators as em-dashes in the + affected-hives block and the seen log (held by the emit ratchet). Moving + them to plain hyphens needs a parser that accepts both forms for issues + already filed. diff --git a/src/pkg/discord/bot.go b/src/pkg/discord/bot.go index edfd4189eb..100211f870 100644 --- a/src/pkg/discord/bot.go +++ b/src/pkg/discord/bot.go @@ -16,6 +16,7 @@ import ( "unicode/utf8" "github.com/hivecommons/hive/crux/paneclass" + "github.com/hivecommons/hive/pkg/prosefilter" ) const ( @@ -488,6 +489,9 @@ func (b *Bot) do(method, url string, payload, out any) error { if data, err = json.Marshal(payload); err != nil { return err } + // Every bot message, embed, edit, thread and channel name goes + // through here: apply the shared outbound dash rule (#19103). + data, _ = prosefilter.SanitizeJSON(data, discordProseKeys) } for attempt := 0; ; attempt++ { var body io.Reader @@ -1015,7 +1019,18 @@ func (b *Bot) updateTopic(snap *statusSnapshot) { } } +// discordProseKeys are the Discord payload fields people read. +var discordProseKeys = prosefilter.Keys{ + "content": prosefilter.Prose, + "description": prosefilter.Prose, + "value": prosefilter.Prose, + "title": prosefilter.Title, + "name": prosefilter.Title, + "topic": prosefilter.Title, +} + func (b *Bot) setChannelTopic(topic string) error { + topic = prosefilter.SanitizeTitle(topic) payload, err := json.Marshal(map[string]string{"topic": topic}) if err != nil { return fmt.Errorf("marshal topic payload: %w", err) diff --git a/src/pkg/discord/bot_prose_test.go b/src/pkg/discord/bot_prose_test.go new file mode 100644 index 0000000000..098132ea1f --- /dev/null +++ b/src/pkg/discord/bot_prose_test.go @@ -0,0 +1,38 @@ +package discord + +import ( + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +// Every bot write goes through Bot.do, which applies the shared outbound dash +// rule (hivecommons/hive#19103) without touching the other payload fields. +func TestBotDoFiltersDashes(t *testing.T) { + var body []byte + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, _ = io.ReadAll(r.Body) + w.WriteHeader(http.StatusOK) + })) + defer ts.Close() + + b := newTestBot(ts, "chan") + if err := b.SendMessageTo("rfc-channel", "RFC opened — please review"); err != nil { + t.Fatalf("SendMessageTo: %v", err) + } + var got struct { + Content string `json:"content"` + } + if err := json.Unmarshal(body, &got); err != nil { + t.Fatalf("unmarshal: %v", err) + } + if got.Content != "RFC opened - please review" { + t.Errorf("content = %q", got.Content) + } + if !strings.Contains(string(body), `"allowed_mentions":{"parse":[]}`) { + t.Errorf("allowed_mentions changed: %s", body) + } +} diff --git a/src/pkg/github/pullrequest.go b/src/pkg/github/pullrequest.go index 1acc2d9b80..669dff3f2e 100644 --- a/src/pkg/github/pullrequest.go +++ b/src/pkg/github/pullrequest.go @@ -474,6 +474,8 @@ func (c *Client) MergePRVia(ctx context.Context, repo string, number int, mergeM if err != nil { return MergePRResult{}, fmt.Errorf("merging PR %s/%s#%d (%s): %w", owner, repo, number, mergeMethod, err) } + // The landing commit message is outbound text like the PR body (#19103). + commitMessage = prosefilter.Sanitize(commitMessage) var res *gh.PullRequestMergeResult boundary := c.mutationBoundary() claim := effects.Claim{ diff --git a/src/pkg/hub/auth_audit.go b/src/pkg/hub/auth_audit.go index c33e9a7a33..e06f9db3dc 100644 --- a/src/pkg/hub/auth_audit.go +++ b/src/pkg/hub/auth_audit.go @@ -8,6 +8,8 @@ import ( "os" "strings" "time" + + "github.com/hivecommons/hive/pkg/prosefilter" ) const ( @@ -179,6 +181,7 @@ func (s *HubServer) reportURLHealth(hives []RegistryEntry, clusterProbed, cluste // best-effort: failure to alert is logged but never fatal. The log Warn above // is always emitted regardless, so an external log-based alert still fires. func (s *HubServer) pushAuthAuditAlert(message string) { + message = prosefilter.Sanitize(message) server := strings.TrimRight(os.Getenv("HIVE_NTFY_SERVER"), "/") topic := os.Getenv("HIVE_NTFY_TOPIC") if server == "" || topic == "" { diff --git a/src/pkg/hub/discord_request_notify.go b/src/pkg/hub/discord_request_notify.go index f079a6eb60..287d6a8896 100644 --- a/src/pkg/hub/discord_request_notify.go +++ b/src/pkg/hub/discord_request_notify.go @@ -14,6 +14,7 @@ import ( "time" "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/prosefilter" ) const ( @@ -129,11 +130,21 @@ func (s *HubServer) deliverDiscordRequestNotification(webhookURL, requestID stri "request_id", requestID, "attempts", discordRequestWebhookMaxAttempts, "error", lastErr) } +// discordWebhookProseKeys are the Discord webhook payload fields people read. +var discordWebhookProseKeys = prosefilter.Keys{ + "content": prosefilter.Prose, + "description": prosefilter.Prose, + "value": prosefilter.Prose, + "title": prosefilter.Title, + "name": prosefilter.Title, +} + func postDiscordRequestWebhook(ctx context.Context, webhookURL string, payload discordWebhookPayload) error { body, err := json.Marshal(payload) if err != nil { return err } + body, _ = prosefilter.SanitizeJSON(body, discordWebhookProseKeys) req, err := http.NewRequestWithContext(ctx, http.MethodPost, webhookURL, bytes.NewReader(body)) if err != nil { return err diff --git a/src/pkg/hub/feedback.go b/src/pkg/hub/feedback.go index 96f8cf7c24..ea88cdcc61 100644 --- a/src/pkg/hub/feedback.go +++ b/src/pkg/hub/feedback.go @@ -18,6 +18,7 @@ import ( "github.com/hivecommons/hive/pkg/config" "github.com/hivecommons/hive/pkg/feedbackbundle" ghauth "github.com/hivecommons/hive/pkg/github" + "github.com/hivecommons/hive/pkg/prosefilter" ) const ( @@ -685,6 +686,12 @@ func writeHubFeedbackDiagnostics(b *strings.Builder, d *feedbackDiagnostics, sub } } func postHubGitHubIssue(ctx context.Context, client *http.Client, apiBase, token, owner, repo, title, body string, labels []string) (feedbackIssueResult, int, error) { + // Only the title is filtered here. The body carries the pkg/fleetreport + // affected-hives block, which UpdateAffected parses back from the live + // issue by its separators, so rewriting it after rendering would break + // later edits; that template is held to the rule by the emit ratchet + // instead (hivecommons/hive#19103). + title = prosefilter.SanitizeTitle(title) ctx = hubGitHubCallerContext(ctx, "feedback_issue") payload := map[string]any{"title": title, "body": body} if labels != nil { diff --git a/src/pkg/hub/github_activity.go b/src/pkg/hub/github_activity.go index 396218e1a2..3f7af8dc3d 100644 --- a/src/pkg/hub/github_activity.go +++ b/src/pkg/hub/github_activity.go @@ -16,6 +16,8 @@ import ( "strings" "sync" "time" + + "github.com/hivecommons/hive/pkg/prosefilter" ) const ( @@ -676,6 +678,7 @@ func (f *GitHubActivityFeed) authorSuppressed(author string, forwardMerge bool, const discordSuppressEmbeds = 1 << 2 func (f *GitHubActivityFeed) postDiscord(ctx context.Context, content string) error { + content = prosefilter.Sanitize(content) payload, err := json.Marshal(map[string]any{"content": content, "flags": discordSuppressEmbeds}) if err != nil { return err diff --git a/src/pkg/hub/github_activity_test.go b/src/pkg/hub/github_activity_test.go index 371158c8b8..58ee1df7ea 100644 --- a/src/pkg/hub/github_activity_test.go +++ b/src/pkg/hub/github_activity_test.go @@ -140,10 +140,10 @@ func TestGitHubActivityPollerDiffsFormatsAndDedupes(t *testing.T) { mu.Lock() got := append([]string(nil), posted...) mu.Unlock() - assertContainsLine(t, got, "🐝 [hive] Issue #2 opened — factory floor · @someone ") - assertContainsLine(t, got, "🐝 [docs] PR #7 merged into v6 — doc fix · @docwriter ") - assertContainsLine(t, got, "🐝 [docs] PR #8 merged into v5 — fast fix · @fastdev ") - assertContainsLine(t, got, "🐝 [infra] PR #3 merged into v6 — 🌱 Forward-merge v5 into v6 · @forward-bot[bot] ") + assertContainsLine(t, got, "🐝 [hive] Issue #2 opened - factory floor · @someone ") + assertContainsLine(t, got, "🐝 [docs] PR #7 merged into v6 - doc fix · @docwriter ") + assertContainsLine(t, got, "🐝 [docs] PR #8 merged into v5 - fast fix · @fastdev ") + assertContainsLine(t, got, "🐝 [infra] PR #3 merged into v6 - 🌱 Forward-merge v5 into v6 · @forward-bot[bot] ") for _, line := range got { if strings.Contains(line, "dependabot") { t.Fatalf("dependabot noise was not filtered: %q", line) diff --git a/src/pkg/hub/slack.go b/src/pkg/hub/slack.go index 97592cc4c2..7840ceabb8 100644 --- a/src/pkg/hub/slack.go +++ b/src/pkg/hub/slack.go @@ -9,6 +9,8 @@ import ( "strings" "sync" "time" + + "github.com/hivecommons/hive/pkg/prosefilter" ) // slackPostMessageURL is Slack's chat.postMessage endpoint. A bot token @@ -134,6 +136,7 @@ var slackClient = &http.Client{Timeout: slackHTTPTimeout} // body is decoded and ok is checked. The token is never logged; the returned // error carries only Slack's error code. func sendSlackDM(token, slackID, message string) error { + message = prosefilter.Sanitize(message) payload, err := json.Marshal(map[string]string{ "channel": slackID, "text": message, diff --git a/src/pkg/inbox/dibs.go b/src/pkg/inbox/dibs.go index 334c2c203b..6f79bf53f1 100644 --- a/src/pkg/inbox/dibs.go +++ b/src/pkg/inbox/dibs.go @@ -10,6 +10,8 @@ import ( "net/http" "strings" "time" + + "github.com/hivecommons/hive/pkg/prosefilter" ) // dibsTimeout bounds one idea submission. @@ -51,8 +53,8 @@ func (d *DibsDestination) Deliver(ctx context.Context, it Item, c Content, _ Req } body := DibsBody(it, c) payload, _ := json.Marshal(dibsIdea{ - Title: firstNonEmpty(c.Title, it.Title), - Body: body, + Title: prosefilter.SanitizeTitle(firstNonEmpty(c.Title, it.Title)), + Body: prosefilter.Sanitize(body), Visibility: "private", Status: "draft", }) diff --git a/src/pkg/linearagent/client.go b/src/pkg/linearagent/client.go index b74032895c..4c6317755d 100644 --- a/src/pkg/linearagent/client.go +++ b/src/pkg/linearagent/client.go @@ -9,6 +9,8 @@ import ( "log/slog" "net/http" "time" + + "github.com/hivecommons/hive/pkg/prosefilter" ) // Client is the CONTROL-PLANE Linear GraphQL client: it acknowledges sessions @@ -266,6 +268,9 @@ type graphQLEnvelope struct { } `json:"errors"` } +// linearAgentProseKeys are the Linear agent-session fields people read. +var linearAgentProseKeys = prosefilter.Keys{"body": prosefilter.Prose, "title": prosefilter.Title, "description": prosefilter.Prose} + // doGraphQL runs one authenticated GraphQL POST. Error strings carry only the // HTTP status and GraphQL error message — never the token or variables (which // hold issue content). @@ -274,6 +279,8 @@ func doGraphQL(ctx context.Context, hc *http.Client, graphqlURL, accessToken, qu if err != nil { return fmt.Errorf("marshal request: %w", err) } + // Agent activity text shown in Linear (#19103). + body, _ = prosefilter.SanitizeJSON(body, linearAgentProseKeys) req, err := http.NewRequestWithContext(ctx, http.MethodPost, graphqlURL, bytes.NewReader(body)) if err != nil { return fmt.Errorf("build request: %w", err) diff --git a/src/pkg/meetings/client.go b/src/pkg/meetings/client.go index 5712b62da6..3aa68f9213 100644 --- a/src/pkg/meetings/client.go +++ b/src/pkg/meetings/client.go @@ -15,6 +15,8 @@ import ( "strings" "sync" "sync/atomic" + + "github.com/hivecommons/hive/pkg/prosefilter" ) // MCP over streamable HTTP (POST /mcp), the transport Quorum serves. The @@ -104,6 +106,9 @@ func (c *Client) post(ctx context.Context, body any, wantID int64) (json.RawMess if err != nil { return nil, err } + // Every chat line and spoken line (chat.post, say, say.prewarm) leaves + // through here: apply the shared outbound dash rule (#19103). + data, _ = prosefilter.SanitizeJSON(data, meetingProseKeys) req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.base+"/mcp", bytes.NewReader(data)) if err != nil { return nil, err @@ -359,6 +364,10 @@ func (c *Client) TranscriptRead(ctx context.Context, nativeID string, afterSeq i return r, err } +// meetingProseKeys are the tool-argument fields that become meeting chat or +// speech. +var meetingProseKeys = prosefilter.Keys{"text": prosefilter.Prose, "texts": prosefilter.Prose} + // ChatPost calls chat.post. func (c *Client) ChatPost(ctx context.Context, nativeID, text string) error { return c.Call(ctx, "chat.post", textArgs{nativeID, text}, nil) diff --git a/src/pkg/meetings/client_prose_test.go b/src/pkg/meetings/client_prose_test.go new file mode 100644 index 0000000000..2450df48b3 --- /dev/null +++ b/src/pkg/meetings/client_prose_test.go @@ -0,0 +1,42 @@ +package meetings + +import ( + "context" + "testing" +) + +// Chat lines and spoken lines leave hive through Client.post, which applies +// the shared outbound dash rule (hivecommons/hive#19103). +func TestClientFiltersDashesInChatAndSpeech(t *testing.T) { + f, srv := newFakeQuorumServer(t) + c := NewClient(srv.URL, "tok", nil) + ctx := context.Background() + f.results["chat.post"] = `{"ok":true}` + f.results["say"] = `{"ok":true}` + f.results["say.prewarm"] = `{"ok":true}` + if err := c.ChatPost(ctx, "q1", "noted — next item"); err != nil { + t.Fatal(err) + } + if err := c.Say(ctx, "q1", "one — two"); err != nil { + t.Fatal(err) + } + if err := c.SayPrewarm(ctx, "q1", []string{"three — four"}); err != nil { + t.Fatal(err) + } + f.mu.Lock() + defer f.mu.Unlock() + n := len(f.args) + if n < 3 { + t.Fatalf("recorded %d tool calls, want at least 3", n) + } + if got := f.args[n-3]["text"]; got != "noted - next item" { + t.Errorf("chat.post text = %v", got) + } + if got := f.args[n-2]["text"]; got != "one - two" { + t.Errorf("say text = %v", got) + } + texts, _ := f.args[n-1]["texts"].([]any) + if len(texts) != 1 || texts[0] != "three - four" { + t.Errorf("say.prewarm texts = %v", f.args[n-1]["texts"]) + } +} diff --git a/src/pkg/notify/notify.go b/src/pkg/notify/notify.go index b553728975..3f01e6d711 100644 --- a/src/pkg/notify/notify.go +++ b/src/pkg/notify/notify.go @@ -12,6 +12,7 @@ import ( "time" "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/prosefilter" ) const ( @@ -59,6 +60,9 @@ const ( ) func (n *Notifier) Send(title, message string, priority Priority) { + // The one funnel for ntfy, Slack and Discord webhook alerts: the shared + // outbound dash rule applies here (hivecommons/hive#19103). + title, message = prosefilter.SanitizeTitle(title), prosefilter.Sanitize(message) if n.hiveID != "" { title = fmt.Sprintf("[%s] %s", n.hiveID, title) } diff --git a/src/pkg/prosefilter/emit_guard_test.go b/src/pkg/prosefilter/emit_guard_test.go index ab0882ecb7..02241770dd 100644 --- a/src/pkg/prosefilter/emit_guard_test.go +++ b/src/pkg/prosefilter/emit_guard_test.go @@ -25,6 +25,10 @@ var emitDirs = []string{ "pkg/prfollowup", "pkg/scheduler", "pkg/review", + // Outbound send sites classed as templates (outbound_sites_test.go). + "pkg/forge", + "pkg/feedbackbundle", + "pkg/fleetreport", } // TestNoNewDashesInEmittedText is a ratchet: the em-dash and en-dash count of diff --git a/src/pkg/prosefilter/outbound_sites_test.go b/src/pkg/prosefilter/outbound_sites_test.go new file mode 100644 index 0000000000..ac0107d4d3 --- /dev/null +++ b/src/pkg/prosefilter/outbound_sites_test.go @@ -0,0 +1,433 @@ +package prosefilter + +import ( + "go/ast" + "go/parser" + "go/token" + "io/fs" + "os" + "path/filepath" + "runtime" + "slices" + "sort" + "strconv" + "strings" + "testing" +) + +// Outbound send-site inventory (hivecommons/hive#19103). +// +// Every non-test function under src/pkg, src/cmd and src/crux that sends an +// HTTP request with a body (http.NewRequest / NewRequestWithContext with a +// method other than GET, HEAD, DELETE or OPTIONS, http.Post, http.PostForm, +// or a client's Post(url, "content/type", body)) must be listed in +// outboundSites with a class. TestOutboundSitesInventory fails when a new +// sender is added without an entry, when an entry no longer names a sender, +// and when a site classed as filtered stops calling the shared filter. That +// keeps the em-dash rule from regressing as new channels are added: a new +// Slack, Discord, meeting, email or forge sender cannot land without someone +// deciding, in review, how its text is filtered. +// +// Hive's own go-github writes do not appear here (they are not raw HTTP +// sends): their text is either agent text passed through relayBody or a +// hive-rendered template guarded by TestNoNewDashesInEmittedText. The +// egress proxy's relay of agent writes and the push broker's commit message +// rule are pinned by requiredCalls below. + +type outboundClass string + +const ( + // classFiltered: the function itself calls prosefilter before sending. + classFiltered outboundClass = "filtered" + // classVia: the text was filtered by the named function before it got + // here (via must itself call prosefilter). + classVia outboundClass = "via" + // classTemplate: hive-rendered text whose source package (source) is + // under the emit ratchet, so no dash can be written into it. Used where a + // send compares the rendered text with what is already on the forge, and + // rewriting after the compare would edit on every pass. + classTemplate outboundClass = "template" + // classMachine: the body carries no text a person reads on a third-party + // surface (tokens, ids, state, labels, heartbeats, model prompts, calls + // to hive's own services). + classMachine outboundClass = "machine" +) + +type outboundSite struct { + class outboundClass + via string + source string + reason string +} + +func filtered() outboundSite { return outboundSite{class: classFiltered} } +func via(fn, reason string) outboundSite { + return outboundSite{class: classVia, via: fn, reason: reason} +} +func template(source, reason string) outboundSite { + return outboundSite{class: classTemplate, source: source, reason: reason} +} +func machine(reason string) outboundSite { return outboundSite{class: classMachine, reason: reason} } + +const ( + reasonSpokeToHub = "hive-to-hub or hub-to-spoke control traffic; hub-side delivery is inventoried separately" + reasonModelPrompt = "model or agent-runtime request; nothing here is published" + reasonAuth = "OAuth, token or credential exchange" + reasonInternalAPI = "call to hive's own API or service" + reasonPlatform = "platform API (Kubernetes, object storage, secrets store)" + reasonStateOnly = "state, labels or ids only; no text" + reasonTranscribe = "audio upload for transcription" + reasonTestFixture = "test fixture server" + reasonFleetReport = "seen-log and affected-hives bookkeeping, compared with the forge copy before an edit" + reasonForgeWrite = "forge comment and issue writes; text is agent text already filtered or hive templates" + reasonFixedMessage = "fixed ASCII commit message" +) + +var outboundSites = map[string]outboundSite{ + // Filtered at the send boundary. + "discord.(*Bot).do": filtered(), + "discord.(*Bot).setChannelTopic": filtered(), + "github.(*Client).createCommitOnBranch": filtered(), + "hub.(*GitHubActivityFeed).postDiscord": filtered(), + "hub.(*HubServer).pushAuthAuditAlert": filtered(), + "hub.postDiscordRequestWebhook": filtered(), + "hub.postHubGitHubIssue": filtered(), + "hub.sendSlackDM": filtered(), + "inbox.(*DibsDestination).Deliver": filtered(), + "linearagent.doGraphQL": filtered(), + "meetings.(*Client).post": filtered(), + "worksource.(*jiraSource).doJSON": filtered(), + "worksource.linearGraphQL": filtered(), + "notify.(*Notifier).sendNtfy": via("notify.(*Notifier).Send", "ntfy alert"), + "notify.(*Notifier).sendSlack": via("notify.(*Notifier).Send", "Slack webhook alert"), + "notify.(*Notifier).sendDiscordWebhook": via("notify.(*Notifier).Send", "Discord webhook alert"), + "main.hubSelfReportRelay": via("hub.postHubGitHubIssue", "spoke self-report; the hub files it"), + "dashboard.(*Server).forwardFeedbackToHub": via("hub.postHubGitHubIssue", "operator feedback; the hub files it"), + + // Hive-rendered text under the emit ratchet. + "hub.postHubGitHubIssueComment": template("pkg/fleetreport", reasonFleetReport), + "hub.patchHubGitHubIssueBody": template("pkg/fleetreport", reasonFleetReport), + "hub.patchHubGitHubIssueComment": template("pkg/fleetreport", reasonFleetReport), + "forge.(*gitLabForge).doWrite": template("pkg/forge", reasonForgeWrite), + "forge.(*giteaForge).doWrite": template("pkg/forge", reasonForgeWrite), + "forge.(*giteaForge).Merge": template("pkg/forge", "merge request on a non-GitHub forge"), + "forge.(*gitLabForge).doAcceptMerge": template("pkg/forge", "merge request on a non-GitHub forge"), + "feedbackbundle.githubJSON": template("pkg/feedbackbundle", "feedback bundle commit message"), + + // No human-read text. + "agent.(*sharedSessionPoolClient).request": machine(reasonInternalAPI), + "api.(*Client).request": machine(reasonInternalAPI), + "auth.(*Provider).fetchIDToken": machine(reasonAuth), + "callback.(*Client).do": machine(reasonInternalAPI), + "client.(*Client).doJSON": machine(reasonInternalAPI), + "connector.(*HTTPClient).once": machine(reasonInternalAPI), + "dashboard.(*Server).forwardNPS": machine(reasonSpokeToHub), + "dashboard.(*Server).handleCopilotAuthStart": machine(reasonAuth), + "dashboard.(*Server).handleReleaseChannelSwitch": machine(reasonSpokeToHub), + "dashboard.(*Server).handleSelfUpgrade": machine(reasonSpokeToHub), + "dashboard.(*Server).hubRequest": machine(reasonSpokeToHub), + "dashboard.(*Server).pollCopilotToken": machine(reasonAuth), + "dashboard.(*Server).postBenchmarksBatch": machine(reasonSpokeToHub), + "dashboard.kubernetesSelfUpgradeRequest": machine(reasonPlatform), + "dashboard.meetingsSpeak": machine("persona voice sample, played back to the operator in the dashboard only"), + "dashboard.npsRelayPost": machine(reasonSpokeToHub), + "dashboard.patchKeyIntoHiveSecrets": machine(reasonPlatform), + "dashboard.probeLiteLLMInference": machine(reasonModelPrompt), + "discord.(*Bot).dashboardPost": machine(reasonInternalAPI), + "fixture.(*Server).attemptEffect": machine(reasonTestFixture), + "flue.(*Adapter).do": machine(reasonModelPrompt), + "forge.(*gitLabForge).doRebase": machine(reasonStateOnly), + "github.PollDeviceFlowWithScope": machine(reasonAuth), + "github.StartDeviceFlowWithScope": machine(reasonAuth), + "hivectl.(*Client).request": machine(reasonInternalAPI), + "hivectl.Register": machine(reasonInternalAPI), + "hivectl.Reissue": machine(reasonInternalAPI), + "hub.(*HubServer).exchangeGitHubLogin": machine(reasonAuth), + "hub.(*HubServer).pushGitHubConfigToSpoke": machine(reasonSpokeToHub), + "hub.(*HubServer).pushVisibilityToSpoke": machine(reasonSpokeToHub), + "hub.(*HubServer).relayWebhookToSpoke": machine(reasonSpokeToHub), + "hub.closeHubGitHubIssue": machine(reasonStateOnly), + "hub.commitBenchmarkFile": machine(reasonFixedMessage), + "hub.createOCIExport": machine(reasonPlatform), + "hub.createOCIFileSystem": machine(reasonPlatform), + "hub.npsRelayAck": machine(reasonSpokeToHub), + "hub.postHubGitHubIssueLabels": machine(reasonStateOnly), + "hub.relayMailOnce": machine(reasonSpokeToHub), + "hub.topRepoRequest": machine(reasonStateOnly), + "hubbackup.(*ObjectStore).do": machine(reasonPlatform), + "intent.(*AlignmentReviewer).reviewUncached": machine(reasonModelPrompt), + "ioscan.(*openAIChatClient).completeOnce": machine(reasonModelPrompt), + "jev.(*Client).Decide": machine(reasonModelPrompt), + "jev.callEndpoint": machine(reasonModelPrompt), + "knowledge.(*Client).postJSON": machine(reasonInternalAPI), + "knowledge.(*Promoter).ingestToLayer": machine(reasonInternalAPI), + "kubejob.(*Launcher).do": machine(reasonPlatform), + "linearagent.postTokenForm": machine(reasonAuth), + "mailin.(*HubClient).do": machine(reasonSpokeToHub), + "main.(*delegateDeciderRunner).RunDeciderTurn": machine(reasonModelPrompt), + "main.(*knowledgeAdviserLLM).complete": machine(reasonModelPrompt), + "main.kbPost": machine(reasonInternalAPI), + "managedagents.(*Adapter).doJSON": machine(reasonModelPrompt), + "meetings.(*Client).CompanionAudio": machine(reasonTranscribe), + "meetllm.complete": machine(reasonModelPrompt), + "meetllm.completeAnthropic": machine(reasonModelPrompt), + "openbao.(*Provider).raw": machine(reasonPlatform), + "openrouter.ExchangeCode": machine(reasonAuth), + "proxy.(*GitHubProxy).handleInferenceRequest": machine(reasonModelPrompt), + "proxy.(*GitHubProxy).inferenceTranslatorHandler": machine(reasonModelPrompt), + "proxy.(*GitHubProxy).retryInferenceOnAuthFailure": machine(reasonModelPrompt), + "proxy.forwardToInference": machine(reasonModelPrompt), + "proxy.retryInferenceOnOverload": machine(reasonModelPrompt), + "retro.(*openAIChatClient).Complete": machine(reasonModelPrompt), + "reviewpreflight.(*Client).do": machine(reasonInternalAPI), + "rotation.KiroProber.Probe": machine(reasonModelPrompt), + "sandbox.(*routerBridge).do": machine(reasonPlatform), + "spoke.SendUpgradingHeartbeat": machine(reasonSpokeToHub), + "spoke.StartTaskStatusPush": machine(reasonSpokeToHub), + "spoke.k8sAPIPatchOnceWithContentType": machine(reasonPlatform), + "spoke.postActivityUpdateAck": machine(reasonSpokeToHub), + "spoke.postHeartbeatToHub": machine(reasonSpokeToHub), + "spoke.postHeartbeatToMirror": machine(reasonSpokeToHub), + "spoke.reportUpgradeOutcome": machine(reasonSpokeToHub), + "trajectory.(*Reviewer).Review": machine(reasonModelPrompt), + "transcribe.(*Deepgram).Transcribe": machine(reasonTranscribe), + "transcribe.(*Gemini).do": machine(reasonTranscribe), + "watsonx.(*TokenMinter).mint": machine(reasonAuth), + "worksource.(*githubProjectsSource).graphql": machine(reasonStateOnly), +} + +// requiredCalls pins the boundaries that are not raw HTTP sends: file +// (relative to src/) -> the function in it that must call the named helper. +var requiredCalls = []struct{ file, fn, call string }{ + {"crux/proxy/github_proxy_mitm.go", "proxy.(*GitHubProxy).proxyHTTPHost", "sanitizeAgentWriteProse"}, + {"crux/pushbroker/pushbroker.go", "pushbroker.(*Broker).Run", "sanitizeHeadCommitMessage"}, + {"pkg/github/write_surface.go", "github.(*Client).relayBody", "Sanitize"}, + {"pkg/github/pullrequest.go", "github.(*Client).CreatePR", "Sanitize"}, + {"pkg/github/pullrequest.go", "github.(*Client).MergePRVia", "Sanitize"}, + {"pkg/notify/notify.go", "notify.(*Notifier).Send", "Sanitize"}, +} + +// safeMethods are the request methods that carry no body to filter. +var safeMethods = map[string]bool{ + "MethodGet": true, "MethodHead": true, "MethodDelete": true, "MethodOptions": true, + `"GET"`: true, `"HEAD"`: true, `"DELETE"`: true, `"OPTIONS"`: true, +} + +// scannedFunc is what the scan records for one function declaration. +type scannedFunc struct { + pos string + sends bool + callsFilter bool + calledIdents map[string]bool +} + +func TestOutboundSitesInventory(t *testing.T) { + srcRoot := srcRootDir(t) + funcs := scanFuncs(t, srcRoot) + + var missing []string + for name, f := range funcs { + if !f.sends { + continue + } + if _, ok := outboundSites[name]; !ok { + missing = append(missing, name+" ("+f.pos+")") + } + } + sort.Strings(missing) + for _, m := range missing { + t.Errorf("outbound send site %s is not in outboundSites: route its text through prosefilter (Sanitize, SanitizeTitle or SanitizeJSON) and add it as filtered(), or classify it with a reason", m) + } + + for name, site := range outboundSites { + f, ok := funcs[name] + if !ok || !f.sends { + t.Errorf("outboundSites entry %s no longer names an outbound send site; remove it", name) + continue + } + switch site.class { + case classFiltered: + if !f.callsFilter { + t.Errorf("%s (%s) is classed filtered but does not call prosefilter", name, f.pos) + } + case classVia: + v, ok := funcs[site.via] + if !ok || !v.callsFilter { + t.Errorf("%s says its text is filtered by %s, which does not exist or does not call prosefilter", name, site.via) + } + case classTemplate: + if !slices.Contains(emitDirs, site.source) { + t.Errorf("%s names template source %s, which is not under the emit ratchet (emitDirs)", name, site.source) + } + case classMachine: + default: + t.Errorf("%s has unknown class %q", name, site.class) + } + if site.class != classFiltered && strings.TrimSpace(site.reason) == "" { + t.Errorf("%s is classed %s without a reason", name, site.class) + } + } + + for _, rc := range requiredCalls { + f, ok := funcs[rc.fn] + if !ok { + t.Errorf("%s: function %s not found", rc.file, rc.fn) + continue + } + if !f.calledIdents[rc.call] { + t.Errorf("%s (%s) must call %s: it is the outbound dash boundary for its path", rc.fn, f.pos, rc.call) + } + } +} + +func srcRootDir(t *testing.T) string { + t.Helper() + _, thisFile, _, ok := runtime.Caller(0) + if !ok { + t.Fatal("runtime.Caller failed") + } + return filepath.Join(filepath.Dir(thisFile), "..", "..") +} + +// scanFuncs parses every non-test Go file under src/pkg, src/cmd and +// src/crux and records, per function, whether it sends a body over HTTP and +// whether it calls the shared filter. +func scanFuncs(t *testing.T, srcRoot string) map[string]*scannedFunc { + t.Helper() + funcs := map[string]*scannedFunc{} + fset := token.NewFileSet() + for _, dir := range []string{"pkg", "cmd", "crux"} { + root := filepath.Join(srcRoot, dir) + err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + switch d.Name() { + case "testdata", "vendor", "node_modules": + return filepath.SkipDir + } + return nil + } + if !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") { + return nil + } + src, rerr := os.ReadFile(path) + if rerr != nil { + return rerr + } + f, perr := parser.ParseFile(fset, path, src, parser.SkipObjectResolution) + if perr != nil { + return perr + } + rel, _ := filepath.Rel(srcRoot, path) + rel = filepath.ToSlash(rel) + for _, decl := range f.Decls { + fd, ok := decl.(*ast.FuncDecl) + if !ok || fd.Body == nil { + continue + } + name := f.Name.Name + "." + funcName(fd) + sf := funcs[name] + if sf == nil { + sf = &scannedFunc{pos: rel + ":" + strconv.Itoa(fset.Position(fd.Pos()).Line), calledIdents: map[string]bool{}} + funcs[name] = sf + } + ast.Inspect(fd.Body, func(n ast.Node) bool { + call, ok := n.(*ast.CallExpr) + if !ok { + return true + } + switch fn := call.Fun.(type) { + case *ast.Ident: + sf.calledIdents[fn.Name] = true + case *ast.SelectorExpr: + sf.calledIdents[fn.Sel.Name] = true + if x, ok := fn.X.(*ast.Ident); ok && x.Name == "prosefilter" { + sf.callsFilter = true + } + if isBodySend(fn, call) { + sf.sends = true + } + } + return true + }) + } + return nil + }) + if err != nil { + t.Fatalf("walk %s: %v", dir, err) + } + } + return funcs +} + +// isBodySend reports whether call sends an HTTP request that can carry a +// body. +func isBodySend(fn *ast.SelectorExpr, call *ast.CallExpr) bool { + x, isIdent := fn.X.(*ast.Ident) + fromHTTP := isIdent && x.Name == "http" + switch fn.Sel.Name { + case "NewRequest", "NewRequestWithContext": + if !fromHTTP { + return false + } + i := 0 + if fn.Sel.Name == "NewRequestWithContext" { + i = 1 + } + if len(call.Args) <= i { + return true + } + return !safeMethods[exprString(call.Args[i])] + case "PostForm": + return fromHTTP + case "Post": + if fromHTTP { + return true + } + if len(call.Args) != 3 { + return false + } + lit, ok := call.Args[1].(*ast.BasicLit) + return ok && lit.Kind == token.STRING + } + return false +} + +func exprString(e ast.Expr) string { + switch v := e.(type) { + case *ast.BasicLit: + return v.Value + case *ast.SelectorExpr: + if x, ok := v.X.(*ast.Ident); ok && x.Name == "http" { + return v.Sel.Name + } + } + return "" +} + +func funcName(d *ast.FuncDecl) string { + if d.Recv == nil || len(d.Recv.List) == 0 { + return d.Name.Name + } + typ := d.Recv.List[0].Type + star := false + if s, ok := typ.(*ast.StarExpr); ok { + star, typ = true, s.X + } + switch g := typ.(type) { + case *ast.IndexExpr: + typ = g.X + case *ast.IndexListExpr: + typ = g.X + } + recv := "?" + if id, ok := typ.(*ast.Ident); ok { + recv = id.Name + } + if star { + return "(*" + recv + ")." + d.Name.Name + } + return recv + "." + d.Name.Name +} diff --git a/src/pkg/prosefilter/prosefilter.go b/src/pkg/prosefilter/prosefilter.go index 5a3b008fec..9d054b5382 100644 --- a/src/pkg/prosefilter/prosefilter.go +++ b/src/pkg/prosefilter/prosefilter.go @@ -1,210 +1,43 @@ -// Package prosefilter is the one place hive rewrites dashes in text that -// leaves the hive for a forge (PR and issue titles and bodies, comments, -// review bodies, commit messages). -// -// The owner's rule is that em-dashes are never acceptable in anything hive or -// its agents write. Agents are told so in the shared base prompt, but a model -// can still emit one, so every outward write path also passes its prose -// through Sanitize before it is sent. -// -// Sanitize rewrites only prose. Fenced code blocks, inline code spans and -// Markdown blockquote lines are left byte-for-byte alone, because they carry -// quoted or pasted content (a diff, a log line, a file excerpt) that must not -// be altered. +// Package prosefilter is the root-module entry point to hive's one outbound +// dash filter. The implementation lives in the crux module +// (github.com/hivecommons/hive/crux/prosefilter) so the GitHub egress proxy, +// which cannot import pkg/, applies the very same rule to agent writes +// (hivecommons/hive#19103). Every function here delegates; do not add a +// second implementation. package prosefilter -import ( - "regexp" - "strings" -) +import cruxprose "github.com/hivecommons/hive/crux/prosefilter" const ( - emDash = "\u2014" - enDash = "\u2013" - - // plainDash is what a dash used as punctuation becomes. - plainDash = "-" - // spacedDash is the replacement between two words. - spacedDash = " - " - - // maxFenceIndent is the Markdown limit on leading spaces for a code fence. - maxFenceIndent = 3 - // minFenceRun is the shortest run of fence characters that opens a fence. - minFenceRun = 3 -) - -var ( - // emDashRE matches an em-dash with the horizontal whitespace around it. - emDashRE = regexp.MustCompile(`[ \t]*` + emDash + `[ \t]*`) - // spacedEnDashRE matches an en-dash used as a dash (whitespace on both - // sides). An en-dash in a range such as 3-5 has no spaces and is left alone - // here; a hard-coded range is fixed at its source. - spacedEnDashRE = regexp.MustCompile(`[ \t]+` + enDash + `[ \t]+`) + emDash = cruxprose.EmDash + enDash = cruxprose.EnDash ) // HasDash reports whether s contains an em-dash or an en-dash. -func HasDash(s string) bool { - return strings.Contains(s, emDash) || strings.Contains(s, enDash) -} - -// Sanitize replaces every em-dash (U+2014), and every en-dash (U+2013) used -// as a spaced dash, in the prose of s with a plain hyphen. Text inside fenced -// code blocks, inline code spans and blockquote lines is untouched. A string -// with no such dash is returned unchanged. -func Sanitize(s string) string { - if !HasDash(s) { - return s - } - lines := strings.Split(s, "\n") - var fenceChar byte - var fenceLen int - for i, line := range lines { - if fenceChar != 0 { - if closesFence(line, fenceChar, fenceLen) { - fenceChar, fenceLen = 0, 0 - } - continue - } - if c, n := opensFence(line); c != 0 { - fenceChar, fenceLen = c, n - continue - } - if isBlockquote(line) { - continue - } - lines[i] = sanitizeLine(line) - } - return strings.Join(lines, "\n") -} - -// SanitizeTitle is Sanitize for a single-line title. A title has no code -// fence or blockquote, so only inline code spans are protected. -func SanitizeTitle(s string) string { - if !HasDash(s) { - return s - } - return sanitizeLine(s) -} - -func opensFence(line string) (byte, int) { - trimmed := strings.TrimLeft(line, " ") - if len(line)-len(trimmed) > maxFenceIndent || trimmed == "" { - return 0, 0 - } - c := trimmed[0] - if c != '`' && c != '~' { - return 0, 0 - } - n := runLen(trimmed, 0, c) - if n < minFenceRun { - return 0, 0 - } - // A backtick fence's info string may not contain a backtick; such a line - // is an inline code span, not a fence. - if c == '`' && strings.Contains(trimmed[n:], "`") { - return 0, 0 - } - return c, n -} - -func closesFence(line string, c byte, openLen int) bool { - trimmed := strings.TrimLeft(line, " ") - if len(line)-len(trimmed) > maxFenceIndent || trimmed == "" || trimmed[0] != c { - return false - } - n := runLen(trimmed, 0, c) - return n >= openLen && strings.TrimSpace(trimmed[n:]) == "" -} - -func isBlockquote(line string) bool { - return strings.HasPrefix(strings.TrimLeft(line, " "), ">") -} - -func runLen(s string, from int, c byte) int { - n := 0 - for from+n < len(s) && s[from+n] == c { - n++ - } - return n -} - -// sanitizeLine rewrites one prose line, skipping inline code spans. -func sanitizeLine(line string) string { - if !HasDash(line) { - return line - } - indentLen := len(line) - len(strings.TrimLeft(line, " \t")) - indent, rest := line[:indentLen], line[indentLen:] - - var b strings.Builder - b.WriteString(indent) - segStart := 0 - flush := func(end int) { - if end > segStart { - b.WriteString(rewriteSegment(rest[segStart:end], segStart == 0, end == len(rest))) - } - } - for i := 0; i < len(rest); { - if rest[i] != '`' { - i++ - continue - } - n := runLen(rest, i, '`') - closeAt := findClosingRun(rest, i+n, n) - if closeAt < 0 { - i += n - continue - } - flush(i) - spanEnd := closeAt + n - b.WriteString(rest[i:spanEnd]) - segStart = spanEnd - i = spanEnd - } - flush(len(rest)) - return b.String() -} +func HasDash(s string) bool { return cruxprose.HasDash(s) } + +// Sanitize replaces em-dashes, and en-dashes used as spaced dashes, in the +// prose of s with a plain hyphen. Code fences, inline code and blockquote +// lines are untouched. +func Sanitize(s string) string { return cruxprose.Sanitize(s) } + +// SanitizeTitle is Sanitize for a single-line title. +func SanitizeTitle(s string) string { return cruxprose.SanitizeTitle(s) } + +// Keys, Kind, Prose, Title and Stats are the shared filter's JSON field +// selectors and change counts. +type ( + Keys = cruxprose.Keys + Kind = cruxprose.Kind + Stats = cruxprose.Stats +) -// findClosingRun returns the index of the next run of exactly n backticks at -// or after from, or -1. -func findClosingRun(s string, from, n int) int { - for i := from; i < len(s); { - if s[i] != '`' { - i++ - continue - } - m := runLen(s, i, '`') - if m == n { - return i - } - i += m - } - return -1 -} +const ( + Prose = cruxprose.Prose + Title = cruxprose.Title +) -// rewriteSegment replaces dashes in one non-code stretch of a line. atStart -// and atEnd say whether the segment touches the start or end of the line's -// prose, so a leading or trailing dash does not leave a stray space. -func rewriteSegment(seg string, atStart, atEnd bool) string { - seg = spacedEnDashRE.ReplaceAllString(seg, spacedDash) - locs := emDashRE.FindAllStringIndex(seg, -1) - if len(locs) == 0 { - return seg - } - var b strings.Builder - prev := 0 - for _, loc := range locs { - b.WriteString(seg[prev:loc[0]]) - switch { - case loc[0] == 0 && atStart: - b.WriteString(plainDash + " ") - case loc[1] == len(seg) && atEnd: - b.WriteString(" " + plainDash) - default: - b.WriteString(spacedDash) - } - prev = loc[1] - } - b.WriteString(seg[prev:]) - return b.String() -} +// SanitizeJSON rewrites the string values under keys, at any depth of a JSON +// document, with the shared rule. Anything that is not JSON, or has nothing +// to change, is returned as is. +func SanitizeJSON(body []byte, keys Keys) ([]byte, Stats) { return cruxprose.SanitizeJSON(body, keys) } diff --git a/src/pkg/prosefilter/prosefilter_test.go b/src/pkg/prosefilter/prosefilter_test.go index 3932825796..f48f3aa179 100644 --- a/src/pkg/prosefilter/prosefilter_test.go +++ b/src/pkg/prosefilter/prosefilter_test.go @@ -2,61 +2,21 @@ package prosefilter import "testing" -const ( - em = "\u2014" - en = "\u2013" -) - -func TestSanitize(t *testing.T) { - cases := []struct { - name, in, want string - }{ - {"no dash is unchanged", "plain - text", "plain - text"}, - {"spaced em-dash", "a " + em + " b", "a - b"}, - {"unspaced em-dash", "a" + em + "b", "a - b"}, - {"several em-dashes", "a " + em + " b " + em + " c", "a - b - c"}, - {"leading em-dash", em + " hive: agent=x", "- hive: agent=x"}, - {"leading em-dash keeps indent", " " + em + " note", " - note"}, - {"trailing em-dash", "wait " + em, "wait -"}, - {"spaced en-dash", "a " + en + " b", "a - b"}, - {"en-dash range untouched", "levels 3" + en + "5", "levels 3" + en + "5"}, - {"inline code untouched", "see `a " + em + " b` and c " + em + " d", "see `a " + em + " b` and c - d"}, - {"double-backtick span untouched", "``a ` " + em + " b`` " + em + " c", "``a ` " + em + " b`` - c"}, - {"unclosed backtick is prose", "a ` b " + em + " c", "a ` b - c"}, - {"fenced block untouched", "x " + em + " y\n```\nfile " + em + " list\n```\nz " + em + " w", "x - y\n```\nfile " + em + " list\n```\nz - w"}, - {"tilde fence untouched", "~~~go\na " + em + " b\n~~~\nc " + em + " d", "~~~go\na " + em + " b\n~~~\nc - d"}, - {"longer closing fence", "````\na " + em + " b\n`````\nc " + em + " d", "````\na " + em + " b\n`````\nc - d"}, - {"short closing fence does not close", "````\na\n```\nb " + em + " c\n````", "````\na\n```\nb " + em + " c\n````"}, - {"blockquote untouched", "> quoted " + em + " text\nmine " + em + " here", "> quoted " + em + " text\nmine - here"}, - {"markdown list", "- file.go " + em + " added", "- file.go - added"}, - {"CRLF preserved", "a " + em + " b\r\nc", "a - b\r\nc"}, - } - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - if got := Sanitize(tc.in); got != tc.want { - t.Fatalf("Sanitize(%q) = %q, want %q", tc.in, got, tc.want) - } - if got := Sanitize(Sanitize(tc.in)); got != tc.want { - t.Fatalf("Sanitize is not idempotent for %q: %q", tc.in, got) - } - }) +// TestDelegatesToCrux pins that the root-module entry point applies the +// shared crux filter rather than a copy of it. +func TestDelegatesToCrux(t *testing.T) { + in := "a " + emDash + " b `c " + emDash + " d`" + if got, want := Sanitize(in), "a - b `c "+emDash+" d`"; got != want { + t.Fatalf("Sanitize = %q, want %q", got, want) } -} - -func TestSanitizeTitle(t *testing.T) { - if got, want := SanitizeTitle("fix: a "+em+" b `c "+em+" d`"), "fix: a - b `c "+em+" d`"; got != want { + if got, want := SanitizeTitle("x "+enDash+" y"), "x - y"; got != want { t.Fatalf("SanitizeTitle = %q, want %q", got, want) } - if got := SanitizeTitle("fix: plain"); got != "fix: plain" { - t.Fatalf("clean title changed: %q", got) - } -} - -func TestHasDash(t *testing.T) { - if HasDash("a - b") { - t.Fatal("hyphen reported as dash") + if !HasDash(emDash) || HasDash("-") { + t.Fatal("HasDash wrong") } - if !HasDash("a "+em+" b") || !HasDash("a "+en+" b") { - t.Fatal("em or en dash not detected") + out, st := SanitizeJSON([]byte(`{"body":"a `+emDash+` b"}`), Keys{"body": Prose, "title": Title}) + if string(out) != `{"body":"a - b"}` || st.Fields != 1 { + t.Fatalf("SanitizeJSON = %s (%+v)", out, st) } } diff --git a/src/pkg/prosefilter/testdata/emit_dash_baseline.json b/src/pkg/prosefilter/testdata/emit_dash_baseline.json index 1c4bdeabf2..66913553bf 100644 --- a/src/pkg/prosefilter/testdata/emit_dash_baseline.json +++ b/src/pkg/prosefilter/testdata/emit_dash_baseline.json @@ -3,6 +3,12 @@ "pkg/advisory/policy.go": 1, "pkg/dupsweep/render.go": 6, "pkg/escalation/escalation.go": 10, + "pkg/feedbackbundle/bundle.go": 1, + "pkg/fleetreport/affected.go": 5, + "pkg/fleetreport/fleetreport.go": 4, + "pkg/fleetreport/identity.go": 3, + "pkg/fleetreport/runtime.go": 5, + "pkg/fleetreport/slogfeed/slogfeed.go": 3, "pkg/github/advisory.go": 10, "pkg/github/agent_repo_scope.go": 1, "pkg/github/app.go": 6, diff --git a/src/pkg/worksource/jira.go b/src/pkg/worksource/jira.go index 827b99ffce..2f4253dc25 100644 --- a/src/pkg/worksource/jira.go +++ b/src/pkg/worksource/jira.go @@ -13,6 +13,8 @@ import ( "net/url" "strings" "time" + + "github.com/hivecommons/hive/pkg/prosefilter" ) const ( @@ -549,6 +551,9 @@ func jiraADFDocument(text string) map[string]any { } } +// jiraProseKeys are the Jira payload fields people read. +var jiraProseKeys = prosefilter.Keys{"summary": prosefilter.Title, "description": prosefilter.Prose, "body": prosefilter.Prose, "text": prosefilter.Prose} + func (s *jiraSource) doJSON(ctx context.Context, method, reqURL string, payload any, wantStatus int, out any) error { if s.tlsErr != nil { return s.tlsErr @@ -561,6 +566,12 @@ func (s *jiraSource) doJSON(ctx context.Context, method, reqURL string, payload } body = &buf } + if buf, ok := body.(*bytes.Buffer); ok { + // Issue summaries, descriptions and comments, plain or ADF text + // nodes (#19103). + clean, _ := prosefilter.SanitizeJSON(buf.Bytes(), jiraProseKeys) + body = bytes.NewReader(clean) + } req, err := http.NewRequestWithContext(ctx, method, reqURL, body) if err != nil { return fmt.Errorf("build request: %w", err) diff --git a/src/pkg/worksource/linear.go b/src/pkg/worksource/linear.go index f05c1ecbfc..2ac58d3da3 100644 --- a/src/pkg/worksource/linear.go +++ b/src/pkg/worksource/linear.go @@ -24,6 +24,7 @@ import ( "time" "github.com/hivecommons/hive/pkg/github" + "github.com/hivecommons/hive/pkg/prosefilter" ) // defaultLinearBaseURL is Linear's single GraphQL endpoint. @@ -489,6 +490,9 @@ func (s *LinearSource) doQuery(ctx context.Context, query string, vars map[strin return &resp, nil } +// linearProseKeys are the Linear variables people read. +var linearProseKeys = prosefilter.Keys{"title": prosefilter.Title, "description": prosefilter.Prose, "body": prosefilter.Prose} + // linearGraphQL posts one GraphQL request to Linear with the bare API-key // Authorization header Linear expects and returns the raw response body once // it has passed the HTTP-status and top-level `errors` checks. Shared by the @@ -505,6 +509,9 @@ func linearGraphQL(ctx context.Context, client *http.Client, baseURL, apiKey, qu if err != nil { return nil, fmt.Errorf("marshal request: %w", err) } + // Issue titles, descriptions and comments (#19103); the query text + // itself is under no prose key and is never touched. + body, _ = prosefilter.SanitizeJSON(body, linearProseKeys) req, err := http.NewRequestWithContext(ctx, http.MethodPost, baseURL, bytes.NewReader(body)) if err != nil { return nil, fmt.Errorf("build request: %w", err) diff --git a/src/pkg/worksource/linear_advisory.go b/src/pkg/worksource/linear_advisory.go index 93c211e49a..b833260656 100644 --- a/src/pkg/worksource/linear_advisory.go +++ b/src/pkg/worksource/linear_advisory.go @@ -7,6 +7,8 @@ import ( "fmt" "net/http" "strings" + + "github.com/hivecommons/hive/pkg/prosefilter" ) // LinearAdvisoryMarker is the stable footer every hive-authored digest comment @@ -15,7 +17,12 @@ import ( // digest stays ONE comment rewritten in place instead of a new comment per // governor eval. Kept visible rather than an HTML comment because Linear's // markdown renderer does not reliably preserve HTML comments. -const LinearAdvisoryMarker = "_Hive advisory digest (hive-advisory-digest) — rewritten every governor cycle._" +const LinearAdvisoryMarker = "_Hive advisory digest (hive-advisory-digest) - rewritten every governor cycle._" + +// legacyLinearAdvisoryMarker is the footer digests carried before outbound +// text lost its em-dashes (hivecommons/hive#19103); a comment carrying it is +// still the hive's digest and is updated in place. +const legacyLinearAdvisoryMarker = "_Hive advisory digest (hive-advisory-digest) \u2014 rewritten every governor cycle._" // ErrLinearAdvisoryIssueUnset is returned when the Linear target is selected // but governor.advisory.linear_issue is empty. It names the missing key so @@ -80,7 +87,9 @@ func (p *LinearAdvisoryPoster) PostDigest(ctx context.Context, identifier, diges if p == nil || strings.TrimSpace(p.apiKey) == "" { return ErrLinearAdvisoryAPIKeyUnset } - body := strings.TrimRight(digest, "\n") + "\n\n" + LinearAdvisoryMarker + // Filter before the compare below so the skip-if-unchanged guard sees + // the text Linear will actually hold (hivecommons/hive#19103). + body := prosefilter.Sanitize(strings.TrimRight(digest, "\n") + "\n\n" + LinearAdvisoryMarker) raw, err := linearGraphQL(ctx, p.client, p.baseURL, p.apiKey, linearAdvisoryIssueQuery, map[string]interface{}{"id": identifier}) if err != nil { @@ -107,7 +116,7 @@ func (p *LinearAdvisoryPoster) PostDigest(ctx context.Context, identifier, diges } for _, c := range lookup.Data.Issue.Comments.Nodes { - if !strings.Contains(c.Body, LinearAdvisoryMarker) { + if !strings.Contains(c.Body, LinearAdvisoryMarker) && !strings.Contains(c.Body, legacyLinearAdvisoryMarker) { continue } if c.Body == body {