Skip to content
Merged
1 change: 1 addition & 0 deletions changelog.d/fixed-19103-agent-writes-no-em-dash.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- Em-dashes no longer reach GitHub or any other outbound channel through paths the first filter missed ([#19103](https://github.com/hivecommons/hive/issues/19103)). The shared dash filter moved into the `crux` module so the GitHub egress proxy now applies it to agent writes it relays (`gh api` comments, review replies, PR and issue titles and bodies, contents-API commit messages, and the text inputs of comment, review, PR and issue GraphQL mutations, plus Linear mutation variables), touching only JSON text fields and never git traffic or non-JSON bodies. The push broker rewrites the agent's HEAD commit message, and notifications (ntfy, Slack, Discord), Discord bot messages, meeting chat and spoken text, hub-filed issues and comments, Linear, Jira and DIBS writes, and squash commit messages hive writes all pass through the same filter. A new CI inventory test fails when an outbound HTTP send is added without the filter or a stated reason. Set `HIVE_AGENT_PROSE_FILTER=false` to stop rewriting agent-authored text.
154 changes: 154 additions & 0 deletions src/crux/prosefilter/json.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,154 @@
package prosefilter

import (
"bytes"
"encoding/json"
"os"
"strings"
)

// Kind says how a JSON field's text is filtered.
type Kind int

const (
// Prose is multi-line Markdown text: Sanitize.
Prose Kind = iota + 1
// Title is single-line text: SanitizeTitle.
Title
)

// Keys maps a JSON object key to the filter its string values get.
type Keys map[string]Kind

// Stats counts what SanitizeJSON changed: text fields rewritten, and dashes
// removed. It never carries the text itself, so it is safe to log.
type Stats struct {
Fields int
Dashes int
}

// AgentWritesEnv is the operator switch for rewriting text that an agent
// wrote itself (its GitHub writes through the egress proxy, its HEAD commit
// message in the push broker). It is on unless set to one of
// agentWritesOffValues. Text hive itself composes is always filtered.
const AgentWritesEnv = "HIVE_AGENT_PROSE_FILTER"

var agentWritesOffValues = map[string]bool{"false": true, "0": true, "off": true, "no": true}

// AgentWritesEnabled reports whether AgentWritesEnv leaves the agent-write
// rewrite on.
func AgentWritesEnabled() bool {
return !agentWritesOffValues[strings.ToLower(strings.TrimSpace(os.Getenv(AgentWritesEnv)))]
}

// SanitizeJSON rewrites, at any depth of a JSON document, the string values
// (and the strings of a string array) stored under a key in keys, using the
// same rule as Sanitize. Every other value, number formatting included, is
// preserved. A body that is not JSON, holds no dash, or has nothing to change
// is returned unchanged with zero Stats; only a changed body is re-encoded
// (keys sorted, no HTML escaping).
func SanitizeJSON(body []byte, keys Keys) ([]byte, Stats) {
if len(keys) == 0 || !mayHoldDash(body) {
return body, Stats{}
}
dec := json.NewDecoder(bytes.NewReader(body))
dec.UseNumber()
var doc any
if dec.Decode(&doc) != nil || dec.More() {
return body, Stats{}
}
var st Stats
doc = sanitizeTree(doc, keys, &st)
if st.Fields == 0 {
return body, Stats{}
}
out, err := marshalNoEscape(doc)
if err != nil {
return body, Stats{}
}
return out, st
}

// mayHoldDash reports whether body could contain an em- or en-dash, raw or as
// a JSON \u escape.
func mayHoldDash(body []byte) bool {
if bytes.Contains(body, []byte(emDash)) || bytes.Contains(body, []byte(enDash)) {
return true
}
lower := bytes.ToLower(body)
return bytes.Contains(lower, []byte(`—`)) || bytes.Contains(lower, []byte(`–`))
}

func sanitizeTree(v any, keys Keys, st *Stats) any {
switch t := v.(type) {
case map[string]any:
for k, child := range t {
if kind, ok := keys[k]; ok {
switch c := child.(type) {
case string:
t[k] = sanitizeCounted(c, kind, st)
continue
case []any:
if allStrings(c) {
for i, s := range c {
c[i] = sanitizeCounted(s.(string), kind, st)
}
continue
}
}
}
t[k] = sanitizeTree(child, keys, st)
}
return t
case []any:
for i, child := range t {
t[i] = sanitizeTree(child, keys, st)
}
return t
default:
return v
}
}

func allStrings(items []any) bool {
for _, it := range items {
if _, ok := it.(string); !ok {
return false
}
}
return len(items) > 0
}

func sanitizeCounted(s string, kind Kind, st *Stats) string {
if !HasDash(s) {
return s
}
var clean string
if kind == Title {
clean = SanitizeTitle(s)
} else {
clean = Sanitize(s)
}
if clean != s {
st.Fields++
st.Dashes += DashCount(s) - DashCount(clean)
}
return clean
}

// DashCount is the number of em-dashes and en-dashes in s.
func DashCount(s string) int {
return strings.Count(s, emDash) + strings.Count(s, enDash)
}

// marshalNoEscape is json.Marshal without HTML escaping, so a body that only
// had its dashes rewritten does not also gain < escapes.
func marshalNoEscape(v any) ([]byte, error) {
var buf bytes.Buffer
enc := json.NewEncoder(&buf)
enc.SetEscapeHTML(false)
if err := enc.Encode(v); err != nil {
return nil, err
}
return bytes.TrimRight(buf.Bytes(), "\n"), nil
}
68 changes: 68 additions & 0 deletions src/crux/prosefilter/json_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
package prosefilter

import (
"encoding/json"
"testing"
)

func TestSanitizeJSON(t *testing.T) {
keys := Keys{"body": Prose, "title": Title, "texts": Prose}
cases := []struct {
name, in, want string
fields int
}{
{"top-level body", `{"body":"a ` + em + ` b"}`, `{"body":"a - b"}`, 1},
{"escaped dash", `{"body":"a — b"}`, `{"body":"a - b"}`, 1},
{"title", `{"title":"x ` + em + ` y"}`, `{"title":"x - y"}`, 1},
{"nested", `{"input":{"body":"a ` + em + ` b","id":"T_1"}}`, `{"input":{"body":"a - b","id":"T_1"}}`, 1},
{"array items", `{"comments":[{"body":"a ` + em + ` b","line":3}]}`, `{"comments":[{"body":"a - b","line":3}]}`, 1},
{"string array under prose key", `{"texts":["a ` + em + ` b","c"]}`, `{"texts":["a - b","c"]}`, 1},
{"numbers kept exactly", `{"body":"a ` + em + ` b","n":12345678901234567890,"f":1.50}`, `{"body":"a - b","f":1.50,"n":12345678901234567890}`, 1},
{"no HTML escaping", `{"body":"<a> & ` + em + `"}`, `{"body":"<a> & -"}`, 1},
{"other key untouched", `{"ref":"a ` + em + ` b"}`, `{"ref":"a ` + em + ` b"}`, 0},
{"non-string under key untouched", `{"body":{"ref":"a ` + em + ` b"}}`, `{"body":{"ref":"a ` + em + ` b"}}`, 0},
{"no dash", `{"body":"plain"}`, `{"body":"plain"}`, 0},
{"not JSON", `plain ` + em + ` text`, `plain ` + em + ` text`, 0},
{"trailing data", `{"body":"a ` + em + ` b"} {}`, `{"body":"a ` + em + ` b"} {}`, 0},
{"code span kept", `{"body":"` + "`a " + em + " b`" + `"}`, `{"body":"` + "`a " + em + " b`" + `"}`, 0},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
got, st := SanitizeJSON([]byte(tc.in), keys)
if string(got) != tc.want {
t.Fatalf("SanitizeJSON = %s, want %s", got, tc.want)
}
if st.Fields != tc.fields {
t.Fatalf("Fields = %d, want %d", st.Fields, tc.fields)
}
if tc.fields > 0 && st.Dashes == 0 {
t.Fatal("Dashes not counted")
}
if tc.fields > 0 && !json.Valid(got) {
t.Fatalf("output is not JSON: %s", got)
}
})
}
}

func TestSanitizeJSONNoKeys(t *testing.T) {
in := []byte(`{"body":"a ` + em + ` b"}`)
if got, st := SanitizeJSON(in, nil); string(got) != string(in) || st.Fields != 0 {
t.Fatalf("nil keys changed the body: %s", got)
}
}

func TestDashCount(t *testing.T) {
if got := DashCount("a " + em + " b " + en + " c"); got != 2 {
t.Fatalf("DashCount = %d, want 2", got)
}
}

func TestAgentWritesEnabled(t *testing.T) {
for v, want := range map[string]bool{"": true, "true": true, "false": false, "0": false, "OFF": false, "no": false} {
t.Setenv(AgentWritesEnv, v)
if got := AgentWritesEnabled(); got != want {
t.Errorf("%s=%q: AgentWritesEnabled = %v, want %v", AgentWritesEnv, v, got, want)
}
}
}
Loading
Loading