diff --git a/changelog.d/security-logread-secret-json-escaped.md b/changelog.d/security-logread-secret-json-escaped.md new file mode 100644 index 0000000000..b716a51328 --- /dev/null +++ b/changelog.d/security-logread-secret-json-escaped.md @@ -0,0 +1 @@ +- `log_read` (admin MCP) and `GET /api/logs/read`: one of the hive's own secret values is now redacted when it holds a quote, a backslash or a control character. The hive log, the request log and the output sources are stored as JSON, so such a value is on disk only in its JSON-escaped form (`pa\"ss\\word`). The read-time redaction looked for the plain, URL-encoded and base64 forms only, so it missed it and the value came back readable in the returned line. The JSON-escaped form, with and without HTML escaping, is now redacted as well. Values made only of letters, digits and ordinary punctuation were always redacted. diff --git a/src/crux/logread/secret_json_form_test.go b/src/crux/logread/secret_json_form_test.go new file mode 100644 index 0000000000..18c506c27a --- /dev/null +++ b/src/crux/logread/secret_json_form_test.go @@ -0,0 +1,80 @@ +package logread + +import ( + "bytes" + "encoding/json" + "log/slog" + "os" + "path/filepath" + "strings" + "testing" +) + +// The hive log is slog JSON. A secret value that holds a quote, a backslash +// or a control character is written there only in its JSON-escaped form, so +// the plain value never matched and the secret came back readable. +func TestOwnSecretIsRedactedInItsJSONEscapedForm(t *testing.T) { + for name, secret := range map[string]string{ + "plain": `plain-secret-0123456789`, + "quote, backslash": `pa"ss\word-0123456789`, + "tab and newline": "line1-secret\tcol\nline2-0123", + "escape character": "esc\x1bseq-secret-0123456789", + "html characters": `a&c-secret-0123456789`, + "non-ASCII letters": `pässwörd-geheim-0123456789`, + } { + t.Run(name, func(t *testing.T) { + dir := t.TempDir() + var buf bytes.Buffer + lg := slog.New(slog.NewJSONHandler(&buf, nil)) + lg.Info("connecting", "password", secret) + lg.Info("tail") + if err := os.WriteFile(filepath.Join(dir, HiveLogFilename), buf.Bytes(), 0o600); err != nil { + t.Fatal(err) + } + res := Read(Request{}, Options{LogDir: dir, Secrets: NewSecretScrubber([]string{secret})}) + if res.LineCount != 2 { + t.Fatalf("read %d lines, want 2", res.LineCount) + } + redacted := 0 + for _, line := range res.Lines { + var record map[string]any + if err := json.Unmarshal([]byte(line.Text), &record); err == nil && record["password"] == secret { + t.Errorf("the secret is readable in the returned line: %s", line.Text) + } + redacted += strings.Count(line.Text, secretValueMarker) + } + if redacted != 1 { + t.Errorf("redaction markers in the returned lines = %d, want 1 (redactions=%v)", redacted, res.Redactions) + } + }) + } +} + +// A record written with encoding/json's default HTML escaping (as the output +// sources' Go writer does) is covered too. +func TestOwnSecretIsRedactedInHTMLEscapedJSON(t *testing.T) { + const secret = `a&c "quoted" secret-0123456789` + record, err := json.Marshal(map[string]string{"text": "token is " + secret + " end"}) + if err != nil { + t.Fatal(err) + } + out, n := NewSecretScrubber([]string{secret}).Replace(string(record)) + if n != 1 || strings.Contains(out, "secret-0123456789") { + t.Fatalf("replaced %d, result %s", n, out) + } +} + +// Ordinary text is left alone, and a value with nothing to escape gains no +// new form. +func TestJSONEscapedFormsAddNothingForPlainValues(t *testing.T) { + const secret = "plain-secret-0123456789" + for _, form := range jsonEscapedForms(secret) { + if form != secret { + t.Errorf("plain value gained the form %q", form) + } + } + text := `{"msg":"nothing secret here","path":"C:\\temp\\file \"quoted\""}` + if out, n := NewSecretScrubber([]string{`pa"ss\word-0123456789`}).Replace(text); n != 0 || out != text { + t.Errorf("unrelated text changed: %d replacements, %s", n, out) + } +} diff --git a/src/crux/logread/secrets.go b/src/crux/logread/secrets.go index 30e782cf08..70a6de3345 100644 --- a/src/crux/logread/secrets.go +++ b/src/crux/logread/secrets.go @@ -2,6 +2,7 @@ package logread import ( "encoding/base64" + "encoding/json" "io/fs" "net/url" "os" @@ -16,11 +17,11 @@ const SecretValueKind = "hive-secret-value" const secretValueMarker = "" // SecretScrubber replaces this hive's own secret values in log text: each -// value of at least MinSecretValueLen characters in its plain, URL-encoded and -// base64 forms, the base64 forms at all three byte alignments so the value is -// found inside a longer encoded string (an "Authorization: Basic" header -// encoding user:value). It is one precompiled multi-value replacer, built in -// process for one read; the values are never returned. +// value of at least MinSecretValueLen characters in its plain, URL-encoded, +// JSON-escaped and base64 forms, the base64 forms at all three byte alignments +// so the value is found inside a longer encoded string (an "Authorization: +// Basic" header encoding user:value). It is one precompiled multi-value +// replacer, built in process for one read; the values are never returned. type SecretScrubber struct { r *strings.Replacer } @@ -78,6 +79,7 @@ func (s *SecretScrubber) Replace(text string) (string, int) { // secretForms returns the encodings of v that can appear in a log line. func secretForms(v string) []string { forms := []string{v, url.QueryEscape(v), url.PathEscape(v)} + forms = append(forms, jsonEscapedForms(v)...) for _, enc := range []*base64.Encoding{base64.StdEncoding, base64.URLEncoding} { forms = append(forms, enc.EncodeToString([]byte(v))) raw := enc.WithPadding(base64.NoPadding) @@ -88,6 +90,28 @@ func secretForms(v string) []string { return forms } +// jsonEscapedForms returns v as it is written inside a JSON string, with and +// without HTML escaping. The hive log and the request log are slog JSON and +// the output sources store JSON records, so a value that holds a quote, a +// backslash or a control character is on disk only in this form, and the +// plain form never matches it. +func jsonEscapedForms(v string) []string { + var forms []string + for _, escapeHTML := range []bool{false, true} { + var b strings.Builder + enc := json.NewEncoder(&b) + enc.SetEscapeHTML(escapeHTML) + if err := enc.Encode(v); err != nil { + continue + } + quoted := strings.TrimSuffix(b.String(), "\n") + if len(quoted) >= 2 { + forms = append(forms, quoted[1:len(quoted)-1]) + } + } + return forms +} + // base64Fragment returns the base64 characters fully determined by v when v // starts k bytes into the encoded input. The characters that mix bits of v // with the unknown neighbouring bytes are dropped from both ends. diff --git a/src/docs/design/admin-mcp.md b/src/docs/design/admin-mcp.md index 220ec4b671..41e41b41ac 100644 --- a/src/docs/design/admin-mcp.md +++ b/src/docs/design/admin-mcp.md @@ -447,7 +447,9 @@ setting. multi-value replacer over this hive's own secret values of at least 12 characters (the dashboard token, the terminal `user:token` login, the terminal key, secret file contents, and environment values whose names contain TOKEN, KEY, SECRET, PASSWORD, CREDENTIAL or - PASSPHRASE) in plain, URL-encoded and base64 forms at all three alignments, then every + PASSPHRASE) in plain, URL-encoded, JSON-escaped and base64 forms (base64 at all three + alignments; JSON-escaped because the logs are JSON, so a value holding a quote, a backslash + or a control character is on disk only in that form), then every `pkg/logscrub` category including `basic-auth`. `redactions[]` counts replacements by kind; a line that already held redaction-marker text on disk is flagged `pre_existing_marker`. Every result carries the fixed `untrusted_content` marker: log text is data, never