Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/security-logread-secret-json-escaped.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- `log_read` (admin MCP) and `GET /api/logs/read`: one of the hive's own secret values is now redacted when it holds a quote, a backslash or a control character. The hive log, the request log and the output sources are stored as JSON, so such a value is on disk only in its JSON-escaped form (`pa\"ss\\word`). The read-time redaction looked for the plain, URL-encoded and base64 forms only, so it missed it and the value came back readable in the returned line. The JSON-escaped form, with and without HTML escaping, is now redacted as well. Values made only of letters, digits and ordinary punctuation were always redacted.
80 changes: 80 additions & 0 deletions src/crux/logread/secret_json_form_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
package logread

import (
"bytes"
"encoding/json"
"log/slog"
"os"
"path/filepath"
"strings"
"testing"
)

// The hive log is slog JSON. A secret value that holds a quote, a backslash
// or a control character is written there only in its JSON-escaped form, so
// the plain value never matched and the secret came back readable.
func TestOwnSecretIsRedactedInItsJSONEscapedForm(t *testing.T) {
for name, secret := range map[string]string{
"plain": `plain-secret-0123456789`,
"quote, backslash": `pa"ss\word-0123456789`,
"tab and newline": "line1-secret\tcol\nline2-0123",
"escape character": "esc\x1bseq-secret-0123456789",
"html characters": `a<b>&c-secret-0123456789`,
"non-ASCII letters": `pässwörd-geheim-0123456789`,
} {
t.Run(name, func(t *testing.T) {
dir := t.TempDir()
var buf bytes.Buffer
lg := slog.New(slog.NewJSONHandler(&buf, nil))
lg.Info("connecting", "password", secret)
lg.Info("tail")
if err := os.WriteFile(filepath.Join(dir, HiveLogFilename), buf.Bytes(), 0o600); err != nil {
t.Fatal(err)
}
res := Read(Request{}, Options{LogDir: dir, Secrets: NewSecretScrubber([]string{secret})})
if res.LineCount != 2 {
t.Fatalf("read %d lines, want 2", res.LineCount)
}
redacted := 0
for _, line := range res.Lines {
var record map[string]any
if err := json.Unmarshal([]byte(line.Text), &record); err == nil && record["password"] == secret {
t.Errorf("the secret is readable in the returned line: %s", line.Text)
}
redacted += strings.Count(line.Text, secretValueMarker)
}
if redacted != 1 {
t.Errorf("redaction markers in the returned lines = %d, want 1 (redactions=%v)", redacted, res.Redactions)
}
})
}
}

// A record written with encoding/json's default HTML escaping (as the output
// sources' Go writer does) is covered too.
func TestOwnSecretIsRedactedInHTMLEscapedJSON(t *testing.T) {
const secret = `a<b>&c "quoted" secret-0123456789`
record, err := json.Marshal(map[string]string{"text": "token is " + secret + " end"})
if err != nil {
t.Fatal(err)
}
out, n := NewSecretScrubber([]string{secret}).Replace(string(record))
if n != 1 || strings.Contains(out, "secret-0123456789") {
t.Fatalf("replaced %d, result %s", n, out)
}
}

// Ordinary text is left alone, and a value with nothing to escape gains no
// new form.
func TestJSONEscapedFormsAddNothingForPlainValues(t *testing.T) {
const secret = "plain-secret-0123456789"
for _, form := range jsonEscapedForms(secret) {
if form != secret {
t.Errorf("plain value gained the form %q", form)
}
}
text := `{"msg":"nothing secret here","path":"C:\\temp\\file \"quoted\""}`
if out, n := NewSecretScrubber([]string{`pa"ss\word-0123456789`}).Replace(text); n != 0 || out != text {
t.Errorf("unrelated text changed: %d replacements, %s", n, out)
}
}
34 changes: 29 additions & 5 deletions src/crux/logread/secrets.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package logread

import (
"encoding/base64"
"encoding/json"
"io/fs"
"net/url"
"os"
Expand All @@ -16,11 +17,11 @@ const SecretValueKind = "hive-secret-value"
const secretValueMarker = "<redacted:" + SecretValueKind + ">"

// SecretScrubber replaces this hive's own secret values in log text: each
// value of at least MinSecretValueLen characters in its plain, URL-encoded and
// base64 forms, the base64 forms at all three byte alignments so the value is
// found inside a longer encoded string (an "Authorization: Basic" header
// encoding user:value). It is one precompiled multi-value replacer, built in
// process for one read; the values are never returned.
// value of at least MinSecretValueLen characters in its plain, URL-encoded,
// JSON-escaped and base64 forms, the base64 forms at all three byte alignments
// so the value is found inside a longer encoded string (an "Authorization:
// Basic" header encoding user:value). It is one precompiled multi-value
// replacer, built in process for one read; the values are never returned.
type SecretScrubber struct {
r *strings.Replacer
}
Expand Down Expand Up @@ -78,6 +79,7 @@ func (s *SecretScrubber) Replace(text string) (string, int) {
// secretForms returns the encodings of v that can appear in a log line.
func secretForms(v string) []string {
forms := []string{v, url.QueryEscape(v), url.PathEscape(v)}
forms = append(forms, jsonEscapedForms(v)...)
for _, enc := range []*base64.Encoding{base64.StdEncoding, base64.URLEncoding} {
forms = append(forms, enc.EncodeToString([]byte(v)))
raw := enc.WithPadding(base64.NoPadding)
Expand All @@ -88,6 +90,28 @@ func secretForms(v string) []string {
return forms
}

// jsonEscapedForms returns v as it is written inside a JSON string, with and
// without HTML escaping. The hive log and the request log are slog JSON and
// the output sources store JSON records, so a value that holds a quote, a
// backslash or a control character is on disk only in this form, and the
// plain form never matches it.
func jsonEscapedForms(v string) []string {
var forms []string
for _, escapeHTML := range []bool{false, true} {
var b strings.Builder
enc := json.NewEncoder(&b)
enc.SetEscapeHTML(escapeHTML)
if err := enc.Encode(v); err != nil {
continue
}
quoted := strings.TrimSuffix(b.String(), "\n")
if len(quoted) >= 2 {
forms = append(forms, quoted[1:len(quoted)-1])
}
}
return forms
}

// base64Fragment returns the base64 characters fully determined by v when v
// starts k bytes into the encoded input. The characters that mix bits of v
// with the unknown neighbouring bytes are dropped from both ends.
Expand Down
4 changes: 3 additions & 1 deletion src/docs/design/admin-mcp.md
Original file line number Diff line number Diff line change
Expand Up @@ -447,7 +447,9 @@ setting.
multi-value replacer over this hive's own secret values of at least 12 characters (the
dashboard token, the terminal `user:token` login, the terminal key, secret file contents,
and environment values whose names contain TOKEN, KEY, SECRET, PASSWORD, CREDENTIAL or
PASSPHRASE) in plain, URL-encoded and base64 forms at all three alignments, then every
PASSPHRASE) in plain, URL-encoded, JSON-escaped and base64 forms (base64 at all three
alignments; JSON-escaped because the logs are JSON, so a value holding a quote, a backslash
or a control character is on disk only in that form), then every
`pkg/logscrub` category including `basic-auth`. `redactions[]` counts replacements by kind;
a line that already held redaction-marker text on disk is flagged `pre_existing_marker`.
Every result carries the fixed `untrusted_content` marker: log text is data, never
Expand Down
Loading