diff --git a/CLAUDE.md b/CLAUDE.md index da792d348c..5759bbab2b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -185,6 +185,12 @@ hook type and tool name; its five-minute in-memory window retains hashes, not pa Distinct long Observations sharing a prefix are stored; exact repeats within the window deduplicate. Dedup runs before the existing privacy sanitization and does not change what gets stored. +### Observation redaction +`mem::observe` scrubs structured payload values before capture, preserving their shape and +redacting strings under secret-named keys. `stripPrivateData()` also scrubs JSON-shaped secret +assignments and an unclosed `` block through the end of the text. This is defensive +redaction, not a guarantee that every possible secret format is detected. + ### Observations on a full disk or a locked database Telemetry hooks never retry, so the daemon keeps what it can. When an Observation's row write hits a full disk or `database is locked` (another connection holds the write lock; the store diff --git a/src/functions/observe.ts b/src/functions/observe.ts index 06ceebc9a0..eed07e2a7a 100644 --- a/src/functions/observe.ts +++ b/src/functions/observe.ts @@ -4,7 +4,7 @@ import type { CompressedObservation, RawObservation, HookPayload, Origin, Sessio const TOOL_HOOKS = new Set(["pre_tool_use", "post_tool_use", "post_tool_failure"]); import { KV, STREAM, generateId } from "../state/schema.js"; import { StateKV } from "../state/kv.js"; -import { stripPrivateData } from "./privacy.js"; +import { stripPrivateDataDeep } from "./privacy.js"; import { DedupMap } from "./dedup.js"; import { withKeyedLock } from "../state/keyed-mutex.js"; import { isAutoCompressEnabled } from "../config.js"; @@ -412,14 +412,7 @@ export function registerObserveFunction( } } - let sanitizedRaw: unknown = payload.data; - try { - const jsonStr = JSON.stringify(payload.data); - const sanitized = stripPrivateData(jsonStr); - sanitizedRaw = JSON.parse(sanitized); - } catch { - sanitizedRaw = stripPrivateData(String(payload.data)); - } + const sanitizedRaw = stripPrivateDataDeep(payload.data); let originChannel: Origin["channel"] = "agent"; if (payload.hookType === "prompt_submit") originChannel = "user"; diff --git a/src/functions/privacy.ts b/src/functions/privacy.ts index a9d44720c0..891b7878ab 100644 --- a/src/functions/privacy.ts +++ b/src/functions/privacy.ts @@ -1,8 +1,11 @@ import type { ISdk } from "../engine/types.js"; -const PRIVATE_TAG_RE = /[\s\S]*?<\/private>/gi; +const PRIVATE_TAG_RE = /[\s\S]*?(?:<\\*\/private>|$)/gi; const SECRET_PATTERN_SOURCES = [ + /\\*["'][\w-]*(?:api[_-]?key|secret|password|passwd|pwd)\\*["']\s*[=:]\s*\\*["'][^"'\\]+\\*["']/gi, + /(?:\\*["'][\w-]*(?:api[_-]?key|secret|token|password|credential|auth)\\*["']|(?:api[_-]?key|secret|token|password|credential|auth))\s*[=:]\s*\\*["']?[A-Za-z0-9_\-/.+]{20,}\\*["']?/gi, + /(?:\\*["'][\w-]*(?:api[_-]?key|secret|password|passwd|pwd)\\*["']|(?:api[_-]?key|secret|password|passwd|pwd))\s*[=:]\s*\\*["']?(?!(?:true|false|null)\b)[^\s"'\\]+\\*["']?/gi, /(?:api[_-]?key|secret|token|password|credential|auth)[\s]*[=:]\s*["']?[A-Za-z0-9_\-/.+]{20,}["']?/gi, /(?:api[_-]?key|secret|password|passwd|pwd)[\s]*[=:]\s*["']?[^\s"']+["']?/gi, /Bearer\s+[A-Za-z0-9._\-+/=]{20,}/gi, @@ -32,6 +35,23 @@ export function stripPrivateData(input: string): string { return result.replace(URL_CREDENTIAL_RE, "$1[REDACTED_SECRET]$2"); } +const SECRET_KEY_RE = /(?:api[_-]?key|secret|token|password|passwd|credentials?|auth|authorization|private[_-]?key)$/i; + +export function stripPrivateDataDeep(value: unknown, key?: string): unknown { + if (typeof value === "string") { + return key !== undefined && value !== "" && SECRET_KEY_RE.test(key) + ? "[REDACTED_SECRET]" + : stripPrivateData(value); + } + if (Array.isArray(value)) return value.map((item) => stripPrivateDataDeep(item)); + if (value !== null && typeof value === "object") { + return Object.fromEntries( + Object.entries(value).map(([k, v]) => [stripPrivateData(k), stripPrivateDataDeep(v, k)]), + ); + } + return value; +} + // Every function that persists Agent-supplied free text scrubs it first, // so a pasted token never lands in the store whichever surface wrote it. export function scrubFields(data: T, ...fields: (keyof T)[]): T { diff --git a/test/capture-redaction.test.ts b/test/capture-redaction.test.ts new file mode 100644 index 0000000000..a1c7a6f366 --- /dev/null +++ b/test/capture-redaction.test.ts @@ -0,0 +1,64 @@ +import { describe, it, expect, vi } from "vitest"; +import { mockKV, mockSdk } from "./helpers/mocks.js"; +import { registerObserveFunction } from "../src/functions/observe.js"; +import { DedupMap } from "../src/functions/dedup.js"; +import { KV } from "../src/state/schema.js"; + +vi.mock("../src/logger.js", () => ({ + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, +})); + +const SESSION = "ses_redaction"; + +async function observeToolCall(toolInput: Record, toolOutput = "ok") { + const sdk = mockSdk({ looseTrigger: true }); + const kv = mockKV(); + registerObserveFunction(sdk as never, kv as never, new DedupMap()); + await sdk.trigger("mem::observe", { + sessionId: SESSION, + project: "/home/user/repo", + cwd: "/home/user/repo", + hookType: "post_tool_use", + timestamp: "2026-11-18T10:00:00.000Z", + data: { tool_name: "Bash", tool_input: toolInput, tool_output: toolOutput }, + }); + const rows = [...(kv.store.get(KV.observations(SESSION))?.values() ?? [])]; + expect(rows).toHaveLength(1); + return rows[0] as Record; +} + +describe("capture redaction keeps the Observation (#435)", () => { + it("keeps the tool call when a secret assignment ends a string value", async () => { + const obs = await observeToolCall({ command: "export API_KEY=abc123" }); + expect(obs["toolName"]).toBe("Bash"); + expect(obs["narrative"]).toContain("export"); + expect(JSON.stringify(obs)).not.toContain("abc123"); + }); + + it("redacts secrets held as JSON values under secret-named keys", async () => { + const obs = await observeToolCall({ + password: "hunter2hunter2", + accessToken: "abcdefghijklmnopqrstuvwxyz0123", + client_secret: "s3cr3t-value", + max_tokens: 4096, + }); + const text = JSON.stringify(obs); + expect(text).not.toContain("hunter2hunter2"); + expect(text).not.toContain("abcdefghijklmnopqrstuvwxyz0123"); + expect(text).not.toContain("s3cr3t-value"); + expect(text).toContain("4096"); + }); +}); + +describe("capture redaction of tool output text (#435)", () => { + it("redacts a token read from a JSON config file", async () => { + const hex = "0123456789abcdef0123456789abcdef0123456789abcdef"; + const obs = await observeToolCall( + { file_path: "/home/user/.config/app/server.json" }, + `{\n "endpoint": "ws://0.0.0.0:6768",\n "authToken": "${hex}"\n}`, + ); + expect(obs["toolName"]).toBe("Bash"); + expect(JSON.stringify(obs)).not.toContain(hex); + expect(JSON.stringify(obs)).toContain("6768"); + }); +}); diff --git a/test/privacy.test.ts b/test/privacy.test.ts index 48b817f404..1b3427d96d 100644 --- a/test/privacy.test.ts +++ b/test/privacy.test.ts @@ -1,5 +1,5 @@ import { describe, it, expect } from "vitest"; -import { stripPrivateData } from "../src/functions/privacy.js"; +import { stripPrivateData, stripPrivateDataDeep } from "../src/functions/privacy.js"; describe("stripPrivateData scrubber boundaries", () => { it.each([ @@ -140,3 +140,82 @@ describe("stripPrivateData", () => { expect(stripPrivateData(input)).toBe("[REDACTED_SECRET]"); }); }); + +describe("JSON-shaped secrets (#435)", () => { + const HEX = "0123456789abcdef0123456789abcdef0123456789abcdef"; + + it("redacts a secret under a quoted key in JSON text", () => { + const out = stripPrivateData(`{ "authToken": "${HEX}", "port": 6768 }`); + expect(out).not.toContain(HEX); + expect(out).toContain('"port": 6768'); + }); + + it("redacts a secret under an escaped quoted key in double-encoded JSON", () => { + const out = stripPrivateData(`{\\"token\\": \\"${HEX}\\"}`); + expect(out).not.toContain(HEX); + }); + + it("redacts a short password under a quoted key", () => { + expect(stripPrivateData(`{"password": "hunter2"}`)).not.toContain("hunter2"); + }); + + it("continues to redact boolean-like password strings and unquoted assignments", () => { + expect(stripPrivateData('{"password":"true"}')).not.toContain('true'); + expect(stripPrivateData('password=true')).not.toContain('true'); + }); + + it("leaves a token count alone", () => { + expect(stripPrivateData(`{"max_tokens": 4096}`)).toBe(`{"max_tokens": 4096}`); + }); + + it("redacts an unclosed private block to the end of the text", () => { + expect(stripPrivateData("keep cut by a hook before the close")).toBe( + "keep [REDACTED]", + ); + }); + + it("ends a private block at an escaped close tag", () => { + const src = String.raw`const RE = /[\s\S]*?<\/private>/gi; const keep = 1;`; + expect(stripPrivateData(src)).toContain("const keep = 1;"); + }); + + it.each([ + String.raw`docker run -v "$PWD":/src golang`, + String.raw`if [[ "$PWD" == "$work" ]]; then`, + "ls: cannot access '/srv/nkos/admin-password': No such file or directory", + String.raw`{"has_secret":true,"redirects":[]}`, + ])("leaves text that only looks like a quoted key: %s", (text) => { + expect(stripPrivateData(text)).toBe(text); + }); +}); + +describe("stripPrivateDataDeep", () => { + it("redacts string values under secret-named keys at any depth", () => { + const out = stripPrivateDataDeep({ + tool_input: { headers: { Authorization: "abc", "x-api-key": "k1" }, accessToken: "t" }, + list: [{ client_secret: "s" }], + }); + expect(out).toEqual({ + tool_input: { + headers: { Authorization: "[REDACTED_SECRET]", "x-api-key": "[REDACTED_SECRET]" }, + accessToken: "[REDACTED_SECRET]", + }, + list: [{ client_secret: "[REDACTED_SECRET]" }], + }); + }); + + it("keeps shape, numbers, booleans, nulls and unrelated keys", () => { + const input = { max_tokens: 4096, has_secret: true, token: "", n: null, file_path: "/a/b" }; + expect(stripPrivateDataDeep(input)).toEqual(input); + }); + + it("scrubs secret assignments in object keys as well as values", () => { + expect(JSON.stringify(stripPrivateDataDeep({ "password=abc123": "value" }))).not.toContain("abc123"); + }); + + it("scrubs secrets inside ordinary string values", () => { + expect(stripPrivateDataDeep({ command: "export API_KEY=abc123" })).toEqual({ + command: "export [REDACTED_SECRET]", + }); + }); +});