Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions CLAUDE.md
Original file line number Diff line number Diff line change
Expand Up @@ -185,6 +185,12 @@ hook type and tool name; its five-minute in-memory window retains hashes, not pa
Distinct long Observations sharing a prefix are stored; exact repeats within the window deduplicate.
Dedup runs before the existing privacy sanitization and does not change what gets stored.

### Observation redaction
`mem::observe` scrubs structured payload values before capture, preserving their shape and
redacting strings under secret-named keys. `stripPrivateData()` also scrubs JSON-shaped secret
assignments and an unclosed `<private>` block through the end of the text. This is defensive
redaction, not a guarantee that every possible secret format is detected.

### Observations on a full disk or a locked database
Telemetry hooks never retry, so the daemon keeps what it can. When an Observation's row write
hits a full disk or `database is locked` (another connection holds the write lock; the store
Expand Down
11 changes: 2 additions & 9 deletions src/functions/observe.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import type { CompressedObservation, RawObservation, HookPayload, Origin, Sessio
const TOOL_HOOKS = new Set(["pre_tool_use", "post_tool_use", "post_tool_failure"]);
import { KV, STREAM, generateId } from "../state/schema.js";
import { StateKV } from "../state/kv.js";
import { stripPrivateData } from "./privacy.js";
import { stripPrivateDataDeep } from "./privacy.js";
import { DedupMap } from "./dedup.js";
import { withKeyedLock } from "../state/keyed-mutex.js";
import { isAutoCompressEnabled } from "../config.js";
Expand Down Expand Up @@ -412,14 +412,7 @@ export function registerObserveFunction(
}
}

let sanitizedRaw: unknown = payload.data;
try {
const jsonStr = JSON.stringify(payload.data);
const sanitized = stripPrivateData(jsonStr);
sanitizedRaw = JSON.parse(sanitized);
} catch {
sanitizedRaw = stripPrivateData(String(payload.data));
}
const sanitizedRaw = stripPrivateDataDeep(payload.data);

let originChannel: Origin["channel"] = "agent";
if (payload.hookType === "prompt_submit") originChannel = "user";
Expand Down
22 changes: 21 additions & 1 deletion src/functions/privacy.ts
Original file line number Diff line number Diff line change
@@ -1,8 +1,11 @@
import type { ISdk } from "../engine/types.js";

const PRIVATE_TAG_RE = /<private>[\s\S]*?<\/private>/gi;
const PRIVATE_TAG_RE = /<private>[\s\S]*?(?:<\\*\/private>|$)/gi;

const SECRET_PATTERN_SOURCES = [
/\\*["'][\w-]*(?:api[_-]?key|secret|password|passwd|pwd)\\*["']\s*[=:]\s*\\*["'][^"'\\]+\\*["']/gi,
/(?:\\*["'][\w-]*(?:api[_-]?key|secret|token|password|credential|auth)\\*["']|(?:api[_-]?key|secret|token|password|credential|auth))\s*[=:]\s*\\*["']?[A-Za-z0-9_\-/.+]{20,}\\*["']?/gi,
/(?:\\*["'][\w-]*(?:api[_-]?key|secret|password|passwd|pwd)\\*["']|(?:api[_-]?key|secret|password|passwd|pwd))\s*[=:]\s*\\*["']?(?!(?:true|false|null)\b)[^\s"'\\]+\\*["']?/gi,
/(?:api[_-]?key|secret|token|password|credential|auth)[\s]*[=:]\s*["']?[A-Za-z0-9_\-/.+]{20,}["']?/gi,
/(?:api[_-]?key|secret|password|passwd|pwd)[\s]*[=:]\s*["']?[^\s"']+["']?/gi,
/Bearer\s+[A-Za-z0-9._\-+/=]{20,}/gi,
Expand Down Expand Up @@ -32,6 +35,23 @@ export function stripPrivateData(input: string): string {
return result.replace(URL_CREDENTIAL_RE, "$1[REDACTED_SECRET]$2");
}

const SECRET_KEY_RE = /(?:api[_-]?key|secret|token|password|passwd|credentials?|auth|authorization|private[_-]?key)$/i;

export function stripPrivateDataDeep(value: unknown, key?: string): unknown {
if (typeof value === "string") {
return key !== undefined && value !== "" && SECRET_KEY_RE.test(key)
? "[REDACTED_SECRET]"
: stripPrivateData(value);
}
if (Array.isArray(value)) return value.map((item) => stripPrivateDataDeep(item));
if (value !== null && typeof value === "object") {
return Object.fromEntries(
Object.entries(value).map(([k, v]) => [stripPrivateData(k), stripPrivateDataDeep(v, k)]),
);
}
return value;
}

// Every function that persists Agent-supplied free text scrubs it first,
// so a pasted token never lands in the store whichever surface wrote it.
export function scrubFields<T extends object>(data: T, ...fields: (keyof T)[]): T {
Expand Down
64 changes: 64 additions & 0 deletions test/capture-redaction.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
import { describe, it, expect, vi } from "vitest";
import { mockKV, mockSdk } from "./helpers/mocks.js";
import { registerObserveFunction } from "../src/functions/observe.js";
import { DedupMap } from "../src/functions/dedup.js";
import { KV } from "../src/state/schema.js";

vi.mock("../src/logger.js", () => ({
logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() },
}));

const SESSION = "ses_redaction";

async function observeToolCall(toolInput: Record<string, unknown>, toolOutput = "ok") {
const sdk = mockSdk({ looseTrigger: true });
const kv = mockKV();
registerObserveFunction(sdk as never, kv as never, new DedupMap());
await sdk.trigger("mem::observe", {
sessionId: SESSION,
project: "/home/user/repo",
cwd: "/home/user/repo",
hookType: "post_tool_use",
timestamp: "2026-11-18T10:00:00.000Z",
data: { tool_name: "Bash", tool_input: toolInput, tool_output: toolOutput },
});
const rows = [...(kv.store.get(KV.observations(SESSION))?.values() ?? [])];
expect(rows).toHaveLength(1);
return rows[0] as Record<string, unknown>;
}

describe("capture redaction keeps the Observation (#435)", () => {
it("keeps the tool call when a secret assignment ends a string value", async () => {
const obs = await observeToolCall({ command: "export API_KEY=abc123" });
expect(obs["toolName"]).toBe("Bash");
expect(obs["narrative"]).toContain("export");
expect(JSON.stringify(obs)).not.toContain("abc123");
});

it("redacts secrets held as JSON values under secret-named keys", async () => {
const obs = await observeToolCall({
password: "hunter2hunter2",
accessToken: "abcdefghijklmnopqrstuvwxyz0123",
client_secret: "s3cr3t-value",
max_tokens: 4096,
});
const text = JSON.stringify(obs);
expect(text).not.toContain("hunter2hunter2");
expect(text).not.toContain("abcdefghijklmnopqrstuvwxyz0123");
expect(text).not.toContain("s3cr3t-value");
expect(text).toContain("4096");
});
});

describe("capture redaction of tool output text (#435)", () => {
it("redacts a token read from a JSON config file", async () => {
const hex = "0123456789abcdef0123456789abcdef0123456789abcdef";
const obs = await observeToolCall(
{ file_path: "/home/user/.config/app/server.json" },
`{\n "endpoint": "ws://0.0.0.0:6768",\n "authToken": "${hex}"\n}`,
);
expect(obs["toolName"]).toBe("Bash");
expect(JSON.stringify(obs)).not.toContain(hex);
expect(JSON.stringify(obs)).toContain("6768");
});
});
81 changes: 80 additions & 1 deletion test/privacy.test.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { describe, it, expect } from "vitest";
import { stripPrivateData } from "../src/functions/privacy.js";
import { stripPrivateData, stripPrivateDataDeep } from "../src/functions/privacy.js";

describe("stripPrivateData scrubber boundaries", () => {
it.each([
Expand Down Expand Up @@ -140,3 +140,82 @@ describe("stripPrivateData", () => {
expect(stripPrivateData(input)).toBe("[REDACTED_SECRET]");
});
});

describe("JSON-shaped secrets (#435)", () => {
const HEX = "0123456789abcdef0123456789abcdef0123456789abcdef";

it("redacts a secret under a quoted key in JSON text", () => {
const out = stripPrivateData(`{ "authToken": "${HEX}", "port": 6768 }`);
expect(out).not.toContain(HEX);
expect(out).toContain('"port": 6768');
});

it("redacts a secret under an escaped quoted key in double-encoded JSON", () => {
const out = stripPrivateData(`{\\"token\\": \\"${HEX}\\"}`);
expect(out).not.toContain(HEX);
});

it("redacts a short password under a quoted key", () => {
expect(stripPrivateData(`{"password": "hunter2"}`)).not.toContain("hunter2");
});

it("continues to redact boolean-like password strings and unquoted assignments", () => {
expect(stripPrivateData('{"password":"true"}')).not.toContain('true');
expect(stripPrivateData('password=true')).not.toContain('true');
});

it("leaves a token count alone", () => {
expect(stripPrivateData(`{"max_tokens": 4096}`)).toBe(`{"max_tokens": 4096}`);
});

it("redacts an unclosed private block to the end of the text", () => {
expect(stripPrivateData("keep <private>cut by a hook before the close")).toBe(
"keep [REDACTED]",
);
});

it("ends a private block at an escaped close tag", () => {
const src = String.raw`const RE = /<private>[\s\S]*?<\/private>/gi; const keep = 1;`;
expect(stripPrivateData(src)).toContain("const keep = 1;");
});

it.each([
String.raw`docker run -v "$PWD":/src golang`,
String.raw`if [[ "$PWD" == "$work" ]]; then`,
"ls: cannot access '/srv/nkos/admin-password': No such file or directory",
String.raw`{"has_secret":true,"redirects":[]}`,
])("leaves text that only looks like a quoted key: %s", (text) => {
expect(stripPrivateData(text)).toBe(text);
});
});

describe("stripPrivateDataDeep", () => {
it("redacts string values under secret-named keys at any depth", () => {
const out = stripPrivateDataDeep({
tool_input: { headers: { Authorization: "abc", "x-api-key": "k1" }, accessToken: "t" },
list: [{ client_secret: "s" }],
});
expect(out).toEqual({
tool_input: {
headers: { Authorization: "[REDACTED_SECRET]", "x-api-key": "[REDACTED_SECRET]" },
accessToken: "[REDACTED_SECRET]",
},
list: [{ client_secret: "[REDACTED_SECRET]" }],
});
});

it("keeps shape, numbers, booleans, nulls and unrelated keys", () => {
const input = { max_tokens: 4096, has_secret: true, token: "", n: null, file_path: "/a/b" };
expect(stripPrivateDataDeep(input)).toEqual(input);
});

it("scrubs secret assignments in object keys as well as values", () => {
expect(JSON.stringify(stripPrivateDataDeep({ "password=abc123": "value" }))).not.toContain("abc123");
});

it("scrubs secrets inside ordinary string values", () => {
expect(stripPrivateDataDeep({ command: "export API_KEY=abc123" })).toEqual({
command: "export [REDACTED_SECRET]",
});
});
});