diff --git a/plugin/scripts/_capture.mjs b/plugin/scripts/_capture.mjs index 83bc5d0d17..5ccd50a168 100644 --- a/plugin/scripts/_capture.mjs +++ b/plugin/scripts/_capture.mjs @@ -332,8 +332,10 @@ const SECRET_PATTERN_SOURCES = [ const PRIVATE_KEY_BLOCK_RE = /-----BEGIN [A-Z0-9 ]*PRIVATE KEY(?: BLOCK)?-----[\s\S]*?(?:-----END [A-Z0-9 ]*PRIVATE KEY(?: BLOCK)?-----|$)/g; const URL_CREDENTIALS_RE = /\b([a-z][a-z0-9+.-]*:\/\/)[^\s/?#@:"'<>]*:[^\s/?#@"'<>]+@/gi; function stripPrivateData(input) { - let result = redactPrivateBlocks(input); - result = result.replace(new RegExp(PRIVATE_KEY_BLOCK_RE.source, PRIVATE_KEY_BLOCK_RE.flags), "[REDACTED_SECRET]"); + return stripSecrets(redactPrivateBlocks(input)); +} +function stripSecrets(input) { + let result = input.replace(new RegExp(PRIVATE_KEY_BLOCK_RE.source, PRIVATE_KEY_BLOCK_RE.flags), "[REDACTED_SECRET]"); result = result.replace(new RegExp(URL_CREDENTIALS_RE.source, URL_CREDENTIALS_RE.flags), "$1[REDACTED_SECRET]@"); for (const source of SECRET_PATTERN_SOURCES) { const pattern = new RegExp(source.source, source.flags); @@ -342,12 +344,13 @@ function stripPrivateData(input) { return result; } function scrubRecord(value, options = {}) { - if (typeof value === "string") return stripPrivateData(value); + const strip = options.strip ?? stripPrivateData; + if (typeof value === "string") return strip(value); if (Array.isArray(value)) return value.map((item) => scrubRecord(item, options)); if (value !== null && typeof value === "object") { const proto = Object.getPrototypeOf(value); if (proto !== Object.prototype && proto !== null) return value; - const entries = Object.entries(value).map(([key, item]) => [options.scrubKeys ? stripPrivateData(key) : key, item]); + const entries = Object.entries(value).map(([key, item]) => [options.scrubKeys ? strip(key) : key, item]); const reserved = new Set(entries.map(([key]) => key)); const used = /* @__PURE__ */ new Set(); return Object.fromEntries(entries.map(([key, item]) => { diff --git a/plugin/scripts/post-tool-failure.mjs b/plugin/scripts/post-tool-failure.mjs index 85cf906e5c..aa941c63e3 100755 --- a/plugin/scripts/post-tool-failure.mjs +++ b/plugin/scripts/post-tool-failure.mjs @@ -3,6 +3,85 @@ import { execSync } from "node:child_process"; import { realpathSync } from "node:fs"; import { basename, dirname, resolve } from "node:path"; import { captureObservation, isDrainChild, runDrainChild, withEventId } from "./_capture.mjs"; +//#region src/functions/privacy.ts +const PRIVATE_TAG_TOKEN_RE = /)[^>]*>|<\/private\s*>/gi; +function redactPrivateBlocks(input) { + const tokens = new RegExp(PRIVATE_TAG_TOKEN_RE.source, PRIVATE_TAG_TOKEN_RE.flags); + let out = ""; + let cursor = 0; + let depth = 0; + for (let match = tokens.exec(input); match; match = tokens.exec(input)) { + const closing = match[0].startsWith(" 0) return `${out}[REDACTED]`; + return out + input.slice(cursor); +} +const SECRET_PATTERN_SOURCES = [ + /(?:api[_-]?key|secret|token|password|credential|auth)[\s]*[=:]\s*["']?[A-Za-z0-9_\-/.+]{20,}["']?/gi, + /Bearer\s+[A-Za-z0-9._\-+/=]{20,}/gi, + /(?]*:[^\s/?#@"'<>]+@/gi; +function stripPrivateData(input) { + return stripSecrets(redactPrivateBlocks(input)); +} +function stripSecrets(input) { + let result = input.replace(new RegExp(PRIVATE_KEY_BLOCK_RE.source, PRIVATE_KEY_BLOCK_RE.flags), "[REDACTED_SECRET]"); + result = result.replace(new RegExp(URL_CREDENTIALS_RE.source, URL_CREDENTIALS_RE.flags), "$1[REDACTED_SECRET]@"); + for (const source of SECRET_PATTERN_SOURCES) { + const pattern = new RegExp(source.source, source.flags); + result = result.replace(pattern, "[REDACTED_SECRET]"); + } + return result; +} +function scrubRecord(value, options = {}) { + const strip = options.strip ?? stripPrivateData; + if (typeof value === "string") return strip(value); + if (Array.isArray(value)) return value.map((item) => scrubRecord(item, options)); + if (value !== null && typeof value === "object") { + const proto = Object.getPrototypeOf(value); + if (proto !== Object.prototype && proto !== null) return value; + const entries = Object.entries(value).map(([key, item]) => [options.scrubKeys ? strip(key) : key, item]); + const reserved = new Set(entries.map(([key]) => key)); + const used = /* @__PURE__ */ new Set(); + return Object.fromEntries(entries.map(([key, item]) => { + let unique = key; + if (used.has(unique)) { + let suffix = 2; + do + unique = `${key}_${suffix++}`; + while (reserved.has(unique) || used.has(unique)); + } + used.add(unique); + return [unique, scrubRecord(item, options)]; + })); + } + return value; +} +//#endregion //#region src/hooks/_capture-filter.ts const DEFAULT_DENY_PATTERNS = [ "memory_*", @@ -43,11 +122,57 @@ function shouldCaptureTool(toolName) { if (allow) return matchesAny(toolName, allow); return !matchesAny(toolName, [...DEFAULT_DENY_PATTERNS, ...parseEnvList(process.env["AGENTMEMORY_CAPTURE_DENY"]) ?? []]); } -function truncateText(value, max) { +function cut(value, max) { if (value.length <= max) return value; const end = /[\uD800-\uDBFF]/.test(value[max - 1] ?? "") ? max - 1 : max; return value.slice(0, end); } +function truncateText(value, max) { + if (value.length <= max) return value; + return cut(stripPrivateData(value), max); +} +const JSON_WHITESPACE_ESCAPES = { + n: "\n", + r: "\r", + t: " " +}; +const SECRET_KEY_WORDS = new Set([ + "apikey", + "secret", + "token", + "password", + "passwd", + "credential", + "credentials", + "auth", + "authorization" +]); +function isSecretKeyName(key) { + if (!/^[\w.-]{1,64}$/.test(key)) return false; + const words = key.replace(/([a-z0-9])([A-Z])/g, "$1_$2").toLowerCase().split(/[_.-]+/); + return words.some((word, i) => SECRET_KEY_WORDS.has(word) || word === "api" && words[i + 1] === "key"); +} +function redactSecretKeys(value) { + if (Array.isArray(value)) return value.map(redactSecretKeys); + if (value === null || typeof value !== "object") return value; + const proto = Object.getPrototypeOf(value); + if (proto !== Object.prototype && proto !== null) return value; + return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, typeof item === "string" && item && isSecretKeyName(key) ? "[REDACTED_SECRET]" : redactSecretKeys(item)])); +} +function scrubSecrets(value) { + return scrubRecord(redactSecretKeys(value), { + scrubKeys: true, + strip: stripSecrets + }); +} +function redactSerialized(serialized) { + return stripPrivateData(serialized.replace(/\\([\\nrt])/g, (escape, c) => JSON_WHITESPACE_ESCAPES[c] ?? escape)).replace(/[\n\r\t]/g, (c) => JSON.stringify(c).slice(1, -1)); +} +function truncateJson(value, max) { + const str = JSON.stringify(value); + if (str.length <= max) return str; + return cut(redactSerialized(JSON.stringify(scrubSecrets(value))), max); +} //#endregion //#region src/hooks/_project.ts function resolveProject(cwd) { @@ -121,8 +246,8 @@ async function main() { timestamp: (/* @__PURE__ */ new Date()).toISOString(), data: { tool_name: toolName, - tool_input: typeof toolInput === "string" ? truncateText(toolInput, 4e3) : truncateText(JSON.stringify(toolInput ?? ""), 4e3), - error: typeof error === "string" ? truncateText(error, 4e3) : truncateText(JSON.stringify(error ?? ""), 4e3) + tool_input: typeof toolInput === "string" ? truncateText(toolInput, 4e3) : truncateJson(toolInput ?? "", 4e3), + error: typeof error === "string" ? truncateText(error, 4e3) : truncateJson(error ?? "", 4e3) } }, data), OBSERVE_TIMEOUT_MS); setTimeout(() => process.exit(0), EXIT_CAP_MS).unref(); diff --git a/plugin/scripts/post-tool-use.mjs b/plugin/scripts/post-tool-use.mjs index 08c542899a..bda61f130a 100755 --- a/plugin/scripts/post-tool-use.mjs +++ b/plugin/scripts/post-tool-use.mjs @@ -3,6 +3,85 @@ import { execSync } from "node:child_process"; import { realpathSync } from "node:fs"; import { basename, dirname, resolve } from "node:path"; import { captureObservation, isDrainChild, runDrainChild, withEventId } from "./_capture.mjs"; +//#region src/functions/privacy.ts +const PRIVATE_TAG_TOKEN_RE = /)[^>]*>|<\/private\s*>/gi; +function redactPrivateBlocks(input) { + const tokens = new RegExp(PRIVATE_TAG_TOKEN_RE.source, PRIVATE_TAG_TOKEN_RE.flags); + let out = ""; + let cursor = 0; + let depth = 0; + for (let match = tokens.exec(input); match; match = tokens.exec(input)) { + const closing = match[0].startsWith(" 0) return `${out}[REDACTED]`; + return out + input.slice(cursor); +} +const SECRET_PATTERN_SOURCES = [ + /(?:api[_-]?key|secret|token|password|credential|auth)[\s]*[=:]\s*["']?[A-Za-z0-9_\-/.+]{20,}["']?/gi, + /Bearer\s+[A-Za-z0-9._\-+/=]{20,}/gi, + /(?]*:[^\s/?#@"'<>]+@/gi; +function stripPrivateData(input) { + return stripSecrets(redactPrivateBlocks(input)); +} +function stripSecrets(input) { + let result = input.replace(new RegExp(PRIVATE_KEY_BLOCK_RE.source, PRIVATE_KEY_BLOCK_RE.flags), "[REDACTED_SECRET]"); + result = result.replace(new RegExp(URL_CREDENTIALS_RE.source, URL_CREDENTIALS_RE.flags), "$1[REDACTED_SECRET]@"); + for (const source of SECRET_PATTERN_SOURCES) { + const pattern = new RegExp(source.source, source.flags); + result = result.replace(pattern, "[REDACTED_SECRET]"); + } + return result; +} +function scrubRecord(value, options = {}) { + const strip = options.strip ?? stripPrivateData; + if (typeof value === "string") return strip(value); + if (Array.isArray(value)) return value.map((item) => scrubRecord(item, options)); + if (value !== null && typeof value === "object") { + const proto = Object.getPrototypeOf(value); + if (proto !== Object.prototype && proto !== null) return value; + const entries = Object.entries(value).map(([key, item]) => [options.scrubKeys ? strip(key) : key, item]); + const reserved = new Set(entries.map(([key]) => key)); + const used = /* @__PURE__ */ new Set(); + return Object.fromEntries(entries.map(([key, item]) => { + let unique = key; + if (used.has(unique)) { + let suffix = 2; + do + unique = `${key}_${suffix++}`; + while (reserved.has(unique) || used.has(unique)); + } + used.add(unique); + return [unique, scrubRecord(item, options)]; + })); + } + return value; +} +//#endregion //#region src/hooks/_capture-filter.ts const DEFAULT_DENY_PATTERNS = [ "memory_*", @@ -49,22 +128,57 @@ function captureOutputMax() { const parsed = Number.parseInt(raw, 10); return Number.isFinite(parsed) && parsed > 0 ? parsed : 8e3; } -function truncateText(value, max) { +function cut(value, max) { if (value.length <= max) return value; const end = /[\uD800-\uDBFF]/.test(value[max - 1] ?? "") ? max - 1 : max; return value.slice(0, end); } +const JSON_WHITESPACE_ESCAPES = { + n: "\n", + r: "\r", + t: " " +}; +const SECRET_KEY_WORDS = new Set([ + "apikey", + "secret", + "token", + "password", + "passwd", + "credential", + "credentials", + "auth", + "authorization" +]); +function isSecretKeyName(key) { + if (!/^[\w.-]{1,64}$/.test(key)) return false; + const words = key.replace(/([a-z0-9])([A-Z])/g, "$1_$2").toLowerCase().split(/[_.-]+/); + return words.some((word, i) => SECRET_KEY_WORDS.has(word) || word === "api" && words[i + 1] === "key"); +} +function redactSecretKeys(value) { + if (Array.isArray(value)) return value.map(redactSecretKeys); + if (value === null || typeof value !== "object") return value; + const proto = Object.getPrototypeOf(value); + if (proto !== Object.prototype && proto !== null) return value; + return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, typeof item === "string" && item && isSecretKeyName(key) ? "[REDACTED_SECRET]" : redactSecretKeys(item)])); +} +function scrubSecrets(value) { + return scrubRecord(redactSecretKeys(value), { + scrubKeys: true, + strip: stripSecrets + }); +} +function redactSerialized(serialized) { + return stripPrivateData(serialized.replace(/\\([\\nrt])/g, (escape, c) => JSON_WHITESPACE_ESCAPES[c] ?? escape)).replace(/[\n\r\t]/g, (c) => JSON.stringify(c).slice(1, -1)); +} +function truncateWithSuffix(value, max, suffix) { + const safe = stripPrivateData(value); + if (safe.length <= max) return safe; + return max <= suffix.length ? suffix.slice(0, max) : cut(safe, max - suffix.length) + suffix; +} function truncateCaptureOutput(value, max) { - if (typeof value === "string" && value.length > max) { - const suffix = "\n[...truncated]"; - return max <= 15 ? suffix.slice(0, max) : truncateText(value, max - 15) + suffix; - } + if (typeof value === "string" && value.length > max) return truncateWithSuffix(value, max, "\n[...truncated]"); if (typeof value === "object" && value !== null) { - const str = JSON.stringify(value); - if (str.length > max) { - const suffix = "...[truncated]"; - return max <= 14 ? suffix.slice(0, max) : truncateText(str, max - 14) + suffix; - } + if (JSON.stringify(value).length > max) return truncateWithSuffix(redactSerialized(JSON.stringify(scrubSecrets(value))), max, "...[truncated]"); return value; } return value; diff --git a/plugin/scripts/session-end.mjs b/plugin/scripts/session-end.mjs index d64c0b2797..8761e46c23 100755 --- a/plugin/scripts/session-end.mjs +++ b/plugin/scripts/session-end.mjs @@ -3,12 +3,71 @@ import { readFileSync, realpathSync } from "node:fs"; import { execSync } from "node:child_process"; import { basename, dirname, resolve } from "node:path"; import { REST_URL, authHeaders, captureObservation, isDrainChild, runDrainChild, transportBlocked, withEventId } from "./_capture.mjs"; +//#region src/functions/privacy.ts +const PRIVATE_TAG_TOKEN_RE = /)[^>]*>|<\/private\s*>/gi; +function redactPrivateBlocks(input) { + const tokens = new RegExp(PRIVATE_TAG_TOKEN_RE.source, PRIVATE_TAG_TOKEN_RE.flags); + let out = ""; + let cursor = 0; + let depth = 0; + for (let match = tokens.exec(input); match; match = tokens.exec(input)) { + const closing = match[0].startsWith(" 0) return `${out}[REDACTED]`; + return out + input.slice(cursor); +} +const SECRET_PATTERN_SOURCES = [ + /(?:api[_-]?key|secret|token|password|credential|auth)[\s]*[=:]\s*["']?[A-Za-z0-9_\-/.+]{20,}["']?/gi, + /Bearer\s+[A-Za-z0-9._\-+/=]{20,}/gi, + /(?]*:[^\s/?#@"'<>]+@/gi; +function stripPrivateData(input) { + return stripSecrets(redactPrivateBlocks(input)); +} +function stripSecrets(input) { + let result = input.replace(new RegExp(PRIVATE_KEY_BLOCK_RE.source, PRIVATE_KEY_BLOCK_RE.flags), "[REDACTED_SECRET]"); + result = result.replace(new RegExp(URL_CREDENTIALS_RE.source, URL_CREDENTIALS_RE.flags), "$1[REDACTED_SECRET]@"); + for (const source of SECRET_PATTERN_SOURCES) { + const pattern = new RegExp(source.source, source.flags); + result = result.replace(pattern, "[REDACTED_SECRET]"); + } + return result; +} +//#endregion //#region src/hooks/_capture-filter.ts -function truncateText(value, max) { +function cut(value, max) { if (value.length <= max) return value; const end = /[\uD800-\uDBFF]/.test(value[max - 1] ?? "") ? max - 1 : max; return value.slice(0, end); } +function truncateText(value, max) { + if (value.length <= max) return value; + return cut(stripPrivateData(value), max); +} //#endregion //#region src/hooks/_project.ts function resolveProject(cwd) { diff --git a/plugin/scripts/subagent-stop.mjs b/plugin/scripts/subagent-stop.mjs index a01bff6cd0..4b92d4c17f 100755 --- a/plugin/scripts/subagent-stop.mjs +++ b/plugin/scripts/subagent-stop.mjs @@ -3,12 +3,71 @@ import { execSync } from "node:child_process"; import { realpathSync } from "node:fs"; import { basename, dirname, resolve } from "node:path"; import { captureObservation, isDrainChild, runDrainChild, stripPrivateData, withEventId } from "./_capture.mjs"; +//#region src/functions/privacy.ts +const PRIVATE_TAG_TOKEN_RE = /)[^>]*>|<\/private\s*>/gi; +function redactPrivateBlocks(input) { + const tokens = new RegExp(PRIVATE_TAG_TOKEN_RE.source, PRIVATE_TAG_TOKEN_RE.flags); + let out = ""; + let cursor = 0; + let depth = 0; + for (let match = tokens.exec(input); match; match = tokens.exec(input)) { + const closing = match[0].startsWith(" 0) return `${out}[REDACTED]`; + return out + input.slice(cursor); +} +const SECRET_PATTERN_SOURCES = [ + /(?:api[_-]?key|secret|token|password|credential|auth)[\s]*[=:]\s*["']?[A-Za-z0-9_\-/.+]{20,}["']?/gi, + /Bearer\s+[A-Za-z0-9._\-+/=]{20,}/gi, + /(?]*:[^\s/?#@"'<>]+@/gi; +function stripPrivateData$1(input) { + return stripSecrets(redactPrivateBlocks(input)); +} +function stripSecrets(input) { + let result = input.replace(new RegExp(PRIVATE_KEY_BLOCK_RE.source, PRIVATE_KEY_BLOCK_RE.flags), "[REDACTED_SECRET]"); + result = result.replace(new RegExp(URL_CREDENTIALS_RE.source, URL_CREDENTIALS_RE.flags), "$1[REDACTED_SECRET]@"); + for (const source of SECRET_PATTERN_SOURCES) { + const pattern = new RegExp(source.source, source.flags); + result = result.replace(pattern, "[REDACTED_SECRET]"); + } + return result; +} +//#endregion //#region src/hooks/_capture-filter.ts -function truncateText(value, max) { +function cut(value, max) { if (value.length <= max) return value; const end = /[\uD800-\uDBFF]/.test(value[max - 1] ?? "") ? max - 1 : max; return value.slice(0, end); } +function truncateText(value, max) { + if (value.length <= max) return value; + return cut(stripPrivateData$1(value), max); +} //#endregion //#region src/hooks/_project.ts function resolveProject(cwd) { diff --git a/plugin/scripts/task-completed.mjs b/plugin/scripts/task-completed.mjs index d92bb7240e..090cdf2de6 100755 --- a/plugin/scripts/task-completed.mjs +++ b/plugin/scripts/task-completed.mjs @@ -3,12 +3,71 @@ import { execSync } from "node:child_process"; import { realpathSync } from "node:fs"; import { basename, dirname, resolve } from "node:path"; import { captureObservation, isDrainChild, runDrainChild, withEventId } from "./_capture.mjs"; +//#region src/functions/privacy.ts +const PRIVATE_TAG_TOKEN_RE = /)[^>]*>|<\/private\s*>/gi; +function redactPrivateBlocks(input) { + const tokens = new RegExp(PRIVATE_TAG_TOKEN_RE.source, PRIVATE_TAG_TOKEN_RE.flags); + let out = ""; + let cursor = 0; + let depth = 0; + for (let match = tokens.exec(input); match; match = tokens.exec(input)) { + const closing = match[0].startsWith(" 0) return `${out}[REDACTED]`; + return out + input.slice(cursor); +} +const SECRET_PATTERN_SOURCES = [ + /(?:api[_-]?key|secret|token|password|credential|auth)[\s]*[=:]\s*["']?[A-Za-z0-9_\-/.+]{20,}["']?/gi, + /Bearer\s+[A-Za-z0-9._\-+/=]{20,}/gi, + /(?]*:[^\s/?#@"'<>]+@/gi; +function stripPrivateData(input) { + return stripSecrets(redactPrivateBlocks(input)); +} +function stripSecrets(input) { + let result = input.replace(new RegExp(PRIVATE_KEY_BLOCK_RE.source, PRIVATE_KEY_BLOCK_RE.flags), "[REDACTED_SECRET]"); + result = result.replace(new RegExp(URL_CREDENTIALS_RE.source, URL_CREDENTIALS_RE.flags), "$1[REDACTED_SECRET]@"); + for (const source of SECRET_PATTERN_SOURCES) { + const pattern = new RegExp(source.source, source.flags); + result = result.replace(pattern, "[REDACTED_SECRET]"); + } + return result; +} +//#endregion //#region src/hooks/_capture-filter.ts -function truncateText(value, max) { +function cut(value, max) { if (value.length <= max) return value; const end = /[\uD800-\uDBFF]/.test(value[max - 1] ?? "") ? max - 1 : max; return value.slice(0, end); } +function truncateText(value, max) { + if (value.length <= max) return value; + return cut(stripPrivateData(value), max); +} //#endregion //#region src/hooks/_project.ts function resolveProject(cwd) { diff --git a/src/functions/privacy.ts b/src/functions/privacy.ts index e4f4a13d68..3eece0d48d 100644 --- a/src/functions/privacy.ts +++ b/src/functions/privacy.ts @@ -51,8 +51,11 @@ const URL_CREDENTIALS_RE = /\b([a-z][a-z0-9+.-]*:\/\/)[^\s/?#@:"'<>]*:[^\s/?#@"'<>]+@/gi; export function stripPrivateData(input: string): string { - let result = redactPrivateBlocks(input); - result = result.replace(new RegExp(PRIVATE_KEY_BLOCK_RE.source, PRIVATE_KEY_BLOCK_RE.flags), "[REDACTED_SECRET]"); + return stripSecrets(redactPrivateBlocks(input)); +} + +export function stripSecrets(input: string): string { + let result = input.replace(new RegExp(PRIVATE_KEY_BLOCK_RE.source, PRIVATE_KEY_BLOCK_RE.flags), "[REDACTED_SECRET]"); result = result.replace(new RegExp(URL_CREDENTIALS_RE.source, URL_CREDENTIALS_RE.flags), "$1[REDACTED_SECRET]@"); for (const source of SECRET_PATTERN_SOURCES) { const pattern = new RegExp(source.source, source.flags); @@ -61,14 +64,18 @@ export function stripPrivateData(input: string): string { return result; } -export function scrubRecord(value: T, options: { scrubKeys?: boolean } = {}): T { - if (typeof value === "string") return stripPrivateData(value) as T; +export function scrubRecord( + value: T, + options: { scrubKeys?: boolean; strip?: (input: string) => string } = {}, +): T { + const strip = options.strip ?? stripPrivateData; + if (typeof value === "string") return strip(value) as T; if (Array.isArray(value)) return value.map((item) => scrubRecord(item, options)) as T; if (value !== null && typeof value === "object") { const proto = Object.getPrototypeOf(value); if (proto !== Object.prototype && proto !== null) return value; const entries = Object.entries(value as Record).map(([key, item]) => [ - options.scrubKeys ? stripPrivateData(key) : key, + options.scrubKeys ? strip(key) : key, item, ] as const); const reserved = new Set(entries.map(([key]) => key)); diff --git a/src/hooks/_capture-filter.ts b/src/hooks/_capture-filter.ts index 762f52420e..799bcf0e5c 100644 --- a/src/hooks/_capture-filter.ts +++ b/src/hooks/_capture-filter.ts @@ -1,3 +1,5 @@ +import { scrubRecord, stripPrivateData, stripSecrets } from "../functions/privacy.js"; + const DEFAULT_DENY_PATTERNS = [ "memory_*", "toolsearch", @@ -60,27 +62,68 @@ export function captureOutputMax(): number { return Number.isFinite(parsed) && parsed > 0 ? parsed : 8000; } -export function truncateText(value: string, max: number): string { +function cut(value: string, max: number): string { if (value.length <= max) return value; const end = /[\uD800-\uDBFF]/.test(value[max - 1] ?? "") ? max - 1 : max; return value.slice(0, end); } +export function truncateText(value: string, max: number): string { + if (value.length <= max) return value; + return cut(stripPrivateData(value), max); +} + +const JSON_WHITESPACE_ESCAPES: Record = { n: "\n", r: "\r", t: "\t" }; + +const SECRET_KEY_WORDS = new Set(["apikey", "secret", "token", "password", "passwd", "credential", "credentials", "auth", "authorization"]); + +function isSecretKeyName(key: string): boolean { + if (!/^[\w.-]{1,64}$/.test(key)) return false; + const words = key.replace(/([a-z0-9])([A-Z])/g, "$1_$2").toLowerCase().split(/[_.-]+/); + return words.some((word, i) => SECRET_KEY_WORDS.has(word) || (word === "api" && words[i + 1] === "key")); +} + +function redactSecretKeys(value: unknown): unknown { + if (Array.isArray(value)) return value.map(redactSecretKeys); + if (value === null || typeof value !== "object") return value; + const proto = Object.getPrototypeOf(value); + if (proto !== Object.prototype && proto !== null) return value; + return Object.fromEntries( + Object.entries(value as Record).map(([key, item]) => [ + key, + typeof item === "string" && item && isSecretKeyName(key) ? "[REDACTED_SECRET]" : redactSecretKeys(item), + ]), + ); +} + +function scrubSecrets(value: unknown): unknown { + return scrubRecord(redactSecretKeys(value), { scrubKeys: true, strip: stripSecrets }); +} + +function redactSerialized(serialized: string): string { + const unescaped = serialized.replace(/\\([\\nrt])/g, (escape, c: string) => JSON_WHITESPACE_ESCAPES[c] ?? escape); + return stripPrivateData(unescaped).replace(/[\n\r\t]/g, (c) => JSON.stringify(c).slice(1, -1)); +} + +export function truncateJson(value: unknown, max: number): string { + const str = JSON.stringify(value); + if (str.length <= max) return str; + return cut(redactSerialized(JSON.stringify(scrubSecrets(value))), max); +} + +function truncateWithSuffix(value: string, max: number, suffix: string): string { + const safe = stripPrivateData(value); + if (safe.length <= max) return safe; + return max <= suffix.length ? suffix.slice(0, max) : cut(safe, max - suffix.length) + suffix; +} + export function truncateCaptureOutput(value: unknown, max: number): unknown { if (typeof value === "string" && value.length > max) { - const suffix = "\n[...truncated]"; - return max <= suffix.length - ? suffix.slice(0, max) - : truncateText(value, max - suffix.length) + suffix; + return truncateWithSuffix(value, max, "\n[...truncated]"); } if (typeof value === "object" && value !== null) { const str = JSON.stringify(value); - if (str.length > max) { - const suffix = "...[truncated]"; - return max <= suffix.length - ? suffix.slice(0, max) - : truncateText(str, max - suffix.length) + suffix; - } + if (str.length > max) return truncateWithSuffix(redactSerialized(JSON.stringify(scrubSecrets(value))), max, "...[truncated]"); return value; } return value; diff --git a/src/hooks/post-tool-failure.ts b/src/hooks/post-tool-failure.ts index 372da6b1a7..9dbe015dcb 100644 --- a/src/hooks/post-tool-failure.ts +++ b/src/hooks/post-tool-failure.ts @@ -1,5 +1,5 @@ #!/usr/bin/env node -import { shouldCaptureTool, truncateText } from "./_capture-filter.js"; +import { shouldCaptureTool, truncateJson, truncateText } from "./_capture-filter.js"; import { resolveProject, hookCwd } from "./_project.js"; import { captureObservation, isDrainChild, runDrainChild, withEventId } from "./_capture.js"; @@ -52,11 +52,11 @@ async function main() { tool_input: typeof toolInput === "string" ? truncateText(toolInput, 4000) - : truncateText(JSON.stringify(toolInput ?? ""), 4000), + : truncateJson(toolInput ?? "", 4000), error: typeof error === "string" ? truncateText(error, 4000) - : truncateText(JSON.stringify(error ?? ""), 4000), + : truncateJson(error ?? "", 4000), }, }, data, diff --git a/test/capture-filter.test.ts b/test/capture-filter.test.ts index 0d576103b7..566f96ec47 100644 --- a/test/capture-filter.test.ts +++ b/test/capture-filter.test.ts @@ -5,6 +5,7 @@ import { preCompactBudget, shouldCaptureTool, truncateCaptureOutput, + truncateJson, truncateText, } from "../src/hooks/_capture-filter.js"; @@ -116,6 +117,102 @@ describe("truncateText", () => { }); }); +describe("truncation redacts before cutting", () => { + const token = "ghp_" + "A".repeat(40); + + it("truncateText leaves text within the limit unchanged", () => { + expect(truncateText(`short ${token}`, 100)).toBe(`short ${token}`); + }); + + it("truncateText does not keep the start of a token the cut lands in", () => { + const out = truncateText("x".repeat(10) + " " + token + " tail", 20); + expect(out).not.toContain("ghp_"); + expect(out.length).toBeLessThanOrEqual(20); + }); + + it("truncateCaptureOutput redacts strings and JSON before cutting", () => { + const str = truncateCaptureOutput("x".repeat(10) + " " + token + "y".repeat(100), 40) as string; + expect(str).not.toContain("ghp_"); + expect(str.length).toBeLessThanOrEqual(40); + const json = truncateCaptureOutput({ out: "x".repeat(10) + " " + token + "y".repeat(100) }, 40) as string; + expect(json).not.toContain("ghp_"); + expect(json.length).toBeLessThanOrEqual(40); + }); +}); + +describe("truncateCaptureOutput redacts structured values before serializing", () => { + it("redacts values under secret-like keys in both helpers", () => { + const value = { password: "A".repeat(100), nested: { apiKey: "K".repeat(40) }, note: "kept" }; + for (const out of [truncateJson(value, 90), truncateCaptureOutput(value, 90) as string]) { + expect(out).not.toContain("AAAA"); + expect(out).not.toContain("KKKK"); + expect(out).toContain("[REDACTED_SECRET]"); + } + }); + + it("keeps values whose key only contains a secret word inside another word", () => { + const value = { tokenization: "words", secretary: "office", access_token: "T".repeat(40), pad: "y".repeat(200) }; + const out = truncateJson(value, 120); + expect(out).toContain("words"); + expect(out).toContain("office"); + expect(out).not.toContain("TTTT"); + }); + + it("leaves a short value with a secret-like key unchanged", () => { + const value = { password: "short" }; + expect(truncateJson(value, 100)).toBe(JSON.stringify(value)); + expect(truncateCaptureOutput(value, 100)).toBe(value); + }); + + it("removes a private block whose tag spans a line break", () => { + const out = truncateCaptureOutput({ log: "hidden-value" + "y".repeat(200) }, 60) as string; + expect(out).not.toContain("hidden"); + }); + + it("removes a private block that opens in one field and closes in the next", () => { + const value = { a: "start ", b: "hidden-value end", pad: "y".repeat(200) }; + expect(truncateCaptureOutput(value, 80) as string).not.toContain("hidden"); + expect(truncateJson(value, 80)).not.toContain("hidden"); + }); + + it("redacts a credential in a key that a line break separates from its scheme", () => { + const key = "Bearer\n" + "A".repeat(32); + const value = { [key]: "x", pad: "y".repeat(200) }; + expect(truncateJson(value, 100)).not.toContain("A".repeat(32)); + expect(truncateCaptureOutput(value, 100) as string).not.toContain("A".repeat(32)); + }); + + it("redacts a quoted credential whose quote JSON escapes", () => { + const value = { cmd: 'secret="' + "B".repeat(32) + '"', pad: "y".repeat(200) }; + expect(truncateJson(value, 100)).not.toContain("B".repeat(20)); + expect(truncateCaptureOutput(value, 100) as string).not.toContain("B".repeat(20)); + }); + + it("redacts a credential that a vertical tab separates from its scheme", () => { + const value = { header: "Bearer\u000b" + "C".repeat(32), pad: "y".repeat(200) }; + expect(truncateJson(value, 100)).not.toContain("C".repeat(20)); + }); + + it("keeps every field when two keys redact to the same text", () => { + const value = { ["token=" + "A".repeat(24)]: "first", ["token=" + "B".repeat(24)]: "second", pad: "y".repeat(200) }; + const out = truncateJson(value, 150); + expect(out).toContain("first"); + expect(out).toContain("second"); + }); + + it("keeps escaped backslashes and line breaks in the cut JSON", () => { + const out = truncateJson({ a: "one\ntwo\\nthree", pad: "y".repeat(200) }, 60); + expect(out.startsWith('{"a":"one\\ntwo\\\\nthree"')).toBe(true); + }); + + it("truncateJson leaves short JSON unchanged and redacts long JSON before cutting", () => { + expect(truncateJson({ a: 1 }, 100)).toBe('{"a":1}'); + const out = truncateJson({ log: "hidden-value" + "y".repeat(200) }, 60); + expect(out).not.toContain("hidden"); + expect(out.length).toBeLessThanOrEqual(60); + }); +}); + describe("truncateCaptureOutput", () => { it("leaves short strings alone", () => { expect(truncateCaptureOutput("hello", 20)).toBe("hello"); diff --git a/test/hook-truncation.test.ts b/test/hook-truncation.test.ts index b14f3ce11e..8543c78059 100644 --- a/test/hook-truncation.test.ts +++ b/test/hook-truncation.test.ts @@ -159,4 +159,15 @@ describe("hook truncation at a surrogate pair", () => { expect(whole).toContain(SMILE); expect(count(whole, "x")).toBe(kept - 2); }); + + it.each(sites.map((site) => [site.name, site] as const))("%s redacts a secret before cutting through it", async (_name, site) => { + const kept = count(await sent(site, "x".repeat(LONG)), "x"); + const token = await sent(site, "x".repeat(kept - 12) + " ghp_" + "A".repeat(40) + " " + "x".repeat(LONG)); + expect(token).not.toContain("ghp_"); + expect(token).not.toContain("AAAA"); + const bearer = await sent(site, "x".repeat(kept - 12) + " Bearer " + "B".repeat(40) + " " + "x".repeat(LONG)); + expect(bearer).not.toContain("BBBB"); + const block = await sent(site, "x".repeat(kept - 12) + " hidden-value " + "x".repeat(LONG)); + expect(block).not.toContain("hidden"); + }); });