diff --git a/src/server/evidence/index.ts b/src/server/evidence/index.ts index 73260bca0..fe82e70d7 100644 --- a/src/server/evidence/index.ts +++ b/src/server/evidence/index.ts @@ -56,6 +56,7 @@ export { isEvidenceRedactionEnabled, redactEvidenceJsonValue, redactEvidenceSecrets, + redactEvidenceUrlSecrets, runWithEvidenceRedaction, } from "./ingestion"; export { diff --git a/src/server/evidence/ingestion.ts b/src/server/evidence/ingestion.ts index 112ca46bf..1ebb29bf8 100644 --- a/src/server/evidence/ingestion.ts +++ b/src/server/evidence/ingestion.ts @@ -8,6 +8,9 @@ import { ensureThreadRagIndex, THREAD_RAG_INDEX_NAME } from "../../mastra/tools/ import { type Queryable, withDatabase } from "../db/client"; import { sanitizePostgresText } from "../db/postgres-unicode"; import { replaceEvidenceLexicalChunks } from "./hybrid-search"; +import { redactEvidenceUrlSecrets } from "./redaction"; + +export { redactEvidenceUrlSecrets } from "./redaction"; export const EVIDENCE_SOURCES = [ "upload", @@ -409,11 +412,12 @@ export function runWithEvidenceRedaction(enabled: boolean, callback: () => T) } export function redactEvidenceSecrets(text: string) { + const urlRedactedText = redactEvidenceUrlSecrets(text); if (!isEvidenceRedactionEnabled()) { - return text; + return urlRedactedText; } - return text + return urlRedactedText .replace( /-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z ]*PRIVATE KEY-----/g, "[redacted-private-key]", diff --git a/src/server/evidence/redaction.ts b/src/server/evidence/redaction.ts new file mode 100644 index 000000000..eac38c2da --- /dev/null +++ b/src/server/evidence/redaction.ts @@ -0,0 +1,141 @@ +const HTTP_URL_PATTERN = /https?:\/\/[^\s<>"'`]+/giu; + +const SENSITIVE_URL_PARAMETER_NAMES = new Set([ + "accesstoken", + "apikey", + "authorization", + "authtoken", + "clientassertion", + "clientsecret", + "code", + "codeverifier", + "credential", + "credentials", + "idtoken", + "jwt", + "password", + "passwd", + "privatekey", + "pwd", + "refreshtoken", + "secret", + "session", + "sessionid", + "sig", + "signature", + "token", +]); + +/** + * Removes credentials carried by HTTP(S) URLs even when general evidence + * redaction is disabled. URLs cross report, Artifact, RAG, and response + * boundaries, so their credential fields are never an operator-visible raw + * evidence exception. + */ +export function redactEvidenceUrlSecrets(text: string): string { + return text.replace(HTTP_URL_PATTERN, (rawUrl) => redactHttpUrl(rawUrl)); +} + +function redactHttpUrl(rawUrl: string): string { + let parsed: URL; + try { + parsed = new URL(rawUrl); + } catch { + return rawUrl; + } + if (parsed.protocol !== "http:" && parsed.protocol !== "https:") { + return rawUrl; + } + + let redacted = rawUrl; + if (parsed.username || parsed.password) { + redacted = removeUrlUserinfo(redacted); + } + return redactSensitiveUrlParameters(redacted); +} + +function removeUrlUserinfo(url: string): string { + const schemeEnd = url.indexOf("://") + 3; + const authorityEnd = firstIndexOf(url, ["/", "?", "#"], schemeEnd); + const at = url.lastIndexOf("@", authorityEnd); + if (schemeEnd < 3 || at < schemeEnd) return url; + return `${url.slice(0, schemeEnd)}${url.slice(at + 1)}`; +} + +function redactSensitiveUrlParameters(url: string): string { + const fragmentStart = url.indexOf("#"); + const possibleQueryStart = url.indexOf("?"); + const queryStart = + possibleQueryStart >= 0 && + (fragmentStart < 0 || possibleQueryStart < fragmentStart) + ? possibleQueryStart + : -1; + const pathEnd = + queryStart >= 0 + ? queryStart + : fragmentStart >= 0 + ? fragmentStart + : url.length; + const queryEnd = fragmentStart >= 0 ? fragmentStart : url.length; + + const path = redactUrlParameterSegment( + url.slice(0, pathEnd), + /([;])([^=?&#;]+)=([^?&#;]*)/gu, + ); + const query = + queryStart >= 0 + ? redactUrlParameterSegment( + url.slice(queryStart, queryEnd), + /([?&;])([^=&#;]+)=([^&#;]*)/gu, + ) + : ""; + const fragment = + fragmentStart >= 0 + ? redactUrlParameterSegment( + url.slice(fragmentStart), + /([#&;])([^=&#;]+)=([^&#;]*)/gu, + ) + : ""; + return `${path}${query}${fragment}`; +} + +function redactUrlParameterSegment(segment: string, pattern: RegExp): string { + return segment.replace( + pattern, + (match, separator: string, encodedName: string) => { + if (!isSensitiveUrlParameterName(encodedName)) return match; + return `${separator}${encodedName}=[redacted]`; + }, + ); +} + +function isSensitiveUrlParameterName(encodedName: string): boolean { + let decodedName = encodedName; + try { + decodedName = decodeURIComponent(encodedName.replaceAll("+", " ")); + } catch { + // An invalid escape is still compared in its original form. + } + const normalized = decodedName.toLowerCase().replace(/[^a-z0-9]/gu, ""); + return ( + SENSITIVE_URL_PARAMETER_NAMES.has(normalized) || + normalized.endsWith("password") || + normalized.endsWith("secret") || + normalized.endsWith("token") || + normalized.endsWith("credential") || + normalized.endsWith("signature") || + normalized.endsWith("sessionid") || + normalized.endsWith("sig") + ); +} + +function firstIndexOf( + value: string, + needles: readonly string[], + from: number, +): number { + const indexes = needles + .map((needle) => value.indexOf(needle, from)) + .filter((index) => index >= 0); + return indexes.length > 0 ? Math.min(...indexes) : value.length; +} diff --git a/src/server/recon/discovery-artifact-normalizer.ts b/src/server/recon/discovery-artifact-normalizer.ts index 7fd67c42d..9b4d1ccbc 100644 --- a/src/server/recon/discovery-artifact-normalizer.ts +++ b/src/server/recon/discovery-artifact-normalizer.ts @@ -1,3 +1,5 @@ +import { redactEvidenceUrlSecrets } from "../evidence/redaction"; + export type DiscoveryArtifactSource = | "upload" | "terminal-note" @@ -224,7 +226,7 @@ export function normalizeDiscoveryArtifacts( } for (const line of artifact.content.split(/\r?\n/u)) { - const excerpt = line.trim(); + const excerpt = redactEvidenceUrlSecrets(line.trim()); if (!excerpt) continue; if (SESSION_OBSERVATION.test(excerpt)) sessionObservations.add(excerpt); @@ -320,9 +322,11 @@ export function normalizeDiscoveryArtifacts( function normalizeUrl(raw: string): string | undefined { const cleaned = raw.replace(/[),.;\]}]+$/u, ""); try { - const parsed = new URL(cleaned); + const parsed = new URL(redactEvidenceUrlSecrets(cleaned)); if (parsed.protocol !== "http:" && parsed.protocol !== "https:") return undefined; + parsed.username = ""; + parsed.password = ""; parsed.hash = ""; return parsed.toString(); } catch { diff --git a/tests/integration/discovery-artifact-normalizer.test.ts b/tests/integration/discovery-artifact-normalizer.test.ts index 11cd5e9f6..0546429ce 100644 --- a/tests/integration/discovery-artifact-normalizer.test.ts +++ b/tests/integration/discovery-artifact-normalizer.test.ts @@ -13,7 +13,7 @@ describe("discovery artifact normalization", () => { source: "lab-command", content: [ "https://app.example.test/login", - "https://app.example.test/oauth/callback?code=example#fragment", + "https://user%40tenant:password@app.example.test/oauth/callback?code=oauth-secret&view=compact#access_token=fragment-secret", "https://app.example.test/admin/", "https://app.example.test/login", ].join("\n"), @@ -37,8 +37,11 @@ describe("discovery artifact normalization", () => { "https://app.example.test/admin/", "https://app.example.test/api/v2/session", "https://app.example.test/login", - "https://app.example.test/oauth/callback?code=example", + "https://app.example.test/oauth/callback?code=[redacted]&view=compact", ]); + expect(JSON.stringify(result)).not.toMatch( + /user%40tenant|password|oauth-secret|fragment-secret/u, + ); expect( result.urls.find((entry) => entry.path === "/login")?.sourceArtifactIds, ).toEqual(["artifact-gau"]); @@ -78,7 +81,10 @@ describe("discovery artifact normalization", () => { "https://app.example.test/api/v2/session", "set-cookie: sessionid=[REDACTED]; HttpOnly; SameSite=Lax", ]); - expect(result.observations.token).toEqual(["www-authenticate: Bearer"]); + expect(result.observations.token).toEqual([ + "https://app.example.test/oauth/callback?code=[redacted]&view=compact#access_token=[redacted]", + "www-authenticate: Bearer", + ]); }); it("persists a redaction-ready, RAG-indexed summary linked to its raw evidence", async () => { diff --git a/tests/integration/evidence-ingestion.test.ts b/tests/integration/evidence-ingestion.test.ts index ae80497fa..f2d17902a 100644 --- a/tests/integration/evidence-ingestion.test.ts +++ b/tests/integration/evidence-ingestion.test.ts @@ -10,6 +10,7 @@ import { EvidenceIndexingTemporarilyUnavailableError, ingestEvidenceBestEffort, redactEvidenceSecrets, + runWithEvidenceRedaction, setEvidenceIngestor, } from "../../src/server/evidence"; @@ -313,6 +314,26 @@ describe("shared evidence ingestion", () => { expect(index.mock.calls[0]?.[0].chunks[0]?.metadata.redacted).toBe(true); }); + it("always removes HTTP URL credentials when general evidence redaction is disabled", () => { + const raw = [ + "GET https://user%40tenant:p%40ss@app.example.test/login?password=query-secret&view=compact#access_token=fragment-secret", + "GET https://app.example.test/login;JSESSIONID=java-session?X-Amz-Signature=aws-signature&X-Goog-Signature=google-signature&ASP.NET_SessionId=asp-session&code_verifier=oauth-verifier&client_assertion=oauth-assertion&view=compact", + "GET https://app.example.test/callback?token=query-secret?tail-secret&view=compact", + ].join("\n"); + const redacted = runWithEvidenceRedaction(false, () => redactEvidenceSecrets(raw)); + + expect(redacted).toBe( + [ + "GET https://app.example.test/login?password=[redacted]&view=compact#access_token=[redacted]", + "GET https://app.example.test/login;JSESSIONID=[redacted]?X-Amz-Signature=[redacted]&X-Goog-Signature=[redacted]&ASP.NET_SessionId=[redacted]&code_verifier=[redacted]&client_assertion=[redacted]&view=compact", + "GET https://app.example.test/callback?token=[redacted]&view=compact", + ].join("\n"), + ); + expect(redacted).not.toMatch( + /user%40tenant|p%40ss|query-secret|tail-secret|fragment-secret|java-session|aws-signature|google-signature|asp-session|oauth-verifier|oauth-assertion/u, + ); + }); + it("chunks awkward log output by line windows with searchable line metadata", async () => { const index = vi.fn(async (input: EvidenceIndexInput) => input.chunks.length); const ingestor = createEvidenceIngestor({ diff --git a/tests/integration/stored-passive-auth-surface-api.test.ts b/tests/integration/stored-passive-auth-surface-api.test.ts index 5c3ec79aa..74d77467c 100644 --- a/tests/integration/stored-passive-auth-surface-api.test.ts +++ b/tests/integration/stored-passive-auth-surface-api.test.ts @@ -10,6 +10,7 @@ import { withDatabase } from "../../src/server/db/client"; import { createArtifactService, createEvidenceIngestor, + runWithEvidenceRedaction, setArtifactService, setEvidenceIngestor, } from "../../src/server/evidence"; @@ -112,18 +113,62 @@ describe("stored passive auth-surface API", () => { ); const artifactService = createArtifactService({ storage: null }); - setArtifactService(artifactService); - const source = await artifactService.createArtifact({ - projectId: project.id, - threadId: thread.id, - targetId: "target-app", - name: "passive-urls.txt", - kind: "log", - contentType: "text/plain", - content: - "https://app.example.test/login\nhttps://app.example.test/api/session", - source: "upload", - indexForRag: false, + const source = { id: "artifact-legacy-passive-urls" }; + const legacySourceContent = [ + "https://url-user:p%40ssword@app.example.test/login/?password=query-secret?tail-secret&view=compact#access_token=fragment-secret", + "https://app.example.test/login;JSESSIONID=java-session?X-Amz-Signature=aws-signature&X-Goog-Signature=google-signature&ASP.NET_SessionId=asp-session&code_verifier=oauth-verifier&client_assertion=oauth-assertion&view=compact", + "https://app.example.test/api/session", + ].join("\n"); + await withDatabase((db) => + db.query( + `INSERT INTO artifacts ( + id, project_id, thread_id, kind, name, content_type, inline_content, metadata + ) + VALUES ($1, $2, $3, 'log', 'passive-urls.txt', 'text/plain', $4, $5::jsonb)`, + [ + source.id, + project.id, + thread.id, + legacySourceContent, + JSON.stringify({ + source: "upload", + targetId: "target-app", + targetIds: ["target-app"], + targetScope: "targets", + }), + ], + ), + ); + const readArtifactText = artifactService.readArtifactText; + if (!readArtifactText) + throw new Error("Artifact text reads are unavailable."); + setArtifactService({ + ...artifactService, + readArtifactText: async (input) => { + if (input.artifactId !== source.id) return readArtifactText(input); + const result = await withDatabase((db) => + db.query<{ inline_content: string }>( + "SELECT inline_content FROM artifacts WHERE id = $1 AND project_id = $2", + [input.artifactId, input.projectId], + ), + ); + const row = result.rows[0]; + if (!row) throw new Error("Legacy source artifact was not found."); + return { + artifactId: source.id, + projectId: project.id, + threadId: thread.id, + name: "passive-urls.txt", + contentType: "text/plain", + sizeBytes: new TextEncoder().encode(row.inline_content).byteLength, + sha256: null, + source: "upload" as const, + targetIds: ["target-app"], + targetScope: "targets" as const, + text: row.inline_content, + truncated: false, + }; + }, }); const foreignSource = await artifactService.createArtifact({ projectId: otherProject.id, @@ -285,16 +330,24 @@ describe("stored passive auth-surface API", () => { unsubscribeProjectEvents(); setArtifactService(artifactService); - const response = await invoke(project.id, { - targetId: "target-app", - taskId: "task-passive-map", - artifacts: [ - { artifactId: source.id }, - { artifactId: projectReference.id }, - ], - }); + const response = await runWithEvidenceRedaction(false, () => + invoke(project.id, { + targetId: "target-app", + taskId: "task-passive-map", + artifacts: [ + { artifactId: source.id }, + { artifactId: projectReference.id }, + ], + }), + ); expect(response.status).toBe(200); const body = (await response.json()) as Record; + const serializedBody = JSON.stringify(body); + expect(serializedBody).not.toMatch( + /url-user|p%40ssword|query-secret|tail-secret|fragment-secret|java-session|aws-signature|google-signature|asp-session|oauth-verifier|oauth-assertion/u, + ); + expect(serializedBody).toContain("app.example.test/login/"); + expect(serializedBody).toContain("view=compact"); expect(body).toMatchObject({ authorizationId: authorization.id, summary: { @@ -337,6 +390,26 @@ describe("stored passive auth-surface API", () => { }), }), ); + const indexedPayload = JSON.stringify(indexed.mock.calls); + expect(indexedPayload).not.toMatch( + /url-user|p%40ssword|query-secret|tail-secret|fragment-secret|java-session|aws-signature|google-signature|asp-session|oauth-verifier|oauth-assertion/u, + ); + expect(indexedPayload).toContain("app.example.test/login/"); + expect(indexedPayload).toContain("view=compact"); + + const reportArtifactId = (body.artifact as { id?: string }).id; + expect(reportArtifactId).toBeTruthy(); + if (!reportArtifactId) + throw new Error("Passive report artifact id was missing."); + const storedReport = await artifactService.readArtifactText?.({ + projectId: project.id, + artifactId: reportArtifactId, + }); + expect(storedReport?.text).not.toMatch( + /url-user|p%40ssword|query-secret|tail-secret|fragment-secret|java-session|aws-signature|google-signature|asp-session|oauth-verifier|oauth-assertion/u, + ); + expect(storedReport?.text).toContain("app.example.test/login/"); + expect(storedReport?.text).toContain("view=compact"); const persistedAttribution = await withDatabase((db) => db.query<{ thread_id: string | null; task_id: string | null }>( "SELECT thread_id, task_id FROM artifacts WHERE id = $1",