#165: Redact credentials in passive discovery URLs

src/server/evidence/index.ts

@@ -56,6 +56,7 @@ export {
   isEvidenceRedactionEnabled,
   redactEvidenceJsonValue,
   redactEvidenceSecrets,
+  redactEvidenceUrlSecrets,
   runWithEvidenceRedaction,
 } from "./ingestion";
 export {

src/server/evidence/ingestion.ts

@@ -8,6 +8,9 @@ import { ensureThreadRagIndex, THREAD_RAG_INDEX_NAME } from "../../mastra/tools/
 import { type Queryable, withDatabase } from "../db/client";
 import { sanitizePostgresText } from "../db/postgres-unicode";
 import { replaceEvidenceLexicalChunks } from "./hybrid-search";
+import { redactEvidenceUrlSecrets } from "./redaction";
+
+export { redactEvidenceUrlSecrets } from "./redaction";
 
 export const EVIDENCE_SOURCES = [
   "upload",
@@ -409,11 +412,12 @@ export function runWithEvidenceRedaction<T>(enabled: boolean, callback: () => T)
 }
 
 export function redactEvidenceSecrets(text: string) {
+  const urlRedactedText = redactEvidenceUrlSecrets(text);
   if (!isEvidenceRedactionEnabled()) {
-    return text;
+    return urlRedactedText;
   }
 
-  return text
+  return urlRedactedText
     .replace(
       /-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z ]*PRIVATE KEY-----/g,
       "[redacted-private-key]",

src/server/evidence/redaction.ts

@@ -0,0 +1,141 @@
+const HTTP_URL_PATTERN = /https?:\/\/[^\s<>"'`]+/giu;
+
+const SENSITIVE_URL_PARAMETER_NAMES = new Set([
+	"accesstoken",
+	"apikey",
+	"authorization",
+	"authtoken",
+	"clientassertion",
+	"clientsecret",
+	"code",
+	"codeverifier",
+	"credential",
+	"credentials",
+	"idtoken",
+	"jwt",
+	"password",
+	"passwd",
+	"privatekey",
+	"pwd",
+	"refreshtoken",
+	"secret",
+	"session",
+	"sessionid",
+	"sig",
+	"signature",
+	"token",
+]);
+
+/**
+ * Removes credentials carried by HTTP(S) URLs even when general evidence
+ * redaction is disabled. URLs cross report, Artifact, RAG, and response
+ * boundaries, so their credential fields are never an operator-visible raw
+ * evidence exception.
+ */
+export function redactEvidenceUrlSecrets(text: string): string {
+	return text.replace(HTTP_URL_PATTERN, (rawUrl) => redactHttpUrl(rawUrl));
+}
+
+function redactHttpUrl(rawUrl: string): string {
+	let parsed: URL;
+	try {
+		parsed = new URL(rawUrl);
+	} catch {
+		return rawUrl;
+	}
+	if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
+		return rawUrl;
+	}
+
+	let redacted = rawUrl;
+	if (parsed.username || parsed.password) {
+		redacted = removeUrlUserinfo(redacted);
+	}
+	return redactSensitiveUrlParameters(redacted);
+}
+
+function removeUrlUserinfo(url: string): string {
+	const schemeEnd = url.indexOf("://") + 3;
+	const authorityEnd = firstIndexOf(url, ["/", "?", "#"], schemeEnd);
+	const at = url.lastIndexOf("@", authorityEnd);
+	if (schemeEnd < 3 || at < schemeEnd) return url;
+	return `${url.slice(0, schemeEnd)}${url.slice(at + 1)}`;
+}
+
+function redactSensitiveUrlParameters(url: string): string {
+	const fragmentStart = url.indexOf("#");
+	const possibleQueryStart = url.indexOf("?");
+	const queryStart =
+		possibleQueryStart >= 0 &&
+		(fragmentStart < 0 || possibleQueryStart < fragmentStart)
+			? possibleQueryStart
+			: -1;
+	const pathEnd =
+		queryStart >= 0
+			? queryStart
+			: fragmentStart >= 0
+				? fragmentStart
+				: url.length;
+	const queryEnd = fragmentStart >= 0 ? fragmentStart : url.length;
+
+	const path = redactUrlParameterSegment(
+		url.slice(0, pathEnd),
+		/([;])([^=?&#;]+)=([^?&#;]*)/gu,
+	);
+	const query =
+		queryStart >= 0
+			? redactUrlParameterSegment(
+					url.slice(queryStart, queryEnd),
+					/([?&;])([^=&#;]+)=([^&#;]*)/gu,
+				)
+			: "";
+	const fragment =
+		fragmentStart >= 0
+			? redactUrlParameterSegment(
+					url.slice(fragmentStart),
+					/([#&;])([^=&#;]+)=([^&#;]*)/gu,
+				)
+			: "";
+	return `${path}${query}${fragment}`;
+}
+
+function redactUrlParameterSegment(segment: string, pattern: RegExp): string {
+	return segment.replace(
+		pattern,
+		(match, separator: string, encodedName: string) => {
+			if (!isSensitiveUrlParameterName(encodedName)) return match;
+			return `${separator}${encodedName}=[redacted]`;
+		},
+	);
+}
+
+function isSensitiveUrlParameterName(encodedName: string): boolean {
+	let decodedName = encodedName;
+	try {
+		decodedName = decodeURIComponent(encodedName.replaceAll("+", " "));
+	} catch {
+		// An invalid escape is still compared in its original form.
+	}
+	const normalized = decodedName.toLowerCase().replace(/[^a-z0-9]/gu, "");
+	return (
+		SENSITIVE_URL_PARAMETER_NAMES.has(normalized) ||
+		normalized.endsWith("password") ||
+		normalized.endsWith("secret") ||
+		normalized.endsWith("token") ||
+		normalized.endsWith("credential") ||
+		normalized.endsWith("signature") ||
+		normalized.endsWith("sessionid") ||
+		normalized.endsWith("sig")
+	);
+}
+
+function firstIndexOf(
+	value: string,
+	needles: readonly string[],
+	from: number,
+): number {
+	const indexes = needles
+		.map((needle) => value.indexOf(needle, from))
+		.filter((index) => index >= 0);
+	return indexes.length > 0 ? Math.min(...indexes) : value.length;
+}

src/server/recon/discovery-artifact-normalizer.ts

@@ -1,3 +1,5 @@
+import { redactEvidenceUrlSecrets } from "../evidence/redaction";
+
 export type DiscoveryArtifactSource =
 	| "upload"
 	| "terminal-note"
@@ -224,7 +226,7 @@ export function normalizeDiscoveryArtifacts(
 		}
 
 		for (const line of artifact.content.split(/\r?\n/u)) {
-			const excerpt = line.trim();
+			const excerpt = redactEvidenceUrlSecrets(line.trim());
 			if (!excerpt) continue;
 
 			if (SESSION_OBSERVATION.test(excerpt)) sessionObservations.add(excerpt);
@@ -320,9 +322,11 @@ export function normalizeDiscoveryArtifacts(
 function normalizeUrl(raw: string): string | undefined {
 	const cleaned = raw.replace(/[),.;\]}]+$/u, "");
 	try {
-		const parsed = new URL(cleaned);
+		const parsed = new URL(redactEvidenceUrlSecrets(cleaned));
 		if (parsed.protocol !== "http:" && parsed.protocol !== "https:")
 			return undefined;
+		parsed.username = "";
+		parsed.password = "";
 		parsed.hash = "";
 		return parsed.toString();
 	} catch {

tests/integration/discovery-artifact-normalizer.test.ts

@@ -13,7 +13,7 @@ describe("discovery artifact normalization", () => {
 				source: "lab-command",
 				content: [
 					"https://app.example.test/login",
-					"https://app.example.test/oauth/callback?code=example#fragment",
+					"https://user%40tenant:password@app.example.test/oauth/callback?code=oauth-secret&view=compact#access_token=fragment-secret",
 					"https://app.example.test/admin/",
 					"https://app.example.test/login",
 				].join("\n"),
@@ -37,8 +37,11 @@ describe("discovery artifact normalization", () => {
 			"https://app.example.test/admin/",
 			"https://app.example.test/api/v2/session",
 			"https://app.example.test/login",
-			"https://app.example.test/oauth/callback?code=example",
+			"https://app.example.test/oauth/callback?code=[redacted]&view=compact",
 		]);
+		expect(JSON.stringify(result)).not.toMatch(
+			/user%40tenant|password|oauth-secret|fragment-secret/u,
+		);
 		expect(
 			result.urls.find((entry) => entry.path === "/login")?.sourceArtifactIds,
 		).toEqual(["artifact-gau"]);
@@ -78,7 +81,10 @@ describe("discovery artifact normalization", () => {
 			"https://app.example.test/api/v2/session",
 			"set-cookie: sessionid=[REDACTED]; HttpOnly; SameSite=Lax",
 		]);
-		expect(result.observations.token).toEqual(["www-authenticate: Bearer"]);
+		expect(result.observations.token).toEqual([
+			"https://app.example.test/oauth/callback?code=[redacted]&view=compact#access_token=[redacted]",
+			"www-authenticate: Bearer",
+		]);
 	});
 
 	it("persists a redaction-ready, RAG-indexed summary linked to its raw evidence", async () => {

tests/integration/evidence-ingestion.test.ts

@@ -10,6 +10,7 @@ import {
   EvidenceIndexingTemporarilyUnavailableError,
   ingestEvidenceBestEffort,
   redactEvidenceSecrets,
+  runWithEvidenceRedaction,
   setEvidenceIngestor,
 } from "../../src/server/evidence";
 
@@ -313,6 +314,26 @@ describe("shared evidence ingestion", () => {
     expect(index.mock.calls[0]?.[0].chunks[0]?.metadata.redacted).toBe(true);
   });
 
+  it("always removes HTTP URL credentials when general evidence redaction is disabled", () => {
+    const raw = [
+      "GET https://user%40tenant:p%40ss@app.example.test/login?password=query-secret&view=compact#access_token=fragment-secret",
+      "GET https://app.example.test/login;JSESSIONID=java-session?X-Amz-Signature=aws-signature&X-Goog-Signature=google-signature&ASP.NET_SessionId=asp-session&code_verifier=oauth-verifier&client_assertion=oauth-assertion&view=compact",
+      "GET https://app.example.test/callback?token=query-secret?tail-secret&view=compact",
+    ].join("\n");
+    const redacted = runWithEvidenceRedaction(false, () => redactEvidenceSecrets(raw));
+
+    expect(redacted).toBe(
+      [
+        "GET https://app.example.test/login?password=[redacted]&view=compact#access_token=[redacted]",
+        "GET https://app.example.test/login;JSESSIONID=[redacted]?X-Amz-Signature=[redacted]&X-Goog-Signature=[redacted]&ASP.NET_SessionId=[redacted]&code_verifier=[redacted]&client_assertion=[redacted]&view=compact",
+        "GET https://app.example.test/callback?token=[redacted]&view=compact",
+      ].join("\n"),
+    );
+    expect(redacted).not.toMatch(
+      /user%40tenant|p%40ss|query-secret|tail-secret|fragment-secret|java-session|aws-signature|google-signature|asp-session|oauth-verifier|oauth-assertion/u,
+    );
+  });
+
   it("chunks awkward log output by line windows with searchable line metadata", async () => {
     const index = vi.fn(async (input: EvidenceIndexInput) => input.chunks.length);
     const ingestor = createEvidenceIngestor({