#162: Normalize passive discovery artifacts

src/server/recon/discovery-artifact-normalizer.ts

@@ -0,0 +1,345 @@
+export type DiscoveryArtifactSource =
+	| "upload"
+	| "terminal-note"
+	| "lab-command"
+	| "http-probe"
+	| "reference"
+	| (string & {});
+
+export type DiscoveryArtifactInput = {
+	artifactId: string;
+	content: string;
+	source: DiscoveryArtifactSource;
+};
+
+export type AuthSurfaceCategory =
+	| "login"
+	| "logout"
+	| "registration"
+	| "password-recovery"
+	| "oauth"
+	| "sso"
+	| "token"
+	| "session"
+	| "api-key"
+	| "admin";
+
+export type DiscoveryBlockerReason =
+	| "bot-block-detected"
+	| "captcha-detected"
+	| "waf-denied"
+	| "rate-limited"
+	| "auth-required"
+	| "approval-required"
+	| "target-authorization-required"
+	| "workspace-locked"
+	| "network-profile-blocked"
+	| "tool-unavailable";
+
+export type NormalizedDiscoveryUrl = {
+	url: string;
+	host: string;
+	path: string;
+	sourceArtifactIds: string[];
+};
+
+export type AuthSurfaceCandidate = {
+	url: string;
+	path: string;
+	categories: AuthSurfaceCategory[];
+	confidence: "high" | "medium";
+	sourceArtifactIds: string[];
+};
+
+export type DiscoveryBlockerSignal = {
+	reason: DiscoveryBlockerReason;
+	sourceArtifactIds: string[];
+	evidence: string;
+};
+
+export type DiscoveryResponseFamily = {
+	family: "1xx" | "2xx" | "3xx" | "4xx" | "5xx";
+	statuses: number[];
+	sourceArtifactIds: string[];
+};
+
+export type NormalizedDiscovery = {
+	rawArtifactIds: string[];
+	urls: NormalizedDiscoveryUrl[];
+	authCandidates: AuthSurfaceCandidate[];
+	responseFamilies: DiscoveryResponseFamily[];
+	blockerSignals: DiscoveryBlockerSignal[];
+	observations: {
+		session: string[];
+		token: string[];
+	};
+};
+
+const URL_PATTERN = /https?:\/\/[^\s<>"'`]+/giu;
+const STATUS_PATTERNS = [
+	/\bHTTP\/\d(?:\.\d)?\s+(\d{3})\b/giu,
+	/\bHTTP\s+(\d{3})\b/giu,
+	/(?:^|\s)\[(\d{3})\](?=\s|$)/gmu,
+];
+
+const AUTH_ROUTE_RULES: ReadonlyArray<{
+	category: AuthSurfaceCategory;
+	pattern: RegExp;
+	confidence: "high" | "medium";
+}> = [
+	{
+		category: "login",
+		pattern: /(?:^|[/_-])(login|log-in|signin|sign-in)(?:$|[/_-])/iu,
+		confidence: "high",
+	},
+	{
+		category: "logout",
+		pattern: /(?:^|[/_-])(logout|log-out|signout|sign-out)(?:$|[/_-])/iu,
+		confidence: "high",
+	},
+	{
+		category: "registration",
+		pattern: /(?:^|[/_-])(register|registration|signup|sign-up)(?:$|[/_-])/iu,
+		confidence: "high",
+	},
+	{
+		category: "password-recovery",
+		pattern:
+			/(?:forgot|reset|recover)[/_-]?(?:password|account)|password[/_-]?(?:forgot|reset|recover)/iu,
+		confidence: "high",
+	},
+	{
+		category: "oauth",
+		pattern: /(?:^|[/_-])(oauth2?|authorize|callback)(?:$|[/_-])/iu,
+		confidence: "high",
+	},
+	{
+		category: "sso",
+		pattern: /(?:^|[/_-])(sso|saml|oidc)(?:$|[/_-])/iu,
+		confidence: "high",
+	},
+	{
+		category: "token",
+		pattern: /(?:^|[/_-])(token|jwt|refresh)(?:$|[/_-])/iu,
+		confidence: "medium",
+	},
+	{
+		category: "session",
+		pattern: /(?:^|[/_-])(session|sessions)(?:$|[/_-])/iu,
+		confidence: "medium",
+	},
+	{
+		category: "api-key",
+		pattern: /(?:api[/_-]?keys?|keys?[/_-]?api)(?:$|[/_-])/iu,
+		confidence: "medium",
+	},
+	{
+		category: "admin",
+		pattern: /(?:^|[/_-])(admin|administrator)(?:$|[/_-])/iu,
+		confidence: "medium",
+	},
+];
+
+const BLOCKER_RULES: ReadonlyArray<{
+	reason: DiscoveryBlockerReason;
+	pattern: RegExp;
+}> = [
+	{
+		reason: "captcha-detected",
+		pattern: /\b(?:captcha|recaptcha|hcaptcha)\b/iu,
+	},
+	{
+		reason: "bot-block-detected",
+		pattern:
+			/\b(?:bot detected|automated (?:traffic|request)|verify you are human)\b/iu,
+	},
+	{
+		reason: "waf-denied",
+		pattern:
+			/\b(?:web application firewall|waf|access denied|request blocked)\b/iu,
+	},
+	{
+		reason: "rate-limited",
+		pattern: /\b(?:rate limit(?:ed|ing)?|too many requests|http\s*429)\b/iu,
+	},
+	{
+		reason: "auth-required",
+		pattern:
+			/\b(?:authentication required|login required|unauthorized|http\s*401)\b/iu,
+	},
+	{
+		reason: "approval-required",
+		pattern: /\b(?:approval required|missing approval)\b/iu,
+	},
+	{
+		reason: "target-authorization-required",
+		pattern: /\b(?:target authorization required|missing authorization)\b/iu,
+	},
+	{ reason: "workspace-locked", pattern: /\bworkspace (?:is )?locked\b/iu },
+	{
+		reason: "network-profile-blocked",
+		pattern:
+			/\b(?:network profile blocked|wrong network profile|egress denied)\b/iu,
+	},
+	{
+		reason: "tool-unavailable",
+		pattern: /\b(?:tool unavailable|command not found|not installed)\b/iu,
+	},
+];
+
+const SESSION_OBSERVATION =
+	/\b(?:set-cookie|cookie|session(?:id)?|same-site|samesite|httponly)\b/iu;
+const TOKEN_OBSERVATION =
+	/\b(?:bearer|jwt|access[_ -]?token|refresh[_ -]?token|id[_ -]?token|api[_ -]?key)\b/iu;
+
+export function normalizeDiscoveryArtifacts(
+	artifacts: readonly DiscoveryArtifactInput[],
+): NormalizedDiscovery {
+	const rawArtifactIds = uniqueSorted(
+		artifacts.map((artifact) => artifact.artifactId),
+	);
+	const urlSources = new Map<string, Set<string>>();
+	const statusSources = new Map<number, Set<string>>();
+	const blockerSources = new Map<
+		DiscoveryBlockerReason,
+		{ artifactIds: Set<string>; evidence: string }
+	>();
+	const sessionObservations = new Set<string>();
+	const tokenObservations = new Set<string>();
+
+	for (const artifact of artifacts) {
+		for (const rawUrl of artifact.content.match(URL_PATTERN) ?? []) {
+			const url = normalizeUrl(rawUrl);
+			if (!url) continue;
+			addSource(urlSources, url, artifact.artifactId);
+		}
+
+		for (const pattern of STATUS_PATTERNS) {
+			pattern.lastIndex = 0;
+			for (const match of artifact.content.matchAll(pattern)) {
+				const status = Number(match[1]);
+				if (status >= 100 && status <= 599)
+					addSource(statusSources, status, artifact.artifactId);
+			}
+		}
+
+		for (const line of artifact.content.split(/\r?\n/u)) {
+			const excerpt = line.trim();
+			if (!excerpt) continue;
+
+			if (SESSION_OBSERVATION.test(excerpt)) sessionObservations.add(excerpt);
+			if (TOKEN_OBSERVATION.test(excerpt)) tokenObservations.add(excerpt);
+
+			for (const rule of BLOCKER_RULES) {
+				if (!rule.pattern.test(excerpt)) continue;
+				const current = blockerSources.get(rule.reason);
+				if (current) {
+					current.artifactIds.add(artifact.artifactId);
+				} else {
+					blockerSources.set(rule.reason, {
+						artifactIds: new Set([artifact.artifactId]),
+						evidence: excerpt.slice(0, 500),
+					});
+				}
+			}
+		}
+	}
+
+	const urls = [...urlSources.entries()]
+		.map(([url, artifactIds]) => {
+			const parsed = new URL(url);
+			return {
+				url,
+				host: parsed.host,
+				path: parsed.pathname || "/",
+				sourceArtifactIds: uniqueSorted(artifactIds),
+			} satisfies NormalizedDiscoveryUrl;
+		})
+		.sort((left, right) => left.url.localeCompare(right.url));
+
+	const authCandidates = urls.flatMap((entry) => {
+		const searchable = `${entry.path}${new URL(entry.url).search}`;
+		const matches = AUTH_ROUTE_RULES.filter((rule) =>
+			rule.pattern.test(searchable),
+		);
+		if (matches.length === 0) return [];
+		return [
+			{
+				url: entry.url,
+				path: entry.path,
+				categories: matches.map((match) => match.category),
+				confidence: matches.some((match) => match.confidence === "high")
+					? "high"
+					: "medium",
+				sourceArtifactIds: entry.sourceArtifactIds,
+			} satisfies AuthSurfaceCandidate,
+		];
+	});
+
+	const families = new Map<
+		DiscoveryResponseFamily["family"],
+		{ statuses: Set<number>; artifactIds: Set<string> }
+	>();
+	for (const [status, artifactIds] of statusSources) {
+		const family =
+			`${Math.floor(status / 100)}xx` as DiscoveryResponseFamily["family"];
+		const current = families.get(family) ?? {
+			statuses: new Set<number>(),
+			artifactIds: new Set<string>(),
+		};
+		current.statuses.add(status);
+		for (const artifactId of artifactIds) current.artifactIds.add(artifactId);
+		families.set(family, current);
+	}
+
+	return {
+		rawArtifactIds,
+		urls,
+		authCandidates,
+		responseFamilies: [...families.entries()]
+			.map(([family, value]) => ({
+				family,
+				statuses: [...value.statuses].sort((left, right) => left - right),
+				sourceArtifactIds: uniqueSorted(value.artifactIds),
+			}))
+			.sort((left, right) => left.family.localeCompare(right.family)),
+		blockerSignals: [...blockerSources.entries()]
+			.map(([reason, value]) => ({
+				reason,
+				sourceArtifactIds: uniqueSorted(value.artifactIds),
+				evidence: value.evidence,
+			}))
+			.sort((left, right) => left.reason.localeCompare(right.reason)),
+		observations: {
+			session: [...sessionObservations].sort(),
+			token: [...tokenObservations].sort(),
+		},
+	};
+}
+
+function normalizeUrl(raw: string): string | undefined {
+	const cleaned = raw.replace(/[),.;\]}]+$/u, "");
+	try {
+		const parsed = new URL(cleaned);
+		if (parsed.protocol !== "http:" && parsed.protocol !== "https:")
+			return undefined;
+		parsed.hash = "";
+		return parsed.toString();
+	} catch {
+		return undefined;
+	}
+}
+
+function addSource<T>(
+	map: Map<T, Set<string>>,
+	key: T,
+	artifactId: string,
+): void {
+	const sources = map.get(key) ?? new Set<string>();
+	sources.add(artifactId);
+	map.set(key, sources);
+}
+
+function uniqueSorted(values: Iterable<string>): string[] {
+	return [...new Set(values)].sort();
+}

src/server/recon/index.ts

@@ -0,0 +1,12 @@
+export {
+	type AuthSurfaceCandidate,
+	type AuthSurfaceCategory,
+	type DiscoveryArtifactInput,
+	type DiscoveryArtifactSource,
+	type DiscoveryBlockerReason,
+	type DiscoveryBlockerSignal,
+	type DiscoveryResponseFamily,
+	type NormalizedDiscovery,
+	type NormalizedDiscoveryUrl,
+	normalizeDiscoveryArtifacts,
+} from "./discovery-artifact-normalizer";