diff --git a/src/server/recon/discovery-artifact-normalizer.ts b/src/server/recon/discovery-artifact-normalizer.ts new file mode 100644 index 000000000..7fd67c42d --- /dev/null +++ b/src/server/recon/discovery-artifact-normalizer.ts @@ -0,0 +1,345 @@ +export type DiscoveryArtifactSource = + | "upload" + | "terminal-note" + | "lab-command" + | "http-probe" + | "reference" + | (string & {}); + +export type DiscoveryArtifactInput = { + artifactId: string; + content: string; + source: DiscoveryArtifactSource; +}; + +export type AuthSurfaceCategory = + | "login" + | "logout" + | "registration" + | "password-recovery" + | "oauth" + | "sso" + | "token" + | "session" + | "api-key" + | "admin"; + +export type DiscoveryBlockerReason = + | "bot-block-detected" + | "captcha-detected" + | "waf-denied" + | "rate-limited" + | "auth-required" + | "approval-required" + | "target-authorization-required" + | "workspace-locked" + | "network-profile-blocked" + | "tool-unavailable"; + +export type NormalizedDiscoveryUrl = { + url: string; + host: string; + path: string; + sourceArtifactIds: string[]; +}; + +export type AuthSurfaceCandidate = { + url: string; + path: string; + categories: AuthSurfaceCategory[]; + confidence: "high" | "medium"; + sourceArtifactIds: string[]; +}; + +export type DiscoveryBlockerSignal = { + reason: DiscoveryBlockerReason; + sourceArtifactIds: string[]; + evidence: string; +}; + +export type DiscoveryResponseFamily = { + family: "1xx" | "2xx" | "3xx" | "4xx" | "5xx"; + statuses: number[]; + sourceArtifactIds: string[]; +}; + +export type NormalizedDiscovery = { + rawArtifactIds: string[]; + urls: NormalizedDiscoveryUrl[]; + authCandidates: AuthSurfaceCandidate[]; + responseFamilies: DiscoveryResponseFamily[]; + blockerSignals: DiscoveryBlockerSignal[]; + observations: { + session: string[]; + token: string[]; + }; +}; + +const URL_PATTERN = /https?:\/\/[^\s<>"'`]+/giu; +const STATUS_PATTERNS = [ + /\bHTTP\/\d(?:\.\d)?\s+(\d{3})\b/giu, + /\bHTTP\s+(\d{3})\b/giu, + /(?:^|\s)\[(\d{3})\](?=\s|$)/gmu, +]; + +const AUTH_ROUTE_RULES: ReadonlyArray<{ + category: AuthSurfaceCategory; + pattern: RegExp; + confidence: "high" | "medium"; +}> = [ + { + category: "login", + pattern: /(?:^|[/_-])(login|log-in|signin|sign-in)(?:$|[/_-])/iu, + confidence: "high", + }, + { + category: "logout", + pattern: /(?:^|[/_-])(logout|log-out|signout|sign-out)(?:$|[/_-])/iu, + confidence: "high", + }, + { + category: "registration", + pattern: /(?:^|[/_-])(register|registration|signup|sign-up)(?:$|[/_-])/iu, + confidence: "high", + }, + { + category: "password-recovery", + pattern: + /(?:forgot|reset|recover)[/_-]?(?:password|account)|password[/_-]?(?:forgot|reset|recover)/iu, + confidence: "high", + }, + { + category: "oauth", + pattern: /(?:^|[/_-])(oauth2?|authorize|callback)(?:$|[/_-])/iu, + confidence: "high", + }, + { + category: "sso", + pattern: /(?:^|[/_-])(sso|saml|oidc)(?:$|[/_-])/iu, + confidence: "high", + }, + { + category: "token", + pattern: /(?:^|[/_-])(token|jwt|refresh)(?:$|[/_-])/iu, + confidence: "medium", + }, + { + category: "session", + pattern: /(?:^|[/_-])(session|sessions)(?:$|[/_-])/iu, + confidence: "medium", + }, + { + category: "api-key", + pattern: /(?:api[/_-]?keys?|keys?[/_-]?api)(?:$|[/_-])/iu, + confidence: "medium", + }, + { + category: "admin", + pattern: /(?:^|[/_-])(admin|administrator)(?:$|[/_-])/iu, + confidence: "medium", + }, +]; + +const BLOCKER_RULES: ReadonlyArray<{ + reason: DiscoveryBlockerReason; + pattern: RegExp; +}> = [ + { + reason: "captcha-detected", + pattern: /\b(?:captcha|recaptcha|hcaptcha)\b/iu, + }, + { + reason: "bot-block-detected", + pattern: + /\b(?:bot detected|automated (?:traffic|request)|verify you are human)\b/iu, + }, + { + reason: "waf-denied", + pattern: + /\b(?:web application firewall|waf|access denied|request blocked)\b/iu, + }, + { + reason: "rate-limited", + pattern: /\b(?:rate limit(?:ed|ing)?|too many requests|http\s*429)\b/iu, + }, + { + reason: "auth-required", + pattern: + /\b(?:authentication required|login required|unauthorized|http\s*401)\b/iu, + }, + { + reason: "approval-required", + pattern: /\b(?:approval required|missing approval)\b/iu, + }, + { + reason: "target-authorization-required", + pattern: /\b(?:target authorization required|missing authorization)\b/iu, + }, + { reason: "workspace-locked", pattern: /\bworkspace (?:is )?locked\b/iu }, + { + reason: "network-profile-blocked", + pattern: + /\b(?:network profile blocked|wrong network profile|egress denied)\b/iu, + }, + { + reason: "tool-unavailable", + pattern: /\b(?:tool unavailable|command not found|not installed)\b/iu, + }, +]; + +const SESSION_OBSERVATION = + /\b(?:set-cookie|cookie|session(?:id)?|same-site|samesite|httponly)\b/iu; +const TOKEN_OBSERVATION = + /\b(?:bearer|jwt|access[_ -]?token|refresh[_ -]?token|id[_ -]?token|api[_ -]?key)\b/iu; + +export function normalizeDiscoveryArtifacts( + artifacts: readonly DiscoveryArtifactInput[], +): NormalizedDiscovery { + const rawArtifactIds = uniqueSorted( + artifacts.map((artifact) => artifact.artifactId), + ); + const urlSources = new Map>(); + const statusSources = new Map>(); + const blockerSources = new Map< + DiscoveryBlockerReason, + { artifactIds: Set; evidence: string } + >(); + const sessionObservations = new Set(); + const tokenObservations = new Set(); + + for (const artifact of artifacts) { + for (const rawUrl of artifact.content.match(URL_PATTERN) ?? []) { + const url = normalizeUrl(rawUrl); + if (!url) continue; + addSource(urlSources, url, artifact.artifactId); + } + + for (const pattern of STATUS_PATTERNS) { + pattern.lastIndex = 0; + for (const match of artifact.content.matchAll(pattern)) { + const status = Number(match[1]); + if (status >= 100 && status <= 599) + addSource(statusSources, status, artifact.artifactId); + } + } + + for (const line of artifact.content.split(/\r?\n/u)) { + const excerpt = line.trim(); + if (!excerpt) continue; + + if (SESSION_OBSERVATION.test(excerpt)) sessionObservations.add(excerpt); + if (TOKEN_OBSERVATION.test(excerpt)) tokenObservations.add(excerpt); + + for (const rule of BLOCKER_RULES) { + if (!rule.pattern.test(excerpt)) continue; + const current = blockerSources.get(rule.reason); + if (current) { + current.artifactIds.add(artifact.artifactId); + } else { + blockerSources.set(rule.reason, { + artifactIds: new Set([artifact.artifactId]), + evidence: excerpt.slice(0, 500), + }); + } + } + } + } + + const urls = [...urlSources.entries()] + .map(([url, artifactIds]) => { + const parsed = new URL(url); + return { + url, + host: parsed.host, + path: parsed.pathname || "/", + sourceArtifactIds: uniqueSorted(artifactIds), + } satisfies NormalizedDiscoveryUrl; + }) + .sort((left, right) => left.url.localeCompare(right.url)); + + const authCandidates = urls.flatMap((entry) => { + const searchable = `${entry.path}${new URL(entry.url).search}`; + const matches = AUTH_ROUTE_RULES.filter((rule) => + rule.pattern.test(searchable), + ); + if (matches.length === 0) return []; + return [ + { + url: entry.url, + path: entry.path, + categories: matches.map((match) => match.category), + confidence: matches.some((match) => match.confidence === "high") + ? "high" + : "medium", + sourceArtifactIds: entry.sourceArtifactIds, + } satisfies AuthSurfaceCandidate, + ]; + }); + + const families = new Map< + DiscoveryResponseFamily["family"], + { statuses: Set; artifactIds: Set } + >(); + for (const [status, artifactIds] of statusSources) { + const family = + `${Math.floor(status / 100)}xx` as DiscoveryResponseFamily["family"]; + const current = families.get(family) ?? { + statuses: new Set(), + artifactIds: new Set(), + }; + current.statuses.add(status); + for (const artifactId of artifactIds) current.artifactIds.add(artifactId); + families.set(family, current); + } + + return { + rawArtifactIds, + urls, + authCandidates, + responseFamilies: [...families.entries()] + .map(([family, value]) => ({ + family, + statuses: [...value.statuses].sort((left, right) => left - right), + sourceArtifactIds: uniqueSorted(value.artifactIds), + })) + .sort((left, right) => left.family.localeCompare(right.family)), + blockerSignals: [...blockerSources.entries()] + .map(([reason, value]) => ({ + reason, + sourceArtifactIds: uniqueSorted(value.artifactIds), + evidence: value.evidence, + })) + .sort((left, right) => left.reason.localeCompare(right.reason)), + observations: { + session: [...sessionObservations].sort(), + token: [...tokenObservations].sort(), + }, + }; +} + +function normalizeUrl(raw: string): string | undefined { + const cleaned = raw.replace(/[),.;\]}]+$/u, ""); + try { + const parsed = new URL(cleaned); + if (parsed.protocol !== "http:" && parsed.protocol !== "https:") + return undefined; + parsed.hash = ""; + return parsed.toString(); + } catch { + return undefined; + } +} + +function addSource( + map: Map>, + key: T, + artifactId: string, +): void { + const sources = map.get(key) ?? new Set(); + sources.add(artifactId); + map.set(key, sources); +} + +function uniqueSorted(values: Iterable): string[] { + return [...new Set(values)].sort(); +} diff --git a/src/server/recon/index.ts b/src/server/recon/index.ts new file mode 100644 index 000000000..1f6e2a9ec --- /dev/null +++ b/src/server/recon/index.ts @@ -0,0 +1,12 @@ +export { + type AuthSurfaceCandidate, + type AuthSurfaceCategory, + type DiscoveryArtifactInput, + type DiscoveryArtifactSource, + type DiscoveryBlockerReason, + type DiscoveryBlockerSignal, + type DiscoveryResponseFamily, + type NormalizedDiscovery, + type NormalizedDiscoveryUrl, + normalizeDiscoveryArtifacts, +} from "./discovery-artifact-normalizer"; diff --git a/tests/integration/discovery-artifact-normalizer.test.ts b/tests/integration/discovery-artifact-normalizer.test.ts new file mode 100644 index 000000000..e355b2f73 --- /dev/null +++ b/tests/integration/discovery-artifact-normalizer.test.ts @@ -0,0 +1,80 @@ +import { describe, expect, it } from "vitest"; + +import { normalizeDiscoveryArtifacts } from "../../src/server/recon"; + +describe("discovery artifact normalization", () => { + it("turns mixed passive evidence into deduplicated, attributable auth-surface signals", () => { + const result = normalizeDiscoveryArtifacts([ + { + artifactId: "artifact-gau", + source: "lab-command", + content: [ + "https://app.example.test/login", + "https://app.example.test/oauth/callback?code=example#fragment", + "https://app.example.test/admin/", + "https://app.example.test/login", + ].join("\n"), + }, + { + artifactId: "artifact-probe", + source: "http-probe", + content: [ + "HTTP/2 401", + "set-cookie: sessionid=[REDACTED]; HttpOnly; SameSite=Lax", + "www-authenticate: Bearer", + "Authentication required", + "HTTP 429 Too Many Requests - rate limited", + "https://app.example.test/api/v2/session", + ].join("\n"), + }, + ]); + + expect(result.rawArtifactIds).toEqual(["artifact-gau", "artifact-probe"]); + expect(result.urls.map((entry) => entry.url)).toEqual([ + "https://app.example.test/admin/", + "https://app.example.test/api/v2/session", + "https://app.example.test/login", + "https://app.example.test/oauth/callback?code=example", + ]); + expect( + result.urls.find((entry) => entry.path === "/login")?.sourceArtifactIds, + ).toEqual(["artifact-gau"]); + expect( + result.authCandidates.map((candidate) => ({ + path: candidate.path, + categories: candidate.categories, + confidence: candidate.confidence, + })), + ).toEqual([ + { path: "/admin/", categories: ["admin"], confidence: "medium" }, + { + path: "/api/v2/session", + categories: ["session"], + confidence: "medium", + }, + { path: "/login", categories: ["login"], confidence: "high" }, + { path: "/oauth/callback", categories: ["oauth"], confidence: "high" }, + ]); + expect(result.responseFamilies).toEqual([ + { + family: "4xx", + statuses: [401, 429], + sourceArtifactIds: ["artifact-probe"], + }, + ]); + expect(result.blockerSignals.map((signal) => signal.reason)).toEqual([ + "auth-required", + "rate-limited", + ]); + expect( + result.blockerSignals.every((signal) => + signal.sourceArtifactIds.includes("artifact-probe"), + ), + ).toBe(true); + expect(result.observations.session).toEqual([ + "https://app.example.test/api/v2/session", + "set-cookie: sessionid=[REDACTED]; HttpOnly; SameSite=Lax", + ]); + expect(result.observations.token).toEqual(["www-authenticate: Bearer"]); + }); +});