From 329ac6745853d977bda47c6b89e11b7198b934f0 Mon Sep 17 00:00:00 2001 From: Dan Levy Date: Wed, 26 Aug 2026 21:31:05 -0400 Subject: [PATCH 1/2] Normalize passive discovery evidence --- .../recon/discovery-artifact-normalizer.ts | 247 ++++++++++++++++++ src/server/recon/index.ts | 12 + .../discovery-artifact-normalizer.test.ts | 66 +++++ 3 files changed, 325 insertions(+) create mode 100644 src/server/recon/discovery-artifact-normalizer.ts create mode 100644 src/server/recon/index.ts create mode 100644 tests/integration/discovery-artifact-normalizer.test.ts diff --git a/src/server/recon/discovery-artifact-normalizer.ts b/src/server/recon/discovery-artifact-normalizer.ts new file mode 100644 index 000000000..ca167ee28 --- /dev/null +++ b/src/server/recon/discovery-artifact-normalizer.ts @@ -0,0 +1,247 @@ +export type DiscoveryArtifactSource = + | "upload" + | "terminal-note" + | "lab-command" + | "http-probe" + | "reference" + | (string & {}); + +export type DiscoveryArtifactInput = { + artifactId: string; + content: string; + source: DiscoveryArtifactSource; +}; + +export type AuthSurfaceCategory = + | "login" + | "logout" + | "registration" + | "password-recovery" + | "oauth" + | "sso" + | "token" + | "session" + | "api-key" + | "admin"; + +export type DiscoveryBlockerReason = + | "bot-block-detected" + | "captcha-detected" + | "waf-denied" + | "rate-limited" + | "auth-required" + | "approval-required" + | "target-authorization-required" + | "workspace-locked" + | "network-profile-blocked" + | "tool-unavailable"; + +export type NormalizedDiscoveryUrl = { + url: string; + host: string; + path: string; + sourceArtifactIds: string[]; +}; + +export type AuthSurfaceCandidate = { + url: string; + path: string; + categories: AuthSurfaceCategory[]; + confidence: "high" | "medium"; + sourceArtifactIds: string[]; +}; + +export type DiscoveryBlockerSignal = { + reason: DiscoveryBlockerReason; + sourceArtifactIds: string[]; + evidence: string; +}; + +export type DiscoveryResponseFamily = { + family: "1xx" | "2xx" | "3xx" | "4xx" | "5xx"; + statuses: number[]; + sourceArtifactIds: string[]; +}; + +export type NormalizedDiscovery = { + rawArtifactIds: string[]; + urls: NormalizedDiscoveryUrl[]; + authCandidates: AuthSurfaceCandidate[]; + responseFamilies: DiscoveryResponseFamily[]; + blockerSignals: DiscoveryBlockerSignal[]; + observations: { + session: string[]; + token: string[]; + }; +}; + +const URL_PATTERN = /https?:\/\/[^\s<>"'`]+/giu; +const STATUS_PATTERNS = [ + /\bHTTP\/\d(?:\.\d)?\s+(\d{3})\b/giu, + /\bHTTP\s+(\d{3})\b/giu, + /(?:^|\s)\[(\d{3})\](?=\s|$)/gmu, +]; + +const AUTH_ROUTE_RULES: ReadonlyArray<{ + category: AuthSurfaceCategory; + pattern: RegExp; + confidence: "high" | "medium"; +}> = [ + { category: "login", pattern: /(?:^|[\/_-])(login|log-in|signin|sign-in)(?:$|[\/_-])/iu, confidence: "high" }, + { category: "logout", pattern: /(?:^|[\/_-])(logout|log-out|signout|sign-out)(?:$|[\/_-])/iu, confidence: "high" }, + { category: "registration", pattern: /(?:^|[\/_-])(register|registration|signup|sign-up)(?:$|[\/_-])/iu, confidence: "high" }, + { category: "password-recovery", pattern: /(?:forgot|reset|recover)[\/_-]?(?:password|account)|password[\/_-]?(?:forgot|reset|recover)/iu, confidence: "high" }, + { category: "oauth", pattern: /(?:^|[\/_-])(oauth2?|authorize|callback)(?:$|[\/_-])/iu, confidence: "high" }, + { category: "sso", pattern: /(?:^|[\/_-])(sso|saml|oidc)(?:$|[\/_-])/iu, confidence: "high" }, + { category: "token", pattern: /(?:^|[\/_-])(token|jwt|refresh)(?:$|[\/_-])/iu, confidence: "medium" }, + { category: "session", pattern: /(?:^|[\/_-])(session|sessions)(?:$|[\/_-])/iu, confidence: "medium" }, + { category: "api-key", pattern: /(?:api[\/_-]?keys?|keys?[\/_-]?api)(?:$|[\/_-])/iu, confidence: "medium" }, + { category: "admin", pattern: /(?:^|[\/_-])(admin|administrator)(?:$|[\/_-])/iu, confidence: "medium" }, +]; + +const BLOCKER_RULES: ReadonlyArray<{ + reason: DiscoveryBlockerReason; + pattern: RegExp; +}> = [ + { reason: "captcha-detected", pattern: /\b(?:captcha|recaptcha|hcaptcha)\b/iu }, + { reason: "bot-block-detected", pattern: /\b(?:bot detected|automated (?:traffic|request)|verify you are human)\b/iu }, + { reason: "waf-denied", pattern: /\b(?:web application firewall|waf|access denied|request blocked)\b/iu }, + { reason: "rate-limited", pattern: /\b(?:rate limit(?:ed|ing)?|too many requests|http\s*429)\b/iu }, + { reason: "auth-required", pattern: /\b(?:authentication required|login required|unauthorized|http\s*401)\b/iu }, + { reason: "approval-required", pattern: /\b(?:approval required|missing approval)\b/iu }, + { reason: "target-authorization-required", pattern: /\b(?:target authorization required|missing authorization)\b/iu }, + { reason: "workspace-locked", pattern: /\bworkspace (?:is )?locked\b/iu }, + { reason: "network-profile-blocked", pattern: /\b(?:network profile blocked|wrong network profile|egress denied)\b/iu }, + { reason: "tool-unavailable", pattern: /\b(?:tool unavailable|command not found|not installed)\b/iu }, +]; + +const SESSION_OBSERVATION = /\b(?:set-cookie|cookie|session(?:id)?|same-site|samesite|httponly)\b/iu; +const TOKEN_OBSERVATION = /\b(?:bearer|jwt|access[_ -]?token|refresh[_ -]?token|id[_ -]?token|api[_ -]?key)\b/iu; + +export function normalizeDiscoveryArtifacts( + artifacts: readonly DiscoveryArtifactInput[], +): NormalizedDiscovery { + const rawArtifactIds = uniqueSorted(artifacts.map((artifact) => artifact.artifactId)); + const urlSources = new Map>(); + const statusSources = new Map>(); + const blockerSources = new Map; evidence: string }>(); + const sessionObservations = new Set(); + const tokenObservations = new Set(); + + for (const artifact of artifacts) { + for (const rawUrl of artifact.content.match(URL_PATTERN) ?? []) { + const url = normalizeUrl(rawUrl); + if (!url) continue; + addSource(urlSources, url, artifact.artifactId); + } + + for (const pattern of STATUS_PATTERNS) { + pattern.lastIndex = 0; + for (const match of artifact.content.matchAll(pattern)) { + const status = Number(match[1]); + if (status >= 100 && status <= 599) addSource(statusSources, status, artifact.artifactId); + } + } + + for (const line of artifact.content.split(/\r?\n/u)) { + const excerpt = line.trim(); + if (!excerpt) continue; + + if (SESSION_OBSERVATION.test(excerpt)) sessionObservations.add(excerpt); + if (TOKEN_OBSERVATION.test(excerpt)) tokenObservations.add(excerpt); + + for (const rule of BLOCKER_RULES) { + if (!rule.pattern.test(excerpt)) continue; + const current = blockerSources.get(rule.reason); + if (current) { + current.artifactIds.add(artifact.artifactId); + } else { + blockerSources.set(rule.reason, { + artifactIds: new Set([artifact.artifactId]), + evidence: excerpt.slice(0, 500), + }); + } + } + } + } + + const urls = [...urlSources.entries()] + .map(([url, artifactIds]) => { + const parsed = new URL(url); + return { + url, + host: parsed.host, + path: parsed.pathname || "/", + sourceArtifactIds: uniqueSorted(artifactIds), + } satisfies NormalizedDiscoveryUrl; + }) + .sort((left, right) => left.url.localeCompare(right.url)); + + const authCandidates = urls.flatMap((entry) => { + const searchable = `${entry.path}${new URL(entry.url).search}`; + const matches = AUTH_ROUTE_RULES.filter((rule) => rule.pattern.test(searchable)); + if (matches.length === 0) return []; + return [{ + url: entry.url, + path: entry.path, + categories: matches.map((match) => match.category), + confidence: matches.some((match) => match.confidence === "high") ? "high" : "medium", + sourceArtifactIds: entry.sourceArtifactIds, + } satisfies AuthSurfaceCandidate]; + }); + + const families = new Map; artifactIds: Set }>(); + for (const [status, artifactIds] of statusSources) { + const family = `${Math.floor(status / 100)}xx` as DiscoveryResponseFamily["family"]; + const current = families.get(family) ?? { statuses: new Set(), artifactIds: new Set() }; + current.statuses.add(status); + for (const artifactId of artifactIds) current.artifactIds.add(artifactId); + families.set(family, current); + } + + return { + rawArtifactIds, + urls, + authCandidates, + responseFamilies: [...families.entries()] + .map(([family, value]) => ({ + family, + statuses: [...value.statuses].sort((left, right) => left - right), + sourceArtifactIds: uniqueSorted(value.artifactIds), + })) + .sort((left, right) => left.family.localeCompare(right.family)), + blockerSignals: [...blockerSources.entries()] + .map(([reason, value]) => ({ + reason, + sourceArtifactIds: uniqueSorted(value.artifactIds), + evidence: value.evidence, + })) + .sort((left, right) => left.reason.localeCompare(right.reason)), + observations: { + session: [...sessionObservations].sort(), + token: [...tokenObservations].sort(), + }, + }; +} + +function normalizeUrl(raw: string): string | undefined { + const cleaned = raw.replace(/[),.;\]}]+$/u, ""); + try { + const parsed = new URL(cleaned); + if (parsed.protocol !== "http:" && parsed.protocol !== "https:") return undefined; + parsed.hash = ""; + return parsed.toString(); + } catch { + return undefined; + } +} + +function addSource(map: Map>, key: T, artifactId: string): void { + const sources = map.get(key) ?? new Set(); + sources.add(artifactId); + map.set(key, sources); +} + +function uniqueSorted(values: Iterable): string[] { + return [...new Set(values)].sort(); +} diff --git a/src/server/recon/index.ts b/src/server/recon/index.ts new file mode 100644 index 000000000..7232d89a6 --- /dev/null +++ b/src/server/recon/index.ts @@ -0,0 +1,12 @@ +export { + normalizeDiscoveryArtifacts, + type AuthSurfaceCandidate, + type AuthSurfaceCategory, + type DiscoveryArtifactInput, + type DiscoveryArtifactSource, + type DiscoveryBlockerReason, + type DiscoveryBlockerSignal, + type DiscoveryResponseFamily, + type NormalizedDiscovery, + type NormalizedDiscoveryUrl, +} from "./discovery-artifact-normalizer"; diff --git a/tests/integration/discovery-artifact-normalizer.test.ts b/tests/integration/discovery-artifact-normalizer.test.ts new file mode 100644 index 000000000..5046adbe7 --- /dev/null +++ b/tests/integration/discovery-artifact-normalizer.test.ts @@ -0,0 +1,66 @@ +import { describe, expect, it } from "vitest"; + +import { normalizeDiscoveryArtifacts } from "../../src/server/recon"; + +describe("discovery artifact normalization", () => { + it("turns mixed passive evidence into deduplicated, attributable auth-surface signals", () => { + const result = normalizeDiscoveryArtifacts([ + { + artifactId: "artifact-gau", + source: "lab-command", + content: [ + "https://app.example.test/login", + "https://app.example.test/oauth/callback?code=example#fragment", + "https://app.example.test/admin/", + "https://app.example.test/login", + ].join("\n"), + }, + { + artifactId: "artifact-probe", + source: "http-probe", + content: [ + "HTTP/2 401", + "set-cookie: sessionid=[REDACTED]; HttpOnly; SameSite=Lax", + "www-authenticate: Bearer", + "Authentication required", + "HTTP 429 Too Many Requests - rate limited", + "https://app.example.test/api/v2/session", + ].join("\n"), + }, + ]); + + expect(result.rawArtifactIds).toEqual(["artifact-gau", "artifact-probe"]); + expect(result.urls.map((entry) => entry.url)).toEqual([ + "https://app.example.test/admin/", + "https://app.example.test/api/v2/session", + "https://app.example.test/login", + "https://app.example.test/oauth/callback?code=example", + ]); + expect(result.urls.find((entry) => entry.path === "/login")?.sourceArtifactIds).toEqual([ + "artifact-gau", + ]); + expect(result.authCandidates.map((candidate) => ({ + path: candidate.path, + categories: candidate.categories, + confidence: candidate.confidence, + }))).toEqual([ + { path: "/admin/", categories: ["admin"], confidence: "medium" }, + { path: "/api/v2/session", categories: ["session"], confidence: "medium" }, + { path: "/login", categories: ["login"], confidence: "high" }, + { path: "/oauth/callback", categories: ["oauth"], confidence: "high" }, + ]); + expect(result.responseFamilies).toEqual([ + { family: "4xx", statuses: [401, 429], sourceArtifactIds: ["artifact-probe"] }, + ]); + expect(result.blockerSignals.map((signal) => signal.reason)).toEqual([ + "auth-required", + "rate-limited", + ]); + expect(result.blockerSignals.every((signal) => signal.sourceArtifactIds.includes("artifact-probe"))).toBe(true); + expect(result.observations.session).toEqual([ + "https://app.example.test/api/v2/session", + "set-cookie: sessionid=[REDACTED]; HttpOnly; SameSite=Lax", + ]); + expect(result.observations.token).toEqual(["www-authenticate: Bearer"]); + }); +}); From c8620fc05f6337b7d12d6944b80f8a421432f40d Mon Sep 17 00:00:00 2001 From: Dan Levy Date: Wed, 26 Aug 2026 21:31:18 -0400 Subject: [PATCH 2/2] Polish passive discovery normalizer --- .../recon/discovery-artifact-normalizer.ts | 496 +++++++++++------- src/server/recon/index.ts | 20 +- .../discovery-artifact-normalizer.test.ts | 132 ++--- 3 files changed, 380 insertions(+), 268 deletions(-) diff --git a/src/server/recon/discovery-artifact-normalizer.ts b/src/server/recon/discovery-artifact-normalizer.ts index ca167ee28..7fd67c42d 100644 --- a/src/server/recon/discovery-artifact-normalizer.ts +++ b/src/server/recon/discovery-artifact-normalizer.ts @@ -1,247 +1,345 @@ export type DiscoveryArtifactSource = - | "upload" - | "terminal-note" - | "lab-command" - | "http-probe" - | "reference" - | (string & {}); + | "upload" + | "terminal-note" + | "lab-command" + | "http-probe" + | "reference" + | (string & {}); export type DiscoveryArtifactInput = { - artifactId: string; - content: string; - source: DiscoveryArtifactSource; + artifactId: string; + content: string; + source: DiscoveryArtifactSource; }; export type AuthSurfaceCategory = - | "login" - | "logout" - | "registration" - | "password-recovery" - | "oauth" - | "sso" - | "token" - | "session" - | "api-key" - | "admin"; + | "login" + | "logout" + | "registration" + | "password-recovery" + | "oauth" + | "sso" + | "token" + | "session" + | "api-key" + | "admin"; export type DiscoveryBlockerReason = - | "bot-block-detected" - | "captcha-detected" - | "waf-denied" - | "rate-limited" - | "auth-required" - | "approval-required" - | "target-authorization-required" - | "workspace-locked" - | "network-profile-blocked" - | "tool-unavailable"; + | "bot-block-detected" + | "captcha-detected" + | "waf-denied" + | "rate-limited" + | "auth-required" + | "approval-required" + | "target-authorization-required" + | "workspace-locked" + | "network-profile-blocked" + | "tool-unavailable"; export type NormalizedDiscoveryUrl = { - url: string; - host: string; - path: string; - sourceArtifactIds: string[]; + url: string; + host: string; + path: string; + sourceArtifactIds: string[]; }; export type AuthSurfaceCandidate = { - url: string; - path: string; - categories: AuthSurfaceCategory[]; - confidence: "high" | "medium"; - sourceArtifactIds: string[]; + url: string; + path: string; + categories: AuthSurfaceCategory[]; + confidence: "high" | "medium"; + sourceArtifactIds: string[]; }; export type DiscoveryBlockerSignal = { - reason: DiscoveryBlockerReason; - sourceArtifactIds: string[]; - evidence: string; + reason: DiscoveryBlockerReason; + sourceArtifactIds: string[]; + evidence: string; }; export type DiscoveryResponseFamily = { - family: "1xx" | "2xx" | "3xx" | "4xx" | "5xx"; - statuses: number[]; - sourceArtifactIds: string[]; + family: "1xx" | "2xx" | "3xx" | "4xx" | "5xx"; + statuses: number[]; + sourceArtifactIds: string[]; }; export type NormalizedDiscovery = { - rawArtifactIds: string[]; - urls: NormalizedDiscoveryUrl[]; - authCandidates: AuthSurfaceCandidate[]; - responseFamilies: DiscoveryResponseFamily[]; - blockerSignals: DiscoveryBlockerSignal[]; - observations: { - session: string[]; - token: string[]; - }; + rawArtifactIds: string[]; + urls: NormalizedDiscoveryUrl[]; + authCandidates: AuthSurfaceCandidate[]; + responseFamilies: DiscoveryResponseFamily[]; + blockerSignals: DiscoveryBlockerSignal[]; + observations: { + session: string[]; + token: string[]; + }; }; const URL_PATTERN = /https?:\/\/[^\s<>"'`]+/giu; const STATUS_PATTERNS = [ - /\bHTTP\/\d(?:\.\d)?\s+(\d{3})\b/giu, - /\bHTTP\s+(\d{3})\b/giu, - /(?:^|\s)\[(\d{3})\](?=\s|$)/gmu, + /\bHTTP\/\d(?:\.\d)?\s+(\d{3})\b/giu, + /\bHTTP\s+(\d{3})\b/giu, + /(?:^|\s)\[(\d{3})\](?=\s|$)/gmu, ]; const AUTH_ROUTE_RULES: ReadonlyArray<{ - category: AuthSurfaceCategory; - pattern: RegExp; - confidence: "high" | "medium"; + category: AuthSurfaceCategory; + pattern: RegExp; + confidence: "high" | "medium"; }> = [ - { category: "login", pattern: /(?:^|[\/_-])(login|log-in|signin|sign-in)(?:$|[\/_-])/iu, confidence: "high" }, - { category: "logout", pattern: /(?:^|[\/_-])(logout|log-out|signout|sign-out)(?:$|[\/_-])/iu, confidence: "high" }, - { category: "registration", pattern: /(?:^|[\/_-])(register|registration|signup|sign-up)(?:$|[\/_-])/iu, confidence: "high" }, - { category: "password-recovery", pattern: /(?:forgot|reset|recover)[\/_-]?(?:password|account)|password[\/_-]?(?:forgot|reset|recover)/iu, confidence: "high" }, - { category: "oauth", pattern: /(?:^|[\/_-])(oauth2?|authorize|callback)(?:$|[\/_-])/iu, confidence: "high" }, - { category: "sso", pattern: /(?:^|[\/_-])(sso|saml|oidc)(?:$|[\/_-])/iu, confidence: "high" }, - { category: "token", pattern: /(?:^|[\/_-])(token|jwt|refresh)(?:$|[\/_-])/iu, confidence: "medium" }, - { category: "session", pattern: /(?:^|[\/_-])(session|sessions)(?:$|[\/_-])/iu, confidence: "medium" }, - { category: "api-key", pattern: /(?:api[\/_-]?keys?|keys?[\/_-]?api)(?:$|[\/_-])/iu, confidence: "medium" }, - { category: "admin", pattern: /(?:^|[\/_-])(admin|administrator)(?:$|[\/_-])/iu, confidence: "medium" }, + { + category: "login", + pattern: /(?:^|[/_-])(login|log-in|signin|sign-in)(?:$|[/_-])/iu, + confidence: "high", + }, + { + category: "logout", + pattern: /(?:^|[/_-])(logout|log-out|signout|sign-out)(?:$|[/_-])/iu, + confidence: "high", + }, + { + category: "registration", + pattern: /(?:^|[/_-])(register|registration|signup|sign-up)(?:$|[/_-])/iu, + confidence: "high", + }, + { + category: "password-recovery", + pattern: + /(?:forgot|reset|recover)[/_-]?(?:password|account)|password[/_-]?(?:forgot|reset|recover)/iu, + confidence: "high", + }, + { + category: "oauth", + pattern: /(?:^|[/_-])(oauth2?|authorize|callback)(?:$|[/_-])/iu, + confidence: "high", + }, + { + category: "sso", + pattern: /(?:^|[/_-])(sso|saml|oidc)(?:$|[/_-])/iu, + confidence: "high", + }, + { + category: "token", + pattern: /(?:^|[/_-])(token|jwt|refresh)(?:$|[/_-])/iu, + confidence: "medium", + }, + { + category: "session", + pattern: /(?:^|[/_-])(session|sessions)(?:$|[/_-])/iu, + confidence: "medium", + }, + { + category: "api-key", + pattern: /(?:api[/_-]?keys?|keys?[/_-]?api)(?:$|[/_-])/iu, + confidence: "medium", + }, + { + category: "admin", + pattern: /(?:^|[/_-])(admin|administrator)(?:$|[/_-])/iu, + confidence: "medium", + }, ]; const BLOCKER_RULES: ReadonlyArray<{ - reason: DiscoveryBlockerReason; - pattern: RegExp; + reason: DiscoveryBlockerReason; + pattern: RegExp; }> = [ - { reason: "captcha-detected", pattern: /\b(?:captcha|recaptcha|hcaptcha)\b/iu }, - { reason: "bot-block-detected", pattern: /\b(?:bot detected|automated (?:traffic|request)|verify you are human)\b/iu }, - { reason: "waf-denied", pattern: /\b(?:web application firewall|waf|access denied|request blocked)\b/iu }, - { reason: "rate-limited", pattern: /\b(?:rate limit(?:ed|ing)?|too many requests|http\s*429)\b/iu }, - { reason: "auth-required", pattern: /\b(?:authentication required|login required|unauthorized|http\s*401)\b/iu }, - { reason: "approval-required", pattern: /\b(?:approval required|missing approval)\b/iu }, - { reason: "target-authorization-required", pattern: /\b(?:target authorization required|missing authorization)\b/iu }, - { reason: "workspace-locked", pattern: /\bworkspace (?:is )?locked\b/iu }, - { reason: "network-profile-blocked", pattern: /\b(?:network profile blocked|wrong network profile|egress denied)\b/iu }, - { reason: "tool-unavailable", pattern: /\b(?:tool unavailable|command not found|not installed)\b/iu }, + { + reason: "captcha-detected", + pattern: /\b(?:captcha|recaptcha|hcaptcha)\b/iu, + }, + { + reason: "bot-block-detected", + pattern: + /\b(?:bot detected|automated (?:traffic|request)|verify you are human)\b/iu, + }, + { + reason: "waf-denied", + pattern: + /\b(?:web application firewall|waf|access denied|request blocked)\b/iu, + }, + { + reason: "rate-limited", + pattern: /\b(?:rate limit(?:ed|ing)?|too many requests|http\s*429)\b/iu, + }, + { + reason: "auth-required", + pattern: + /\b(?:authentication required|login required|unauthorized|http\s*401)\b/iu, + }, + { + reason: "approval-required", + pattern: /\b(?:approval required|missing approval)\b/iu, + }, + { + reason: "target-authorization-required", + pattern: /\b(?:target authorization required|missing authorization)\b/iu, + }, + { reason: "workspace-locked", pattern: /\bworkspace (?:is )?locked\b/iu }, + { + reason: "network-profile-blocked", + pattern: + /\b(?:network profile blocked|wrong network profile|egress denied)\b/iu, + }, + { + reason: "tool-unavailable", + pattern: /\b(?:tool unavailable|command not found|not installed)\b/iu, + }, ]; -const SESSION_OBSERVATION = /\b(?:set-cookie|cookie|session(?:id)?|same-site|samesite|httponly)\b/iu; -const TOKEN_OBSERVATION = /\b(?:bearer|jwt|access[_ -]?token|refresh[_ -]?token|id[_ -]?token|api[_ -]?key)\b/iu; +const SESSION_OBSERVATION = + /\b(?:set-cookie|cookie|session(?:id)?|same-site|samesite|httponly)\b/iu; +const TOKEN_OBSERVATION = + /\b(?:bearer|jwt|access[_ -]?token|refresh[_ -]?token|id[_ -]?token|api[_ -]?key)\b/iu; export function normalizeDiscoveryArtifacts( - artifacts: readonly DiscoveryArtifactInput[], + artifacts: readonly DiscoveryArtifactInput[], ): NormalizedDiscovery { - const rawArtifactIds = uniqueSorted(artifacts.map((artifact) => artifact.artifactId)); - const urlSources = new Map>(); - const statusSources = new Map>(); - const blockerSources = new Map; evidence: string }>(); - const sessionObservations = new Set(); - const tokenObservations = new Set(); - - for (const artifact of artifacts) { - for (const rawUrl of artifact.content.match(URL_PATTERN) ?? []) { - const url = normalizeUrl(rawUrl); - if (!url) continue; - addSource(urlSources, url, artifact.artifactId); - } - - for (const pattern of STATUS_PATTERNS) { - pattern.lastIndex = 0; - for (const match of artifact.content.matchAll(pattern)) { - const status = Number(match[1]); - if (status >= 100 && status <= 599) addSource(statusSources, status, artifact.artifactId); - } - } - - for (const line of artifact.content.split(/\r?\n/u)) { - const excerpt = line.trim(); - if (!excerpt) continue; - - if (SESSION_OBSERVATION.test(excerpt)) sessionObservations.add(excerpt); - if (TOKEN_OBSERVATION.test(excerpt)) tokenObservations.add(excerpt); - - for (const rule of BLOCKER_RULES) { - if (!rule.pattern.test(excerpt)) continue; - const current = blockerSources.get(rule.reason); - if (current) { - current.artifactIds.add(artifact.artifactId); - } else { - blockerSources.set(rule.reason, { - artifactIds: new Set([artifact.artifactId]), - evidence: excerpt.slice(0, 500), - }); - } - } - } - } - - const urls = [...urlSources.entries()] - .map(([url, artifactIds]) => { - const parsed = new URL(url); - return { - url, - host: parsed.host, - path: parsed.pathname || "/", - sourceArtifactIds: uniqueSorted(artifactIds), - } satisfies NormalizedDiscoveryUrl; - }) - .sort((left, right) => left.url.localeCompare(right.url)); - - const authCandidates = urls.flatMap((entry) => { - const searchable = `${entry.path}${new URL(entry.url).search}`; - const matches = AUTH_ROUTE_RULES.filter((rule) => rule.pattern.test(searchable)); - if (matches.length === 0) return []; - return [{ - url: entry.url, - path: entry.path, - categories: matches.map((match) => match.category), - confidence: matches.some((match) => match.confidence === "high") ? "high" : "medium", - sourceArtifactIds: entry.sourceArtifactIds, - } satisfies AuthSurfaceCandidate]; - }); - - const families = new Map; artifactIds: Set }>(); - for (const [status, artifactIds] of statusSources) { - const family = `${Math.floor(status / 100)}xx` as DiscoveryResponseFamily["family"]; - const current = families.get(family) ?? { statuses: new Set(), artifactIds: new Set() }; - current.statuses.add(status); - for (const artifactId of artifactIds) current.artifactIds.add(artifactId); - families.set(family, current); - } - - return { - rawArtifactIds, - urls, - authCandidates, - responseFamilies: [...families.entries()] - .map(([family, value]) => ({ - family, - statuses: [...value.statuses].sort((left, right) => left - right), - sourceArtifactIds: uniqueSorted(value.artifactIds), - })) - .sort((left, right) => left.family.localeCompare(right.family)), - blockerSignals: [...blockerSources.entries()] - .map(([reason, value]) => ({ - reason, - sourceArtifactIds: uniqueSorted(value.artifactIds), - evidence: value.evidence, - })) - .sort((left, right) => left.reason.localeCompare(right.reason)), - observations: { - session: [...sessionObservations].sort(), - token: [...tokenObservations].sort(), - }, - }; + const rawArtifactIds = uniqueSorted( + artifacts.map((artifact) => artifact.artifactId), + ); + const urlSources = new Map>(); + const statusSources = new Map>(); + const blockerSources = new Map< + DiscoveryBlockerReason, + { artifactIds: Set; evidence: string } + >(); + const sessionObservations = new Set(); + const tokenObservations = new Set(); + + for (const artifact of artifacts) { + for (const rawUrl of artifact.content.match(URL_PATTERN) ?? []) { + const url = normalizeUrl(rawUrl); + if (!url) continue; + addSource(urlSources, url, artifact.artifactId); + } + + for (const pattern of STATUS_PATTERNS) { + pattern.lastIndex = 0; + for (const match of artifact.content.matchAll(pattern)) { + const status = Number(match[1]); + if (status >= 100 && status <= 599) + addSource(statusSources, status, artifact.artifactId); + } + } + + for (const line of artifact.content.split(/\r?\n/u)) { + const excerpt = line.trim(); + if (!excerpt) continue; + + if (SESSION_OBSERVATION.test(excerpt)) sessionObservations.add(excerpt); + if (TOKEN_OBSERVATION.test(excerpt)) tokenObservations.add(excerpt); + + for (const rule of BLOCKER_RULES) { + if (!rule.pattern.test(excerpt)) continue; + const current = blockerSources.get(rule.reason); + if (current) { + current.artifactIds.add(artifact.artifactId); + } else { + blockerSources.set(rule.reason, { + artifactIds: new Set([artifact.artifactId]), + evidence: excerpt.slice(0, 500), + }); + } + } + } + } + + const urls = [...urlSources.entries()] + .map(([url, artifactIds]) => { + const parsed = new URL(url); + return { + url, + host: parsed.host, + path: parsed.pathname || "/", + sourceArtifactIds: uniqueSorted(artifactIds), + } satisfies NormalizedDiscoveryUrl; + }) + .sort((left, right) => left.url.localeCompare(right.url)); + + const authCandidates = urls.flatMap((entry) => { + const searchable = `${entry.path}${new URL(entry.url).search}`; + const matches = AUTH_ROUTE_RULES.filter((rule) => + rule.pattern.test(searchable), + ); + if (matches.length === 0) return []; + return [ + { + url: entry.url, + path: entry.path, + categories: matches.map((match) => match.category), + confidence: matches.some((match) => match.confidence === "high") + ? "high" + : "medium", + sourceArtifactIds: entry.sourceArtifactIds, + } satisfies AuthSurfaceCandidate, + ]; + }); + + const families = new Map< + DiscoveryResponseFamily["family"], + { statuses: Set; artifactIds: Set } + >(); + for (const [status, artifactIds] of statusSources) { + const family = + `${Math.floor(status / 100)}xx` as DiscoveryResponseFamily["family"]; + const current = families.get(family) ?? { + statuses: new Set(), + artifactIds: new Set(), + }; + current.statuses.add(status); + for (const artifactId of artifactIds) current.artifactIds.add(artifactId); + families.set(family, current); + } + + return { + rawArtifactIds, + urls, + authCandidates, + responseFamilies: [...families.entries()] + .map(([family, value]) => ({ + family, + statuses: [...value.statuses].sort((left, right) => left - right), + sourceArtifactIds: uniqueSorted(value.artifactIds), + })) + .sort((left, right) => left.family.localeCompare(right.family)), + blockerSignals: [...blockerSources.entries()] + .map(([reason, value]) => ({ + reason, + sourceArtifactIds: uniqueSorted(value.artifactIds), + evidence: value.evidence, + })) + .sort((left, right) => left.reason.localeCompare(right.reason)), + observations: { + session: [...sessionObservations].sort(), + token: [...tokenObservations].sort(), + }, + }; } function normalizeUrl(raw: string): string | undefined { - const cleaned = raw.replace(/[),.;\]}]+$/u, ""); - try { - const parsed = new URL(cleaned); - if (parsed.protocol !== "http:" && parsed.protocol !== "https:") return undefined; - parsed.hash = ""; - return parsed.toString(); - } catch { - return undefined; - } + const cleaned = raw.replace(/[),.;\]}]+$/u, ""); + try { + const parsed = new URL(cleaned); + if (parsed.protocol !== "http:" && parsed.protocol !== "https:") + return undefined; + parsed.hash = ""; + return parsed.toString(); + } catch { + return undefined; + } } -function addSource(map: Map>, key: T, artifactId: string): void { - const sources = map.get(key) ?? new Set(); - sources.add(artifactId); - map.set(key, sources); +function addSource( + map: Map>, + key: T, + artifactId: string, +): void { + const sources = map.get(key) ?? new Set(); + sources.add(artifactId); + map.set(key, sources); } function uniqueSorted(values: Iterable): string[] { - return [...new Set(values)].sort(); + return [...new Set(values)].sort(); } diff --git a/src/server/recon/index.ts b/src/server/recon/index.ts index 7232d89a6..1f6e2a9ec 100644 --- a/src/server/recon/index.ts +++ b/src/server/recon/index.ts @@ -1,12 +1,12 @@ export { - normalizeDiscoveryArtifacts, - type AuthSurfaceCandidate, - type AuthSurfaceCategory, - type DiscoveryArtifactInput, - type DiscoveryArtifactSource, - type DiscoveryBlockerReason, - type DiscoveryBlockerSignal, - type DiscoveryResponseFamily, - type NormalizedDiscovery, - type NormalizedDiscoveryUrl, + type AuthSurfaceCandidate, + type AuthSurfaceCategory, + type DiscoveryArtifactInput, + type DiscoveryArtifactSource, + type DiscoveryBlockerReason, + type DiscoveryBlockerSignal, + type DiscoveryResponseFamily, + type NormalizedDiscovery, + type NormalizedDiscoveryUrl, + normalizeDiscoveryArtifacts, } from "./discovery-artifact-normalizer"; diff --git a/tests/integration/discovery-artifact-normalizer.test.ts b/tests/integration/discovery-artifact-normalizer.test.ts index 5046adbe7..e355b2f73 100644 --- a/tests/integration/discovery-artifact-normalizer.test.ts +++ b/tests/integration/discovery-artifact-normalizer.test.ts @@ -3,64 +3,78 @@ import { describe, expect, it } from "vitest"; import { normalizeDiscoveryArtifacts } from "../../src/server/recon"; describe("discovery artifact normalization", () => { - it("turns mixed passive evidence into deduplicated, attributable auth-surface signals", () => { - const result = normalizeDiscoveryArtifacts([ - { - artifactId: "artifact-gau", - source: "lab-command", - content: [ - "https://app.example.test/login", - "https://app.example.test/oauth/callback?code=example#fragment", - "https://app.example.test/admin/", - "https://app.example.test/login", - ].join("\n"), - }, - { - artifactId: "artifact-probe", - source: "http-probe", - content: [ - "HTTP/2 401", - "set-cookie: sessionid=[REDACTED]; HttpOnly; SameSite=Lax", - "www-authenticate: Bearer", - "Authentication required", - "HTTP 429 Too Many Requests - rate limited", - "https://app.example.test/api/v2/session", - ].join("\n"), - }, - ]); + it("turns mixed passive evidence into deduplicated, attributable auth-surface signals", () => { + const result = normalizeDiscoveryArtifacts([ + { + artifactId: "artifact-gau", + source: "lab-command", + content: [ + "https://app.example.test/login", + "https://app.example.test/oauth/callback?code=example#fragment", + "https://app.example.test/admin/", + "https://app.example.test/login", + ].join("\n"), + }, + { + artifactId: "artifact-probe", + source: "http-probe", + content: [ + "HTTP/2 401", + "set-cookie: sessionid=[REDACTED]; HttpOnly; SameSite=Lax", + "www-authenticate: Bearer", + "Authentication required", + "HTTP 429 Too Many Requests - rate limited", + "https://app.example.test/api/v2/session", + ].join("\n"), + }, + ]); - expect(result.rawArtifactIds).toEqual(["artifact-gau", "artifact-probe"]); - expect(result.urls.map((entry) => entry.url)).toEqual([ - "https://app.example.test/admin/", - "https://app.example.test/api/v2/session", - "https://app.example.test/login", - "https://app.example.test/oauth/callback?code=example", - ]); - expect(result.urls.find((entry) => entry.path === "/login")?.sourceArtifactIds).toEqual([ - "artifact-gau", - ]); - expect(result.authCandidates.map((candidate) => ({ - path: candidate.path, - categories: candidate.categories, - confidence: candidate.confidence, - }))).toEqual([ - { path: "/admin/", categories: ["admin"], confidence: "medium" }, - { path: "/api/v2/session", categories: ["session"], confidence: "medium" }, - { path: "/login", categories: ["login"], confidence: "high" }, - { path: "/oauth/callback", categories: ["oauth"], confidence: "high" }, - ]); - expect(result.responseFamilies).toEqual([ - { family: "4xx", statuses: [401, 429], sourceArtifactIds: ["artifact-probe"] }, - ]); - expect(result.blockerSignals.map((signal) => signal.reason)).toEqual([ - "auth-required", - "rate-limited", - ]); - expect(result.blockerSignals.every((signal) => signal.sourceArtifactIds.includes("artifact-probe"))).toBe(true); - expect(result.observations.session).toEqual([ - "https://app.example.test/api/v2/session", - "set-cookie: sessionid=[REDACTED]; HttpOnly; SameSite=Lax", - ]); - expect(result.observations.token).toEqual(["www-authenticate: Bearer"]); - }); + expect(result.rawArtifactIds).toEqual(["artifact-gau", "artifact-probe"]); + expect(result.urls.map((entry) => entry.url)).toEqual([ + "https://app.example.test/admin/", + "https://app.example.test/api/v2/session", + "https://app.example.test/login", + "https://app.example.test/oauth/callback?code=example", + ]); + expect( + result.urls.find((entry) => entry.path === "/login")?.sourceArtifactIds, + ).toEqual(["artifact-gau"]); + expect( + result.authCandidates.map((candidate) => ({ + path: candidate.path, + categories: candidate.categories, + confidence: candidate.confidence, + })), + ).toEqual([ + { path: "/admin/", categories: ["admin"], confidence: "medium" }, + { + path: "/api/v2/session", + categories: ["session"], + confidence: "medium", + }, + { path: "/login", categories: ["login"], confidence: "high" }, + { path: "/oauth/callback", categories: ["oauth"], confidence: "high" }, + ]); + expect(result.responseFamilies).toEqual([ + { + family: "4xx", + statuses: [401, 429], + sourceArtifactIds: ["artifact-probe"], + }, + ]); + expect(result.blockerSignals.map((signal) => signal.reason)).toEqual([ + "auth-required", + "rate-limited", + ]); + expect( + result.blockerSignals.every((signal) => + signal.sourceArtifactIds.includes("artifact-probe"), + ), + ).toBe(true); + expect(result.observations.session).toEqual([ + "https://app.example.test/api/v2/session", + "set-cookie: sessionid=[REDACTED]; HttpOnly; SameSite=Lax", + ]); + expect(result.observations.token).toEqual(["www-authenticate: Bearer"]); + }); });