diff --git a/src/app/api/projects/[projectId]/recon/passive-auth-surface/route.ts b/src/app/api/projects/[projectId]/recon/passive-auth-surface/route.ts new file mode 100644 index 000000000..7740e1b44 --- /dev/null +++ b/src/app/api/projects/[projectId]/recon/passive-auth-surface/route.ts @@ -0,0 +1,114 @@ +import { + createStoredPassiveAuthSurface, + PassiveAuthSurfaceArtifactError, + PassiveAuthSurfaceAuthorizationError, + type StoredPassiveArtifactRef, +} from "../../../../../../server/recon"; +import { + assertSameOriginMutatingRequest, + badRequest, + forbidden, + handleApiError, + notFound, + ok, + readJson, +} from "../../../../_shared/http"; + +export const dynamic = "force-dynamic"; + +type Context = { params: Promise<{ projectId: string }> }; + +export async function POST(request: Request, context: Context) { + try { + assertSameOriginMutatingRequest(request); + const { projectId } = await context.params; + const body = parseRequest(await readJson(request)); + const result = await createStoredPassiveAuthSurface({ projectId, ...body }); + return ok({ + authorizationId: result.authorizationId, + normalized: result.normalized, + summary: result.summary, + artifact: result.artifact, + sourceArtifacts: result.sourceArtifacts, + }); + } catch (error) { + if (error instanceof PassiveAuthSurfaceAuthorizationError) { + return forbidden(error.message); + } + if (error instanceof PassiveAuthSurfaceArtifactError) { + return notFound(error.message); + } + if ( + error instanceof Error && + /required|must be|allows?|unique/i.test(error.message) + ) { + return badRequest(error.message); + } + return handleApiError(error, { request }); + } +} + +function parseRequest(value: unknown): { + targetId: string; + threadId?: string; + taskId?: string; + artifacts: StoredPassiveArtifactRef[]; +} { + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new Error("Request body must be an object."); + } + const body = value as Record; + const allowedKeys = new Set(["targetId", "threadId", "taskId", "artifacts"]); + if (Object.keys(body).some((key) => !allowedKeys.has(key))) { + throw new Error( + "Only targetId, threadId, taskId, and stored artifact references are allowed.", + ); + } + const targetId = requiredString(body.targetId, "targetId"); + const threadId = optionalString(body.threadId, "threadId"); + const taskId = optionalString(body.taskId, "taskId"); + if (!Array.isArray(body.artifacts)) { + throw new Error( + "artifacts must be an array of stored artifact references.", + ); + } + const artifacts = body.artifacts.map((value, index) => + parseArtifactRef(value, index), + ); + return { + targetId, + ...(threadId ? { threadId } : {}), + ...(taskId ? { taskId } : {}), + artifacts, + }; +} + +function parseArtifactRef( + value: unknown, + index: number, +): StoredPassiveArtifactRef { + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new Error(`artifacts[${index}] must be a stored artifact reference.`); + } + const reference = value as Record; + if (Object.keys(reference).some((key) => key !== "artifactId")) { + throw new Error(`artifacts[${index}] only allows artifactId.`); + } + const artifactId = requiredString( + reference.artifactId, + `artifacts[${index}].artifactId`, + ); + return { artifactId }; +} + +function requiredString(value: unknown, name: string): string { + if (typeof value !== "string" || !value.trim()) { + throw new Error(`${name} is required.`); + } + return value.trim(); +} + +function optionalString(value: unknown, name: string): string | undefined { + if (value === undefined) return undefined; + return requiredString(value, name); +} diff --git a/src/server/evidence/artifact-service.ts b/src/server/evidence/artifact-service.ts index e3559fc16..bb8da9dcb 100644 --- a/src/server/evidence/artifact-service.ts +++ b/src/server/evidence/artifact-service.ts @@ -11,7 +11,7 @@ import { type EvidenceSource, ingestEvidenceBestEffort, } from "./index"; -import { redactEvidenceSecrets } from "./ingestion"; +import { EVIDENCE_SOURCES, redactEvidenceSecrets } from "./ingestion"; const DEFAULT_MAX_INLINE_BYTES = Number.parseInt( process.env.ARTIFACT_INLINE_MAX_BYTES ?? "1500000", @@ -93,6 +93,9 @@ export type ReadArtifactTextResult = { contentType: string | null; sizeBytes: number | null; sha256: string | null; + source: EvidenceSource | null; + targetIds: string[]; + targetScope: "targets" | "project" | null; text: string; truncated: boolean; }; @@ -296,8 +299,9 @@ export function createArtifactService(config: ArtifactServiceConfig = {}) { size_bytes: number | null; sha256: string | null; inline_content: string | null; + metadata: unknown; }>( - `SELECT id, project_id, thread_id, name, content_type, storage_bucket, storage_key, size_bytes, sha256, inline_content + `SELECT id, project_id, thread_id, name, content_type, storage_bucket, storage_key, size_bytes, sha256, inline_content, metadata FROM artifacts WHERE id = $1 AND project_id = $2`, [input.artifactId, input.projectId], @@ -310,6 +314,7 @@ export function createArtifactService(config: ArtifactServiceConfig = {}) { } const maxBytes = normalizeReadLimit(input.maxBytes); + const provenance = readArtifactProvenance(row.metadata); if (row.inline_content !== null) { const redacted = redactEvidenceSecrets(row.inline_content); const bytes = new TextEncoder().encode(redacted); @@ -323,6 +328,7 @@ export function createArtifactService(config: ArtifactServiceConfig = {}) { contentType: row.content_type, sizeBytes: row.size_bytes, sha256: row.sha256, + ...provenance, text, truncated, }; @@ -358,6 +364,7 @@ export function createArtifactService(config: ArtifactServiceConfig = {}) { contentType: row.content_type, sizeBytes: row.size_bytes, sha256: row.sha256, + ...provenance, text, truncated, }; @@ -546,6 +553,36 @@ function normalizeReadLimit(value: number | undefined) { return Math.max(1, Math.min(Math.floor(value), 5_000_000)); } +function readArtifactProvenance(value: unknown): Pick< + ReadArtifactTextResult, + "source" | "targetIds" | "targetScope" +> { + let metadata: unknown = value; + if (typeof metadata === "string") { + try { + metadata = JSON.parse(metadata); + } catch { + metadata = null; + } + } + if (!metadata || typeof metadata !== "object" || Array.isArray(metadata)) { + return { source: null, targetIds: [], targetScope: null }; + } + const record = metadata as Record; + const source = EVIDENCE_SOURCES.includes(record.source as EvidenceSource) + ? (record.source as EvidenceSource) + : null; + const targetIds = Array.isArray(record.targetIds) + ? [...new Set(record.targetIds.filter((id): id is string => typeof id === "string" && !!id))] + : typeof record.targetId === "string" && record.targetId + ? [record.targetId] + : []; + const targetScope = record.targetScope === "targets" || record.targetScope === "project" + ? record.targetScope + : null; + return { source, targetIds, targetScope }; +} + function isLikelyTextArtifact(name: string, contentType: string | null) { const normalizedContentType = contentType?.toLowerCase().split(";")[0]?.trim(); if ( diff --git a/src/server/evidence/ingestion.ts b/src/server/evidence/ingestion.ts index 31ac73eed..112ca46bf 100644 --- a/src/server/evidence/ingestion.ts +++ b/src/server/evidence/ingestion.ts @@ -23,6 +23,8 @@ export const EVIDENCE_SOURCES = [ "patch-verification", "patch-remediation", "stage-handoff", + "passive-recon", + "reference", ] as const; export type EvidenceSource = (typeof EVIDENCE_SOURCES)[number]; diff --git a/src/server/recon/discovery-artifact-normalizer.ts b/src/server/recon/discovery-artifact-normalizer.ts new file mode 100644 index 000000000..7fd67c42d --- /dev/null +++ b/src/server/recon/discovery-artifact-normalizer.ts @@ -0,0 +1,345 @@ +export type DiscoveryArtifactSource = + | "upload" + | "terminal-note" + | "lab-command" + | "http-probe" + | "reference" + | (string & {}); + +export type DiscoveryArtifactInput = { + artifactId: string; + content: string; + source: DiscoveryArtifactSource; +}; + +export type AuthSurfaceCategory = + | "login" + | "logout" + | "registration" + | "password-recovery" + | "oauth" + | "sso" + | "token" + | "session" + | "api-key" + | "admin"; + +export type DiscoveryBlockerReason = + | "bot-block-detected" + | "captcha-detected" + | "waf-denied" + | "rate-limited" + | "auth-required" + | "approval-required" + | "target-authorization-required" + | "workspace-locked" + | "network-profile-blocked" + | "tool-unavailable"; + +export type NormalizedDiscoveryUrl = { + url: string; + host: string; + path: string; + sourceArtifactIds: string[]; +}; + +export type AuthSurfaceCandidate = { + url: string; + path: string; + categories: AuthSurfaceCategory[]; + confidence: "high" | "medium"; + sourceArtifactIds: string[]; +}; + +export type DiscoveryBlockerSignal = { + reason: DiscoveryBlockerReason; + sourceArtifactIds: string[]; + evidence: string; +}; + +export type DiscoveryResponseFamily = { + family: "1xx" | "2xx" | "3xx" | "4xx" | "5xx"; + statuses: number[]; + sourceArtifactIds: string[]; +}; + +export type NormalizedDiscovery = { + rawArtifactIds: string[]; + urls: NormalizedDiscoveryUrl[]; + authCandidates: AuthSurfaceCandidate[]; + responseFamilies: DiscoveryResponseFamily[]; + blockerSignals: DiscoveryBlockerSignal[]; + observations: { + session: string[]; + token: string[]; + }; +}; + +const URL_PATTERN = /https?:\/\/[^\s<>"'`]+/giu; +const STATUS_PATTERNS = [ + /\bHTTP\/\d(?:\.\d)?\s+(\d{3})\b/giu, + /\bHTTP\s+(\d{3})\b/giu, + /(?:^|\s)\[(\d{3})\](?=\s|$)/gmu, +]; + +const AUTH_ROUTE_RULES: ReadonlyArray<{ + category: AuthSurfaceCategory; + pattern: RegExp; + confidence: "high" | "medium"; +}> = [ + { + category: "login", + pattern: /(?:^|[/_-])(login|log-in|signin|sign-in)(?:$|[/_-])/iu, + confidence: "high", + }, + { + category: "logout", + pattern: /(?:^|[/_-])(logout|log-out|signout|sign-out)(?:$|[/_-])/iu, + confidence: "high", + }, + { + category: "registration", + pattern: /(?:^|[/_-])(register|registration|signup|sign-up)(?:$|[/_-])/iu, + confidence: "high", + }, + { + category: "password-recovery", + pattern: + /(?:forgot|reset|recover)[/_-]?(?:password|account)|password[/_-]?(?:forgot|reset|recover)/iu, + confidence: "high", + }, + { + category: "oauth", + pattern: /(?:^|[/_-])(oauth2?|authorize|callback)(?:$|[/_-])/iu, + confidence: "high", + }, + { + category: "sso", + pattern: /(?:^|[/_-])(sso|saml|oidc)(?:$|[/_-])/iu, + confidence: "high", + }, + { + category: "token", + pattern: /(?:^|[/_-])(token|jwt|refresh)(?:$|[/_-])/iu, + confidence: "medium", + }, + { + category: "session", + pattern: /(?:^|[/_-])(session|sessions)(?:$|[/_-])/iu, + confidence: "medium", + }, + { + category: "api-key", + pattern: /(?:api[/_-]?keys?|keys?[/_-]?api)(?:$|[/_-])/iu, + confidence: "medium", + }, + { + category: "admin", + pattern: /(?:^|[/_-])(admin|administrator)(?:$|[/_-])/iu, + confidence: "medium", + }, +]; + +const BLOCKER_RULES: ReadonlyArray<{ + reason: DiscoveryBlockerReason; + pattern: RegExp; +}> = [ + { + reason: "captcha-detected", + pattern: /\b(?:captcha|recaptcha|hcaptcha)\b/iu, + }, + { + reason: "bot-block-detected", + pattern: + /\b(?:bot detected|automated (?:traffic|request)|verify you are human)\b/iu, + }, + { + reason: "waf-denied", + pattern: + /\b(?:web application firewall|waf|access denied|request blocked)\b/iu, + }, + { + reason: "rate-limited", + pattern: /\b(?:rate limit(?:ed|ing)?|too many requests|http\s*429)\b/iu, + }, + { + reason: "auth-required", + pattern: + /\b(?:authentication required|login required|unauthorized|http\s*401)\b/iu, + }, + { + reason: "approval-required", + pattern: /\b(?:approval required|missing approval)\b/iu, + }, + { + reason: "target-authorization-required", + pattern: /\b(?:target authorization required|missing authorization)\b/iu, + }, + { reason: "workspace-locked", pattern: /\bworkspace (?:is )?locked\b/iu }, + { + reason: "network-profile-blocked", + pattern: + /\b(?:network profile blocked|wrong network profile|egress denied)\b/iu, + }, + { + reason: "tool-unavailable", + pattern: /\b(?:tool unavailable|command not found|not installed)\b/iu, + }, +]; + +const SESSION_OBSERVATION = + /\b(?:set-cookie|cookie|session(?:id)?|same-site|samesite|httponly)\b/iu; +const TOKEN_OBSERVATION = + /\b(?:bearer|jwt|access[_ -]?token|refresh[_ -]?token|id[_ -]?token|api[_ -]?key)\b/iu; + +export function normalizeDiscoveryArtifacts( + artifacts: readonly DiscoveryArtifactInput[], +): NormalizedDiscovery { + const rawArtifactIds = uniqueSorted( + artifacts.map((artifact) => artifact.artifactId), + ); + const urlSources = new Map>(); + const statusSources = new Map>(); + const blockerSources = new Map< + DiscoveryBlockerReason, + { artifactIds: Set; evidence: string } + >(); + const sessionObservations = new Set(); + const tokenObservations = new Set(); + + for (const artifact of artifacts) { + for (const rawUrl of artifact.content.match(URL_PATTERN) ?? []) { + const url = normalizeUrl(rawUrl); + if (!url) continue; + addSource(urlSources, url, artifact.artifactId); + } + + for (const pattern of STATUS_PATTERNS) { + pattern.lastIndex = 0; + for (const match of artifact.content.matchAll(pattern)) { + const status = Number(match[1]); + if (status >= 100 && status <= 599) + addSource(statusSources, status, artifact.artifactId); + } + } + + for (const line of artifact.content.split(/\r?\n/u)) { + const excerpt = line.trim(); + if (!excerpt) continue; + + if (SESSION_OBSERVATION.test(excerpt)) sessionObservations.add(excerpt); + if (TOKEN_OBSERVATION.test(excerpt)) tokenObservations.add(excerpt); + + for (const rule of BLOCKER_RULES) { + if (!rule.pattern.test(excerpt)) continue; + const current = blockerSources.get(rule.reason); + if (current) { + current.artifactIds.add(artifact.artifactId); + } else { + blockerSources.set(rule.reason, { + artifactIds: new Set([artifact.artifactId]), + evidence: excerpt.slice(0, 500), + }); + } + } + } + } + + const urls = [...urlSources.entries()] + .map(([url, artifactIds]) => { + const parsed = new URL(url); + return { + url, + host: parsed.host, + path: parsed.pathname || "/", + sourceArtifactIds: uniqueSorted(artifactIds), + } satisfies NormalizedDiscoveryUrl; + }) + .sort((left, right) => left.url.localeCompare(right.url)); + + const authCandidates = urls.flatMap((entry) => { + const searchable = `${entry.path}${new URL(entry.url).search}`; + const matches = AUTH_ROUTE_RULES.filter((rule) => + rule.pattern.test(searchable), + ); + if (matches.length === 0) return []; + return [ + { + url: entry.url, + path: entry.path, + categories: matches.map((match) => match.category), + confidence: matches.some((match) => match.confidence === "high") + ? "high" + : "medium", + sourceArtifactIds: entry.sourceArtifactIds, + } satisfies AuthSurfaceCandidate, + ]; + }); + + const families = new Map< + DiscoveryResponseFamily["family"], + { statuses: Set; artifactIds: Set } + >(); + for (const [status, artifactIds] of statusSources) { + const family = + `${Math.floor(status / 100)}xx` as DiscoveryResponseFamily["family"]; + const current = families.get(family) ?? { + statuses: new Set(), + artifactIds: new Set(), + }; + current.statuses.add(status); + for (const artifactId of artifactIds) current.artifactIds.add(artifactId); + families.set(family, current); + } + + return { + rawArtifactIds, + urls, + authCandidates, + responseFamilies: [...families.entries()] + .map(([family, value]) => ({ + family, + statuses: [...value.statuses].sort((left, right) => left - right), + sourceArtifactIds: uniqueSorted(value.artifactIds), + })) + .sort((left, right) => left.family.localeCompare(right.family)), + blockerSignals: [...blockerSources.entries()] + .map(([reason, value]) => ({ + reason, + sourceArtifactIds: uniqueSorted(value.artifactIds), + evidence: value.evidence, + })) + .sort((left, right) => left.reason.localeCompare(right.reason)), + observations: { + session: [...sessionObservations].sort(), + token: [...tokenObservations].sort(), + }, + }; +} + +function normalizeUrl(raw: string): string | undefined { + const cleaned = raw.replace(/[),.;\]}]+$/u, ""); + try { + const parsed = new URL(cleaned); + if (parsed.protocol !== "http:" && parsed.protocol !== "https:") + return undefined; + parsed.hash = ""; + return parsed.toString(); + } catch { + return undefined; + } +} + +function addSource( + map: Map>, + key: T, + artifactId: string, +): void { + const sources = map.get(key) ?? new Set(); + sources.add(artifactId); + map.set(key, sources); +} + +function uniqueSorted(values: Iterable): string[] { + return [...new Set(values)].sort(); +} diff --git a/src/server/recon/index.ts b/src/server/recon/index.ts new file mode 100644 index 000000000..df776c753 --- /dev/null +++ b/src/server/recon/index.ts @@ -0,0 +1,31 @@ +export { + type AuthSurfaceCandidate, + type AuthSurfaceCategory, + type DiscoveryArtifactInput, + type DiscoveryArtifactSource, + type DiscoveryBlockerReason, + type DiscoveryBlockerSignal, + type DiscoveryResponseFamily, + type NormalizedDiscovery, + type NormalizedDiscoveryUrl, + normalizeDiscoveryArtifacts, +} from "./discovery-artifact-normalizer"; +export { + buildPassiveAuthSurfaceSummary, + type PassiveAuthSurfaceSummary, + type PersistPassiveAuthSurfaceInput, + type PersistPassiveAuthSurfaceResult, + persistPassiveAuthSurface, +} from "./passive-auth-surface"; +export { + type CreateStoredPassiveAuthSurfaceInput, + type CreateStoredPassiveAuthSurfaceResult, + createStoredPassiveAuthSurface, + MAX_PASSIVE_AUTH_SOURCE_ARTIFACTS, + MAX_PASSIVE_AUTH_SOURCE_BYTES, + PassiveAuthSurfaceArtifactError, + PassiveAuthSurfaceAuthorizationError, + STORED_PASSIVE_ARTIFACT_SOURCES, + type StoredPassiveArtifactRef, + type StoredPassiveArtifactSource, +} from "./stored-passive-auth-surface"; diff --git a/src/server/recon/passive-auth-surface.ts b/src/server/recon/passive-auth-surface.ts new file mode 100644 index 000000000..7e146d167 --- /dev/null +++ b/src/server/recon/passive-auth-surface.ts @@ -0,0 +1,194 @@ +import type { + ArtifactServiceInstance, + CreateArtifactResult, +} from "../evidence"; +import { + type AuthSurfaceCategory, + type DiscoveryArtifactInput, + type DiscoveryBlockerReason, + type NormalizedDiscovery, + normalizeDiscoveryArtifacts, +} from "./discovery-artifact-normalizer"; + +export type PassiveAuthSurfaceSummary = { + targetId: string; + rawArtifactIds: string[]; + rawArtifacts: Array<{ + artifactId: string; + source: DiscoveryArtifactInput["source"]; + }>; + authRoutes: Array<{ + category: AuthSurfaceCategory; + confidence: "high" | "medium"; + urls: string[]; + evidenceArtifactIds: string[]; + }>; + blockers: Array<{ + reason: DiscoveryBlockerReason; + evidenceArtifactIds: string[]; + evidence: string; + }>; + unknowns: string[]; + nextSteps: { + passive: string[]; + approvalGated: string[]; + reportOrPatch: string[]; + }; +}; + +export type PersistPassiveAuthSurfaceInput = { + projectId: string; + threadId?: string; + targetId: string; + taskId?: string; + artifacts: readonly DiscoveryArtifactInput[]; +}; + +export type PersistPassiveAuthSurfaceResult = { + normalized: NormalizedDiscovery; + summary: PassiveAuthSurfaceSummary; + artifact: CreateArtifactResult; +}; + +type ArtifactWriter = Pick; + +export async function persistPassiveAuthSurface( + input: PersistPassiveAuthSurfaceInput, + artifactWriter: ArtifactWriter, +): Promise { + const normalized = normalizeDiscoveryArtifacts(input.artifacts); + const summary = buildPassiveAuthSurfaceSummary( + input.targetId, + normalized, + input.artifacts, + ); + const content = JSON.stringify(summary, null, 2); + const artifact = await artifactWriter.createArtifact({ + projectId: input.projectId, + ...(input.threadId ? { threadId: input.threadId } : {}), + targetId: input.targetId, + ...(input.taskId ? { taskId: input.taskId } : {}), + name: `passive-auth-surface-${input.targetId}.json`, + kind: "report", + contentType: "application/json", + content, + indexText: content, + source: "passive-recon", + indexForRag: true, + agentGenerated: true, + metadata: { + workflow: "passive-auth-surface-v1", + rawArtifactIds: summary.rawArtifactIds, + rawArtifacts: summary.rawArtifacts, + blockerReasons: summary.blockers.map((blocker) => blocker.reason), + authCategories: summary.authRoutes.map((route) => route.category), + }, + }); + + return { normalized, summary, artifact }; +} + +export function buildPassiveAuthSurfaceSummary( + targetId: string, + normalized: NormalizedDiscovery, + sourceArtifacts: readonly Pick< + DiscoveryArtifactInput, + "artifactId" | "source" + >[] = [], +): PassiveAuthSurfaceSummary { + const authRoutes = new Map< + AuthSurfaceCategory, + { + confidence: "high" | "medium"; + urls: Set; + evidenceArtifactIds: Set; + } + >(); + + for (const candidate of normalized.authCandidates) { + for (const category of candidate.categories) { + const group = authRoutes.get(category) ?? { + confidence: "medium" as const, + urls: new Set(), + evidenceArtifactIds: new Set(), + }; + if (candidate.confidence === "high") group.confidence = "high"; + group.urls.add(candidate.url); + for (const artifactId of candidate.sourceArtifactIds) { + group.evidenceArtifactIds.add(artifactId); + } + authRoutes.set(category, group); + } + } + + const groupedRoutes = [...authRoutes.entries()] + .map(([category, group]) => ({ + category, + confidence: group.confidence, + urls: [...group.urls].sort(), + evidenceArtifactIds: [...group.evidenceArtifactIds].sort(), + })) + .sort((left, right) => left.category.localeCompare(right.category)); + const blockerReasons = new Set( + normalized.blockerSignals.map((blocker) => blocker.reason), + ); + + return { + targetId, + rawArtifactIds: [...normalized.rawArtifactIds], + rawArtifacts: sourceArtifacts.map(({ artifactId, source }) => ({ + artifactId, + source, + })), + authRoutes: groupedRoutes, + blockers: normalized.blockerSignals.map((blocker) => ({ + reason: blocker.reason, + evidenceArtifactIds: [...blocker.sourceArtifactIds], + evidence: blocker.evidence, + })), + unknowns: buildUnknowns( + new Set(groupedRoutes.map((route) => route.category)), + ), + nextSteps: { + passive: [ + "Review retrieved reference material and existing project evidence for the mapped routes.", + "Resolve remaining auth-flow unknowns from supplied source, manifests, and historical artifacts.", + ], + approvalGated: blockerReasons.size + ? [ + "Resolve recorded blockers before proposing any active validation.", + "Request a target-bound approval for the narrowest reproducible probe only if passive evidence is insufficient.", + ] + : [ + "Request a target-bound approval before any live request, browser interaction, or credential test.", + ], + reportOrPatch: [ + "Keep mapped routes as evidence-backed hypotheses until validation establishes impact.", + ], + }, + }; +} + +function buildUnknowns(categories: ReadonlySet): string[] { + const unknowns: string[] = []; + if ( + !categories.has("login") && + !categories.has("sso") && + !categories.has("oauth") + ) { + unknowns.push( + "Primary authentication entry point is not established by current evidence.", + ); + } + if (!categories.has("session") && !categories.has("token")) { + unknowns.push( + "Session or token lifecycle is not established by current evidence.", + ); + } + if (!categories.has("password-recovery")) { + unknowns.push( + "Account recovery surface is not established by current evidence.", + ); + } + return unknowns; +} diff --git a/src/server/recon/stored-passive-auth-surface.ts b/src/server/recon/stored-passive-auth-surface.ts new file mode 100644 index 000000000..3d272331e --- /dev/null +++ b/src/server/recon/stored-passive-auth-surface.ts @@ -0,0 +1,197 @@ +import { withDatabase } from "../db/client"; +import { type ArtifactServiceInstance, getArtifactService } from "../evidence"; +import { listActiveTargetAuthorizationRows } from "../targets/authorization-ledger"; +import { + type PersistPassiveAuthSurfaceResult, + persistPassiveAuthSurface, +} from "./passive-auth-surface"; + +export const MAX_PASSIVE_AUTH_SOURCE_ARTIFACTS = 8; +export const MAX_PASSIVE_AUTH_SOURCE_BYTES = 250_000; + +export const STORED_PASSIVE_ARTIFACT_SOURCES = [ + "upload", + "terminal-note", + "command-transcript", + "http-probe", + "reference", +] as const; + +export type StoredPassiveArtifactSource = + (typeof STORED_PASSIVE_ARTIFACT_SOURCES)[number]; + +export type StoredPassiveArtifactRef = { + artifactId: string; +}; + +export type CreateStoredPassiveAuthSurfaceInput = { + projectId: string; + targetId: string; + threadId?: string; + taskId?: string; + artifacts: readonly StoredPassiveArtifactRef[]; +}; + +export type CreateStoredPassiveAuthSurfaceResult = + PersistPassiveAuthSurfaceResult & { + authorizationId: string; + sourceArtifacts: Array<{ + artifactId: string; + source: StoredPassiveArtifactSource; + truncated: false; + }>; + }; + +export class PassiveAuthSurfaceAuthorizationError extends Error { + constructor(message: string) { + super(message); + this.name = "PassiveAuthSurfaceAuthorizationError"; + } +} + +export class PassiveAuthSurfaceArtifactError extends Error { + constructor(message: string) { + super(message); + this.name = "PassiveAuthSurfaceArtifactError"; + } +} + +type Dependencies = { + artifactService?: ArtifactServiceInstance; + listActiveAuthorizations?: ( + projectId: string, + ) => Promise>; +}; + +export async function createStoredPassiveAuthSurface( + input: CreateStoredPassiveAuthSurfaceInput, + dependencies: Dependencies = {}, +): Promise { + assertStoredArtifactRefs(input.artifacts); + const activeAuthorizations = dependencies.listActiveAuthorizations + ? await dependencies.listActiveAuthorizations(input.projectId) + : await withDatabase((db) => + listActiveTargetAuthorizationRows(db, input.projectId), + ); + const authorization = activeAuthorizations.find( + (record) => record.targetId === input.targetId, + ); + if (!authorization) { + throw new PassiveAuthSurfaceAuthorizationError( + `Target ${input.targetId} does not have an active approved authorization in this project.`, + ); + } + + const artifactService = dependencies.artifactService ?? getArtifactService(); + const readArtifactText = artifactService.readArtifactText; + if (!readArtifactText) { + throw new PassiveAuthSurfaceArtifactError( + "Stored artifact text reads are unavailable.", + ); + } + + const artifacts = await Promise.all( + input.artifacts.map(async (reference) => { + let stored: Awaited< + ReturnType> + >; + try { + stored = await readArtifactText({ + projectId: input.projectId, + artifactId: reference.artifactId, + maxBytes: MAX_PASSIVE_AUTH_SOURCE_BYTES, + }); + } catch { + throw new PassiveAuthSurfaceArtifactError( + `Artifact ${reference.artifactId} was not found as readable text in this project.`, + ); + } + if (!stored) { + throw new PassiveAuthSurfaceArtifactError( + `Artifact ${reference.artifactId} was not found as readable text in this project.`, + ); + } + if (stored.truncated) { + throw new PassiveAuthSurfaceArtifactError( + `Artifact ${reference.artifactId} exceeds the ${MAX_PASSIVE_AUTH_SOURCE_BYTES}-byte passive summary limit.`, + ); + } + if (!stored.source || !isStoredPassiveArtifactSource(stored.source)) { + throw new PassiveAuthSurfaceArtifactError( + `Artifact ${reference.artifactId} does not have supported stored provenance.`, + ); + } + if (!isArtifactInPassiveTargetScope(stored, input.targetId)) { + throw new PassiveAuthSurfaceArtifactError( + `Artifact ${reference.artifactId} is not bound to target ${input.targetId}.`, + ); + } + return { + artifactId: stored.artifactId, + source: stored.source, + content: stored.text, + }; + }), + ); + + const result = await persistPassiveAuthSurface( + { + projectId: input.projectId, + targetId: input.targetId, + ...(input.threadId ? { threadId: input.threadId } : {}), + ...(input.taskId ? { taskId: input.taskId } : {}), + artifacts, + }, + artifactService, + ); + + return { + ...result, + authorizationId: authorization.id, + sourceArtifacts: artifacts.map((artifact) => ({ + artifactId: artifact.artifactId, + source: artifact.source, + truncated: false as const, + })), + }; +} + +function isStoredPassiveArtifactSource( + value: string, +): value is StoredPassiveArtifactSource { + return (STORED_PASSIVE_ARTIFACT_SOURCES as readonly string[]).includes(value); +} + +function isArtifactInPassiveTargetScope( + artifact: Pick< + Awaited< + ReturnType> + >, + "source" | "targetIds" | "targetScope" + >, + targetId: string, +): boolean { + if (artifact.targetScope === "targets") { + return artifact.targetIds.includes(targetId); + } + return artifact.targetScope === "project" && artifact.source === "reference"; +} + +function assertStoredArtifactRefs( + artifacts: readonly StoredPassiveArtifactRef[], +): void { + if (artifacts.length === 0) { + throw new Error("At least one stored artifact reference is required."); + } + if (artifacts.length > MAX_PASSIVE_AUTH_SOURCE_ARTIFACTS) { + throw new Error( + `At most ${MAX_PASSIVE_AUTH_SOURCE_ARTIFACTS} stored artifact references are allowed.`, + ); + } + if ( + new Set(artifacts.map((artifact) => artifact.artifactId)).size !== + artifacts.length + ) { + throw new Error("Stored artifact references must be unique."); + } +} diff --git a/tests/integration/discovery-artifact-normalizer.test.ts b/tests/integration/discovery-artifact-normalizer.test.ts new file mode 100644 index 000000000..11cd5e9f6 --- /dev/null +++ b/tests/integration/discovery-artifact-normalizer.test.ts @@ -0,0 +1,141 @@ +import { describe, expect, it, vi } from "vitest"; + +import { + normalizeDiscoveryArtifacts, + persistPassiveAuthSurface, +} from "../../src/server/recon"; + +describe("discovery artifact normalization", () => { + it("turns mixed passive evidence into deduplicated, attributable auth-surface signals", () => { + const result = normalizeDiscoveryArtifacts([ + { + artifactId: "artifact-gau", + source: "lab-command", + content: [ + "https://app.example.test/login", + "https://app.example.test/oauth/callback?code=example#fragment", + "https://app.example.test/admin/", + "https://app.example.test/login", + ].join("\n"), + }, + { + artifactId: "artifact-probe", + source: "http-probe", + content: [ + "HTTP/2 401", + "set-cookie: sessionid=[REDACTED]; HttpOnly; SameSite=Lax", + "www-authenticate: Bearer", + "Authentication required", + "HTTP 429 Too Many Requests - rate limited", + "https://app.example.test/api/v2/session", + ].join("\n"), + }, + ]); + + expect(result.rawArtifactIds).toEqual(["artifact-gau", "artifact-probe"]); + expect(result.urls.map((entry) => entry.url)).toEqual([ + "https://app.example.test/admin/", + "https://app.example.test/api/v2/session", + "https://app.example.test/login", + "https://app.example.test/oauth/callback?code=example", + ]); + expect( + result.urls.find((entry) => entry.path === "/login")?.sourceArtifactIds, + ).toEqual(["artifact-gau"]); + expect( + result.authCandidates.map((candidate) => ({ + path: candidate.path, + categories: candidate.categories, + confidence: candidate.confidence, + })), + ).toEqual([ + { path: "/admin/", categories: ["admin"], confidence: "medium" }, + { + path: "/api/v2/session", + categories: ["session"], + confidence: "medium", + }, + { path: "/login", categories: ["login"], confidence: "high" }, + { path: "/oauth/callback", categories: ["oauth"], confidence: "high" }, + ]); + expect(result.responseFamilies).toEqual([ + { + family: "4xx", + statuses: [401, 429], + sourceArtifactIds: ["artifact-probe"], + }, + ]); + expect(result.blockerSignals.map((signal) => signal.reason)).toEqual([ + "auth-required", + "rate-limited", + ]); + expect( + result.blockerSignals.every((signal) => + signal.sourceArtifactIds.includes("artifact-probe"), + ), + ).toBe(true); + expect(result.observations.session).toEqual([ + "https://app.example.test/api/v2/session", + "set-cookie: sessionid=[REDACTED]; HttpOnly; SameSite=Lax", + ]); + expect(result.observations.token).toEqual(["www-authenticate: Bearer"]); + }); + + it("persists a redaction-ready, RAG-indexed summary linked to its raw evidence", async () => { + const createArtifact = vi.fn(async () => ({ + id: "artifact-summary", + projectId: "project-1", + threadId: "thread-1", + name: "passive-auth-surface-target-1.json", + kind: "report", + indexing: { status: "indexed" as const, chunkCount: 1 }, + })); + + const result = await persistPassiveAuthSurface( + { + projectId: "project-1", + threadId: "thread-1", + targetId: "target-1", + taskId: "task-1", + artifacts: [ + { + artifactId: "artifact-raw", + source: "upload", + content: [ + "https://app.example.test/login", + "HTTP 429 Too Many Requests", + ].join("\n"), + }, + ], + }, + { createArtifact } as never, + ); + + expect(result.summary.authRoutes).toEqual([ + { + category: "login", + confidence: "high", + urls: ["https://app.example.test/login"], + evidenceArtifactIds: ["artifact-raw"], + }, + ]); + expect(result.summary.blockers[0]).toMatchObject({ + reason: "rate-limited", + evidenceArtifactIds: ["artifact-raw"], + }); + expect(createArtifact).toHaveBeenCalledWith( + expect.objectContaining({ + projectId: "project-1", + threadId: "thread-1", + targetId: "target-1", + taskId: "task-1", + source: "passive-recon", + indexForRag: true, + metadata: expect.objectContaining({ + rawArtifactIds: ["artifact-raw"], + blockerReasons: ["rate-limited"], + }), + }), + ); + }); +}); diff --git a/tests/integration/secret-scan.test.ts b/tests/integration/secret-scan.test.ts index de30d2809..08519a96e 100644 --- a/tests/integration/secret-scan.test.ts +++ b/tests/integration/secret-scan.test.ts @@ -32,6 +32,9 @@ function fakeArtifactService(): ArtifactServiceInstance & { contentType: "text/plain", sizeBytes: 128, sha256: "abc123", + source: "upload" as const, + targetIds: ["target-1"], + targetScope: "targets" as const, text: `OPENAI_API_KEY=${openAiKey}`, truncated: true, })), diff --git a/tests/integration/stored-passive-auth-surface-api.test.ts b/tests/integration/stored-passive-auth-surface-api.test.ts new file mode 100644 index 000000000..015927e0b --- /dev/null +++ b/tests/integration/stored-passive-auth-surface-api.test.ts @@ -0,0 +1,229 @@ +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { POST } from "../../src/app/api/projects/[projectId]/recon/passive-auth-surface/route"; +import { getProjectStore } from "../../src/server/chat/projectAdapter"; +import { + createArtifactService, + createEvidenceIngestor, + setArtifactService, + setEvidenceIngestor, +} from "../../src/server/evidence"; +import { + createTargetAuthorization, + upsertProjectTarget, +} from "../../src/server/targets"; + +describe("stored passive auth-surface API", () => { + let databaseRoot: string; + let previousDatabaseUrl: string | undefined; + + beforeEach(async () => { + previousDatabaseUrl = process.env.EH_APP_DB_URL; + databaseRoot = await mkdtemp(join(tmpdir(), "passive-auth-surface-api-")); + process.env.EH_APP_DB_URL = `sqlite://${join(databaseRoot, "app.sqlite")}`; + setArtifactService(createArtifactService({ storage: null })); + }); + + afterEach(async () => { + setArtifactService(undefined); + setEvidenceIngestor(undefined); + if (previousDatabaseUrl === undefined) delete process.env.EH_APP_DB_URL; + else process.env.EH_APP_DB_URL = previousDatabaseUrl; + await rm(databaseRoot, { recursive: true, force: true }); + }); + + it("fails closed on authorization and ownership, then persists attributable RAG-ready evidence", async () => { + const indexed = vi.fn(async () => 1); + setEvidenceIngestor( + createEvidenceIngestor({ indexer: { index: indexed } }), + ); + const store = await getProjectStore(); + const project = await store.createProject({ name: "Passive auth map" }); + const otherProject = await store.createProject({ name: "Other evidence" }); + const thread = await store.createThread(project.id, { title: "Map auth" }); + await upsertProjectTarget(project.id, { + id: "target-app", + threadId: thread.id, + kind: "web", + label: "Authorized app", + locator: "https://app.example.test", + }); + await upsertProjectTarget(project.id, { + id: "target-other", + threadId: thread.id, + kind: "web", + label: "Other app", + locator: "https://other.example.test", + }); + + const artifactService = createArtifactService({ storage: null }); + setArtifactService(artifactService); + const source = await artifactService.createArtifact({ + projectId: project.id, + threadId: thread.id, + targetId: "target-app", + name: "passive-urls.txt", + kind: "log", + contentType: "text/plain", + content: + "https://app.example.test/login\nhttps://app.example.test/api/session", + source: "upload", + indexForRag: false, + }); + const foreignSource = await artifactService.createArtifact({ + projectId: otherProject.id, + projectScoped: true, + name: "foreign.txt", + content: "https://other.example.test/admin", + source: "upload", + indexForRag: false, + }); + const otherTargetSource = await artifactService.createArtifact({ + projectId: project.id, + threadId: thread.id, + targetId: "target-other", + name: "other-target.txt", + content: "https://other.example.test/admin", + source: "http-probe", + indexForRag: false, + }); + const projectReference = await artifactService.createArtifact({ + projectId: project.id, + projectScoped: true, + name: "auth-reference.md", + content: "Review https://app.example.test/oauth/callback passively.", + source: "reference", + indexForRag: false, + }); + const projectUpload = await artifactService.createArtifact({ + projectId: project.id, + projectScoped: true, + name: "unbound-upload.txt", + content: "https://app.example.test/admin", + source: "upload", + indexForRag: false, + }); + const rawClientEvidence = await invoke(project.id, { + targetId: "target-app", + threadId: thread.id, + artifacts: [{ artifactId: source.id }], + rawText: "https://untrusted.example.test/admin", + }); + expect(rawClientEvidence.status).toBe(400); + + const unauthorized = await invoke(project.id, { + targetId: "target-app", + threadId: thread.id, + artifacts: [{ artifactId: source.id }], + }); + expect(unauthorized.status).toBe(403); + + const authorization = await createTargetAuthorization(project.id, { + targetId: "target-app", + status: "approved", + grantedBy: "operator", + scope: { activity: "passive" }, + }); + const wrongProject = await invoke(project.id, { + targetId: "target-app", + threadId: thread.id, + artifacts: [{ artifactId: foreignSource.id }], + }); + expect(wrongProject.status).toBe(404); + const wrongTarget = await invoke(project.id, { + targetId: "target-app", + threadId: thread.id, + artifacts: [{ artifactId: otherTargetSource.id }], + }); + expect(wrongTarget.status).toBe(404); + const unboundCollection = await invoke(project.id, { + targetId: "target-app", + threadId: thread.id, + artifacts: [{ artifactId: projectUpload.id }], + }); + expect(unboundCollection.status).toBe(404); + expect(indexed).not.toHaveBeenCalled(); + + const relabeled = await invoke(project.id, { + targetId: "target-app", + threadId: thread.id, + artifacts: [{ artifactId: source.id, source: "reference" }], + }); + expect(relabeled.status).toBe(400); + + const response = await invoke(project.id, { + targetId: "target-app", + threadId: thread.id, + taskId: "task-passive-map", + artifacts: [ + { artifactId: source.id }, + { artifactId: projectReference.id }, + ], + }); + expect(response.status).toBe(200); + const body = (await response.json()) as Record; + expect(body).toMatchObject({ + authorizationId: authorization.id, + summary: { + targetId: "target-app", + rawArtifactIds: [projectReference.id, source.id].sort(), + rawArtifacts: expect.arrayContaining([ + { artifactId: source.id, source: "upload" }, + { artifactId: projectReference.id, source: "reference" }, + ]), + authRoutes: expect.arrayContaining([ + expect.objectContaining({ + category: "login", + evidenceArtifactIds: [source.id], + }), + ]), + }, + artifact: { + projectId: project.id, + threadId: thread.id, + kind: "report", + indexing: { status: "indexed" }, + }, + sourceArtifacts: [ + { artifactId: source.id, source: "upload", truncated: false }, + { + artifactId: projectReference.id, + source: "reference", + truncated: false, + }, + ], + }); + expect(indexed).toHaveBeenCalledWith( + expect.objectContaining({ + metadata: expect.objectContaining({ + projectId: project.id, + threadId: thread.id, + targetId: "target-app", + taskId: "task-passive-map", + source: "passive-recon", + }), + }), + ); + }); +}); + +function invoke(projectId: string, body: unknown) { + return POST( + new Request( + `http://localhost:3210/api/projects/${projectId}/recon/passive-auth-surface`, + { + method: "POST", + headers: { + "content-type": "application/json", + origin: "http://localhost:3210", + }, + body: JSON.stringify(body), + }, + ), + { params: Promise.resolve({ projectId }) }, + ); +}