From f161e534dde3921f6baf131ea2fc9fff4d73cabf Mon Sep 17 00:00:00 2001 From: Dan Levy Date: Wed, 26 Aug 2026 23:06:46 -0400 Subject: [PATCH 1/4] Persist lab network enforcement evidence --- docs/lab-runtime-hardening.md | 6 + src/server/evidence/ingestion.ts | 2 + src/server/labs/index.ts | 1 + src/server/labs/network-evidence.ts | 185 ++++++++++++++++++ src/server/labs/service.ts | 73 +++++++ .../integration/lab-network-evidence.test.ts | 126 ++++++++++++ tests/integration/project-lab.test.ts | 19 +- 7 files changed, 410 insertions(+), 2 deletions(-) create mode 100644 src/server/labs/network-evidence.ts create mode 100644 tests/integration/lab-network-evidence.test.ts diff --git a/docs/lab-runtime-hardening.md b/docs/lab-runtime-hardening.md index 72406af3f..00ee11024 100644 --- a/docs/lab-runtime-hardening.md +++ b/docs/lab-runtime-hardening.md @@ -101,6 +101,12 @@ For the `approved-targets` profile, the iptables script is dynamically built fro Denied packets are rate-limited and logged with the `EXPLOIT_HUNTER_EGRESS_DENIED` prefix. The controller lifecycle, policy fingerprint, and Docker command trace are inspectable forensic evidence; workloads cannot modify the firewall because they do not possess the capability. +### Durable network evidence + +Lab start and restart now save the external controller's enforcement result through the central Artifact service as project-scoped JSONL. Each record uses the versioned `exploit-hunter.lab-network-evidence.v1` schema and can carry project, thread, task, tool-run, research-run, network-profile, and policy-fingerprint correlation. Enforcement failures are recorded as `enforcement-unavailable`; successful policy installation is recorded separately as `policy-enforced` and is not represented as proof that a connection was allowed. + +The same schema reserves `allowed` and `denied` dispositions for observed DNS resolutions and connection attempts. Controller observations must match the active project and policy fingerprint before ingestion. The current controller does not yet emit per-connection records into this path, so lifecycle artifacts must not be treated as a complete network transcript. Opt-in mitmproxy captures remain the available application-level traffic record, with the protocol and bypass limitations described above. + ### Package-egress enforcement For the `package-egress` profile, the iptables script allows traffic only to well-known package registries and distribution mirrors. This covers npm, Yarn, PyPI, RubyGems, crates.io, GitHub release objects, Debian/Ubuntu apt repositories, and Docker Hub. diff --git a/src/server/evidence/ingestion.ts b/src/server/evidence/ingestion.ts index a5554206a..fc778dd99 100644 --- a/src/server/evidence/ingestion.ts +++ b/src/server/evidence/ingestion.ts @@ -23,6 +23,8 @@ export const EVIDENCE_SOURCES = [ "patch-verification", "patch-remediation", "stage-handoff", + "passive-recon", + "network-observation", ] as const; export type EvidenceSource = (typeof EVIDENCE_SOURCES)[number]; diff --git a/src/server/labs/index.ts b/src/server/labs/index.ts index 6160adeec..87fae589c 100644 --- a/src/server/labs/index.ts +++ b/src/server/labs/index.ts @@ -1,5 +1,6 @@ export * from "./docker-plan"; export * from "./hardening"; +export * from "./network-evidence"; export * from "./network-profiles"; export * from "./repository"; export * from "./runtime"; diff --git a/src/server/labs/network-evidence.ts b/src/server/labs/network-evidence.ts new file mode 100644 index 000000000..bd8e53bd2 --- /dev/null +++ b/src/server/labs/network-evidence.ts @@ -0,0 +1,185 @@ +import { type ArtifactServiceInstance, getArtifactService } from "../evidence"; +import type { LabEgressEnforcementResult } from "./types"; + +export const LAB_NETWORK_EVIDENCE_SCHEMA = "exploit-hunter.lab-network-evidence.v1"; + +export type LabNetworkObservationDisposition = "allowed" | "denied" | "enforcement-unavailable" | "policy-enforced"; + +export type LabNetworkObservation = { + observedAt: string; + disposition: LabNetworkObservationDisposition; + event: "connection-attempt" | "dns-resolution" | "enforcement-state"; + destination?: string; + port?: number; + protocol?: "tcp" | "udp"; + hostname?: string; + resolvedAddresses?: string[]; + reason?: string; + source: string; +}; + +export type LabNetworkEvidenceInput = { + projectId: string; + threadId?: string; + taskId?: string; + toolRunId?: string; + researchRunId?: string; + networkProfile: string; + policyId?: string; + observations: LabNetworkObservation[]; +}; + +export interface LabNetworkEvidenceRecorder { + record(input: LabNetworkEvidenceInput): Promise; +} + +type ArtifactWriter = Pick; + +export function buildLabNetworkEvidenceJsonl(input: LabNetworkEvidenceInput): string { + return `${input.observations + .map((observation) => + JSON.stringify({ + schema: LAB_NETWORK_EVIDENCE_SCHEMA, + projectId: input.projectId, + threadId: input.threadId, + taskId: input.taskId, + toolRunId: input.toolRunId, + researchRunId: input.researchRunId, + networkProfile: input.networkProfile, + policyId: input.policyId, + ...observation, + }), + ) + .join("\n")}\n`; +} + +export function createArtifactLabNetworkEvidenceRecorder( + writer: ArtifactWriter = getArtifactService(), +): LabNetworkEvidenceRecorder { + return { + async record(input) { + if (input.observations.length === 0) return; + const content = buildLabNetworkEvidenceJsonl(input); + await writer.createArtifact({ + projectId: input.projectId, + ...(input.threadId ? { threadId: input.threadId } : {}), + ...(input.taskId ? { taskId: input.taskId } : {}), + ...(input.toolRunId ? { toolRunId: input.toolRunId } : {}), + projectScoped: true, + name: `network-evidence-${input.policyId?.slice(0, 16) ?? "unavailable"}.jsonl`, + kind: "log", + contentType: "application/x-ndjson", + content, + agentGenerated: true, + source: "network-observation", + indexForRag: false, + metadata: { + schema: LAB_NETWORK_EVIDENCE_SCHEMA, + source: "lab-network-observation", + networkProfile: input.networkProfile, + policyId: input.policyId, + researchRunId: input.researchRunId, + observationCount: input.observations.length, + dispositions: [...new Set(input.observations.map((event) => event.disposition))], + }, + }); + }, + }; +} + +export function enforcementObservation( + enforcement: LabEgressEnforcementResult, + observedAt = new Date().toISOString(), +): LabNetworkObservation { + const unavailable = + enforcement.disposition === "unenforced-development" || + enforcement.disposition === "enforcement-failed-safe" || + enforcement.disposition === "enforcement-cleanup-failed"; + return { + observedAt, + disposition: unavailable ? "enforcement-unavailable" : "policy-enforced", + event: "enforcement-state", + reason: enforcement.failure ?? enforcement.remediation, + source: "external-egress-controller", + }; +} + +export function parseLabNetworkObservation( + value: unknown, + expected: Pick, +): LabNetworkObservation { + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new Error("Network observation must be a JSON object."); + } + const record = value as Record; + if (record.schema !== LAB_NETWORK_EVIDENCE_SCHEMA) { + throw new Error("Network observation schema is missing or unsupported."); + } + if (record.projectId !== expected.projectId || record.policyId !== expected.policyId) { + throw new Error("Network observation does not match the active project and policy."); + } + const event = readEnum(record.event, ["connection-attempt", "dns-resolution", "enforcement-state"] as const); + const disposition = readEnum(record.disposition, [ + "allowed", + "denied", + "enforcement-unavailable", + "policy-enforced", + ] as const); + const observedAt = readRequiredText(record.observedAt, "observedAt"); + const source = readRequiredText(record.source, "source"); + const port = record.port; + if (port !== undefined && (!Number.isInteger(port) || Number(port) < 1 || Number(port) > 65_535)) { + throw new Error("Network observation port must be an integer from 1 through 65535."); + } + const resolvedAddresses = record.resolvedAddresses; + if ( + resolvedAddresses !== undefined && + (!Array.isArray(resolvedAddresses) || resolvedAddresses.some((item) => typeof item !== "string")) + ) { + throw new Error("Network observation resolvedAddresses must contain strings."); + } + if ( + event === "connection-attempt" && + (typeof record.destination !== "string" || typeof port !== "number" || !record.protocol) + ) { + throw new Error("Connection observations require destination, port, and protocol."); + } + if ( + event === "dns-resolution" && + (typeof record.hostname !== "string" || !Array.isArray(resolvedAddresses)) + ) { + throw new Error("DNS observations require a hostname and resolved addresses."); + } + if ( + (event === "enforcement-state") !== + (disposition === "policy-enforced" || disposition === "enforcement-unavailable") + ) { + throw new Error("Network observation event and disposition are inconsistent."); + } + return { + observedAt, + event, + disposition, + source, + ...(typeof record.destination === "string" ? { destination: record.destination } : {}), + ...(typeof port === "number" ? { port } : {}), + ...(record.protocol === "tcp" || record.protocol === "udp" ? { protocol: record.protocol } : {}), + ...(typeof record.hostname === "string" ? { hostname: record.hostname } : {}), + ...(Array.isArray(resolvedAddresses) ? { resolvedAddresses: resolvedAddresses as string[] } : {}), + ...(typeof record.reason === "string" ? { reason: record.reason } : {}), + }; +} + +function readRequiredText(value: unknown, field: string): string { + if (typeof value !== "string" || !value.trim()) { + throw new Error(`Network observation ${field} must be a non-empty string.`); + } + return value; +} + +function readEnum(value: unknown, allowed: T): T[number] { + if (typeof value !== "string" || !allowed.includes(value)) { + throw new Error(`Unsupported network observation value: ${String(value)}.`); + } + return value as T[number]; +} diff --git a/src/server/labs/service.ts b/src/server/labs/service.ts index 4366b59e1..36e745f15 100644 --- a/src/server/labs/service.ts +++ b/src/server/labs/service.ts @@ -5,6 +5,11 @@ import { resolveLabIsolationMode, UPSTREAM_KALI_LAB_IMAGE, } from "./hardening"; +import { + createArtifactLabNetworkEvidenceRecorder, + enforcementObservation, + type LabNetworkEvidenceRecorder, +} from "./network-evidence"; import { buildUfwCommandPlan, normalizeLabFirewallConfig } from "./network-profiles"; import { sanitizeJsonObject, withProjectLabRepository } from "./repository"; import { LabEgressEnforcementError, ProjectLabRuntime } from "./runtime"; @@ -50,6 +55,8 @@ export class ProjectLabService { (process.env.PROJECT_LAB_RUNTIME_MODE as "auto" | "docker" | "dry-run" | undefined) ?? "auto", }), + private readonly networkEvidence: LabNetworkEvidenceRecorder = + createArtifactLabNetworkEvidenceRecorder(), ) {} async status(projectId: string): Promise { @@ -201,6 +208,13 @@ export class ProjectLabService { ), }); + await this.recordNetworkEnforcementEvidence({ + projectId, + threadId: requestedThreadId ?? readActiveThreadId(claimed), + networkProfile, + result: runtimeResult, + }); + return this.statusFromLab(projectId, running); } catch (error) { const failure = serializeLabFailure(error); @@ -231,6 +245,15 @@ export class ProjectLabService { }) : claimed.metadata, }); + if (error instanceof LabEgressEnforcementError) { + await this.recordNetworkEnforcementEvidence({ + projectId, + threadId: requestedThreadId ?? readActiveThreadId(claimed), + networkProfile, + enforcement: error.enforcement, + policyId: readEgressPolicyIdFromCommands(error.commands), + }); + } throw error; } } @@ -338,6 +361,15 @@ export class ProjectLabService { }) : lab.metadata, }); + if (error instanceof LabEgressEnforcementError) { + await this.recordNetworkEnforcementEvidence({ + projectId, + threadId: cleanText(input.threadId) ?? readActiveThreadId(lab), + networkProfile, + enforcement: error.enforcement, + policyId: readEgressPolicyIdFromCommands(error.commands), + }); + } throw error; } const runtimeResult = mergeRuntimeResults(destroyResult, startResult); @@ -366,9 +398,41 @@ export class ProjectLabService { ), }); + await this.recordNetworkEnforcementEvidence({ + projectId, + threadId: cleanText(input.threadId) ?? readActiveThreadId(lab), + networkProfile, + result: startResult, + }); + return this.statusFromLab(projectId, running); } + private async recordNetworkEnforcementEvidence(input: { + projectId: string; + threadId?: string; + networkProfile: HumanLabNetworkProfileId; + result?: LabRuntimeResult; + enforcement?: LabEgressEnforcementResult; + policyId?: string; + }) { + const enforcement = input.result?.egressEnforcement ?? input.enforcement; + if (!enforcement || enforcement.disposition === "not-required") return; + const policyId = + input.policyId ?? input.result?.container.labels["exploit-hunter.egress-policy-sha256"]; + try { + await this.networkEvidence.record({ + projectId: input.projectId, + threadId: input.threadId, + networkProfile: input.networkProfile, + policyId, + observations: [enforcementObservation(enforcement)], + }); + } catch (error) { + console.error("[labs] Failed to persist network enforcement evidence.", error); + } + } + async applyFirewall( projectId: string, input: LabCreateInput = {}, @@ -688,6 +752,15 @@ function timestamp(): string { return new Date().toISOString(); } +function readEgressPolicyIdFromCommands(commands: Array<{ args: string[] }>): string | undefined { + const prefix = "exploit-hunter.egress-policy-sha256="; + for (const command of commands) { + const label = command.args.find((arg) => arg.startsWith(prefix)); + if (label) return label.slice(prefix.length); + } + return undefined; +} + function readLabEgressEnforcement(value: unknown): LabEgressEnforcementResult | undefined { if (!value || typeof value !== "object" || Array.isArray(value)) return undefined; const record = value as Record; diff --git a/tests/integration/lab-network-evidence.test.ts b/tests/integration/lab-network-evidence.test.ts new file mode 100644 index 000000000..7225608b3 --- /dev/null +++ b/tests/integration/lab-network-evidence.test.ts @@ -0,0 +1,126 @@ +import { describe, expect, it, vi } from "vitest"; + +import type { ArtifactServiceInstance, CreateArtifactInput } from "../../src/server/evidence"; +import { + buildLabNetworkEvidenceJsonl, + createArtifactLabNetworkEvidenceRecorder, + LAB_NETWORK_EVIDENCE_SCHEMA, + parseLabNetworkObservation, +} from "../../src/server/labs/network-evidence"; + +describe("lab network evidence", () => { + it("persists correlated observations through the central artifact service", async () => { + const createArtifact = vi.fn(async (_input: CreateArtifactInput) => ({ + id: "artifact-network-1", + projectId: "project-1", + threadId: "thread-1", + name: "network-evidence-policy-1.jsonl", + kind: "log", + indexing: { status: "not_attempted" as const, reason: "disabled" }, + })); + const writer = { + createArtifact, + createFinding: vi.fn(), + } satisfies ArtifactServiceInstance; + const recorder = createArtifactLabNetworkEvidenceRecorder(writer); + + await recorder.record({ + projectId: "project-1", + threadId: "thread-1", + taskId: "task-1", + toolRunId: "tool-run-1", + researchRunId: "research-run-1", + networkProfile: "approved-targets", + policyId: "policy-1", + observations: [ + { + observedAt: "2026-08-26T12:00:00.000Z", + event: "connection-attempt", + disposition: "denied", + destination: "203.0.113.8", + port: 443, + protocol: "tcp", + source: "external-egress-controller", + }, + ], + }); + + expect(createArtifact).toHaveBeenCalledOnce(); + expect(createArtifact).toHaveBeenCalledWith( + expect.objectContaining({ + projectId: "project-1", + threadId: "thread-1", + taskId: "task-1", + toolRunId: "tool-run-1", + source: "network-observation", + indexForRag: false, + metadata: expect.objectContaining({ + policyId: "policy-1", + researchRunId: "research-run-1", + dispositions: ["denied"], + }), + }), + ); + const artifactInput = createArtifact.mock.calls[0]?.[0]; + expect(JSON.parse(String(artifactInput?.content).trim())).toMatchObject({ + schema: LAB_NETWORK_EVIDENCE_SCHEMA, + projectId: "project-1", + threadId: "thread-1", + toolRunId: "tool-run-1", + policyId: "policy-1", + disposition: "denied", + destination: "203.0.113.8", + port: 443, + }); + }); + + it("rejects controller observations from another project or policy", () => { + const line = buildLabNetworkEvidenceJsonl({ + projectId: "project-1", + networkProfile: "approved-targets", + policyId: "policy-1", + observations: [ + { + observedAt: "2026-08-26T12:00:00.000Z", + event: "dns-resolution", + disposition: "allowed", + hostname: "target.example", + resolvedAddresses: ["192.0.2.10"], + source: "external-egress-controller", + }, + ], + }); + const parsed = JSON.parse(line); + + expect(() => + parseLabNetworkObservation(parsed, { + projectId: "project-2", + policyId: "policy-1", + }), + ).toThrow(/active project and policy/); + expect(() => + parseLabNetworkObservation(parsed, { + projectId: "project-1", + policyId: "policy-2", + }), + ).toThrow(/active project and policy/); + }); + + it("rejects incomplete connection observations", () => { + expect(() => + parseLabNetworkObservation( + { + schema: LAB_NETWORK_EVIDENCE_SCHEMA, + projectId: "project-1", + policyId: "policy-1", + observedAt: "2026-08-26T12:00:00.000Z", + event: "connection-attempt", + disposition: "allowed", + destination: "192.0.2.10", + source: "external-egress-controller", + }, + { projectId: "project-1", policyId: "policy-1" }, + ), + ).toThrow(/destination, port, and protocol/); + }); +}); diff --git a/tests/integration/project-lab.test.ts b/tests/integration/project-lab.test.ts index 6c7e0c45a..b68abd6e8 100644 --- a/tests/integration/project-lab.test.ts +++ b/tests/integration/project-lab.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; import type { JsonObject, ProjectLabRow } from "../../src/server/db/types"; import type { @@ -330,7 +330,10 @@ describe("project lab lifecycle", () => { egressEnforcementMode: "development", runner: egressFailureRunner(), }); - const service = new ProjectLabService(repository, runtime); + const recordNetworkEvidence = vi.fn(); + const service = new ProjectLabService(repository, runtime, { + record: recordNetworkEvidence, + }); const started = await service.start("project-1", { networkProfile: "package-egress" }); const reloaded = await service.status("project-1"); @@ -343,6 +346,18 @@ describe("project lab lifecycle", () => { effectiveUnrestricted: true, workloadRunning: true, }); + expect(recordNetworkEvidence).toHaveBeenCalledWith( + expect.objectContaining({ + projectId: "project-1", + networkProfile: "package-egress", + observations: [ + expect.objectContaining({ + event: "enforcement-state", + disposition: "enforcement-unavailable", + }), + ], + }), + ); }); it("persists managed fail-safe cleanup state after enforcement failure", async () => { From 27b802183d24ae70dce918a053653f0fdac09d0c Mon Sep 17 00:00:00 2001 From: Dan Levy Date: Thu, 27 Aug 2026 00:31:46 -0400 Subject: [PATCH 2/4] Format lab network evidence changes --- src/server/labs/network-evidence.ts | 348 ++++++++++-------- src/server/labs/service.ts | 3 +- .../integration/lab-network-evidence.test.ts | 231 ++++++------ 3 files changed, 315 insertions(+), 267 deletions(-) diff --git a/src/server/labs/network-evidence.ts b/src/server/labs/network-evidence.ts index bd8e53bd2..d46984e7b 100644 --- a/src/server/labs/network-evidence.ts +++ b/src/server/labs/network-evidence.ts @@ -1,185 +1,231 @@ import { type ArtifactServiceInstance, getArtifactService } from "../evidence"; import type { LabEgressEnforcementResult } from "./types"; -export const LAB_NETWORK_EVIDENCE_SCHEMA = "exploit-hunter.lab-network-evidence.v1"; +export const LAB_NETWORK_EVIDENCE_SCHEMA = + "exploit-hunter.lab-network-evidence.v1"; -export type LabNetworkObservationDisposition = "allowed" | "denied" | "enforcement-unavailable" | "policy-enforced"; +export type LabNetworkObservationDisposition = + | "allowed" + | "denied" + | "enforcement-unavailable" + | "policy-enforced"; export type LabNetworkObservation = { - observedAt: string; - disposition: LabNetworkObservationDisposition; - event: "connection-attempt" | "dns-resolution" | "enforcement-state"; - destination?: string; - port?: number; - protocol?: "tcp" | "udp"; - hostname?: string; - resolvedAddresses?: string[]; - reason?: string; - source: string; + observedAt: string; + disposition: LabNetworkObservationDisposition; + event: "connection-attempt" | "dns-resolution" | "enforcement-state"; + destination?: string; + port?: number; + protocol?: "tcp" | "udp"; + hostname?: string; + resolvedAddresses?: string[]; + reason?: string; + source: string; }; export type LabNetworkEvidenceInput = { - projectId: string; - threadId?: string; - taskId?: string; - toolRunId?: string; - researchRunId?: string; - networkProfile: string; - policyId?: string; - observations: LabNetworkObservation[]; + projectId: string; + threadId?: string; + taskId?: string; + toolRunId?: string; + researchRunId?: string; + networkProfile: string; + policyId?: string; + observations: LabNetworkObservation[]; }; export interface LabNetworkEvidenceRecorder { - record(input: LabNetworkEvidenceInput): Promise; + record(input: LabNetworkEvidenceInput): Promise; } type ArtifactWriter = Pick; -export function buildLabNetworkEvidenceJsonl(input: LabNetworkEvidenceInput): string { - return `${input.observations - .map((observation) => - JSON.stringify({ - schema: LAB_NETWORK_EVIDENCE_SCHEMA, - projectId: input.projectId, - threadId: input.threadId, - taskId: input.taskId, - toolRunId: input.toolRunId, - researchRunId: input.researchRunId, - networkProfile: input.networkProfile, - policyId: input.policyId, - ...observation, - }), - ) - .join("\n")}\n`; +export function buildLabNetworkEvidenceJsonl( + input: LabNetworkEvidenceInput, +): string { + return `${input.observations + .map((observation) => + JSON.stringify({ + schema: LAB_NETWORK_EVIDENCE_SCHEMA, + projectId: input.projectId, + threadId: input.threadId, + taskId: input.taskId, + toolRunId: input.toolRunId, + researchRunId: input.researchRunId, + networkProfile: input.networkProfile, + policyId: input.policyId, + ...observation, + }), + ) + .join("\n")}\n`; } export function createArtifactLabNetworkEvidenceRecorder( - writer: ArtifactWriter = getArtifactService(), + writer: ArtifactWriter = getArtifactService(), ): LabNetworkEvidenceRecorder { - return { - async record(input) { - if (input.observations.length === 0) return; - const content = buildLabNetworkEvidenceJsonl(input); - await writer.createArtifact({ - projectId: input.projectId, - ...(input.threadId ? { threadId: input.threadId } : {}), - ...(input.taskId ? { taskId: input.taskId } : {}), - ...(input.toolRunId ? { toolRunId: input.toolRunId } : {}), - projectScoped: true, - name: `network-evidence-${input.policyId?.slice(0, 16) ?? "unavailable"}.jsonl`, - kind: "log", - contentType: "application/x-ndjson", - content, - agentGenerated: true, - source: "network-observation", - indexForRag: false, - metadata: { - schema: LAB_NETWORK_EVIDENCE_SCHEMA, - source: "lab-network-observation", - networkProfile: input.networkProfile, - policyId: input.policyId, - researchRunId: input.researchRunId, - observationCount: input.observations.length, - dispositions: [...new Set(input.observations.map((event) => event.disposition))], - }, - }); - }, - }; + return { + async record(input) { + if (input.observations.length === 0) return; + const content = buildLabNetworkEvidenceJsonl(input); + await writer.createArtifact({ + projectId: input.projectId, + ...(input.threadId ? { threadId: input.threadId } : {}), + ...(input.taskId ? { taskId: input.taskId } : {}), + ...(input.toolRunId ? { toolRunId: input.toolRunId } : {}), + projectScoped: true, + name: `network-evidence-${input.policyId?.slice(0, 16) ?? "unavailable"}.jsonl`, + kind: "log", + contentType: "application/x-ndjson", + content, + agentGenerated: true, + source: "network-observation", + indexForRag: false, + metadata: { + schema: LAB_NETWORK_EVIDENCE_SCHEMA, + source: "lab-network-observation", + networkProfile: input.networkProfile, + policyId: input.policyId, + researchRunId: input.researchRunId, + observationCount: input.observations.length, + dispositions: [ + ...new Set(input.observations.map((event) => event.disposition)), + ], + }, + }); + }, + }; } export function enforcementObservation( - enforcement: LabEgressEnforcementResult, - observedAt = new Date().toISOString(), + enforcement: LabEgressEnforcementResult, + observedAt = new Date().toISOString(), ): LabNetworkObservation { - const unavailable = - enforcement.disposition === "unenforced-development" || - enforcement.disposition === "enforcement-failed-safe" || - enforcement.disposition === "enforcement-cleanup-failed"; - return { - observedAt, - disposition: unavailable ? "enforcement-unavailable" : "policy-enforced", - event: "enforcement-state", - reason: enforcement.failure ?? enforcement.remediation, - source: "external-egress-controller", - }; + const unavailable = + enforcement.disposition === "unenforced-development" || + enforcement.disposition === "enforcement-failed-safe" || + enforcement.disposition === "enforcement-cleanup-failed"; + return { + observedAt, + disposition: unavailable ? "enforcement-unavailable" : "policy-enforced", + event: "enforcement-state", + reason: enforcement.failure ?? enforcement.remediation, + source: "external-egress-controller", + }; } export function parseLabNetworkObservation( - value: unknown, - expected: Pick, + value: unknown, + expected: Pick, ): LabNetworkObservation { - if (!value || typeof value !== "object" || Array.isArray(value)) { - throw new Error("Network observation must be a JSON object."); - } - const record = value as Record; - if (record.schema !== LAB_NETWORK_EVIDENCE_SCHEMA) { - throw new Error("Network observation schema is missing or unsupported."); - } - if (record.projectId !== expected.projectId || record.policyId !== expected.policyId) { - throw new Error("Network observation does not match the active project and policy."); - } - const event = readEnum(record.event, ["connection-attempt", "dns-resolution", "enforcement-state"] as const); - const disposition = readEnum(record.disposition, [ - "allowed", - "denied", - "enforcement-unavailable", - "policy-enforced", - ] as const); - const observedAt = readRequiredText(record.observedAt, "observedAt"); - const source = readRequiredText(record.source, "source"); - const port = record.port; - if (port !== undefined && (!Number.isInteger(port) || Number(port) < 1 || Number(port) > 65_535)) { - throw new Error("Network observation port must be an integer from 1 through 65535."); - } - const resolvedAddresses = record.resolvedAddresses; - if ( - resolvedAddresses !== undefined && - (!Array.isArray(resolvedAddresses) || resolvedAddresses.some((item) => typeof item !== "string")) - ) { - throw new Error("Network observation resolvedAddresses must contain strings."); - } - if ( - event === "connection-attempt" && - (typeof record.destination !== "string" || typeof port !== "number" || !record.protocol) - ) { - throw new Error("Connection observations require destination, port, and protocol."); - } - if ( - event === "dns-resolution" && - (typeof record.hostname !== "string" || !Array.isArray(resolvedAddresses)) - ) { - throw new Error("DNS observations require a hostname and resolved addresses."); - } - if ( - (event === "enforcement-state") !== - (disposition === "policy-enforced" || disposition === "enforcement-unavailable") - ) { - throw new Error("Network observation event and disposition are inconsistent."); - } - return { - observedAt, - event, - disposition, - source, - ...(typeof record.destination === "string" ? { destination: record.destination } : {}), - ...(typeof port === "number" ? { port } : {}), - ...(record.protocol === "tcp" || record.protocol === "udp" ? { protocol: record.protocol } : {}), - ...(typeof record.hostname === "string" ? { hostname: record.hostname } : {}), - ...(Array.isArray(resolvedAddresses) ? { resolvedAddresses: resolvedAddresses as string[] } : {}), - ...(typeof record.reason === "string" ? { reason: record.reason } : {}), - }; + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new Error("Network observation must be a JSON object."); + } + const record = value as Record; + if (record.schema !== LAB_NETWORK_EVIDENCE_SCHEMA) { + throw new Error("Network observation schema is missing or unsupported."); + } + if ( + record.projectId !== expected.projectId || + record.policyId !== expected.policyId + ) { + throw new Error( + "Network observation does not match the active project and policy.", + ); + } + const event = readEnum(record.event, [ + "connection-attempt", + "dns-resolution", + "enforcement-state", + ] as const); + const disposition = readEnum(record.disposition, [ + "allowed", + "denied", + "enforcement-unavailable", + "policy-enforced", + ] as const); + const observedAt = readRequiredText(record.observedAt, "observedAt"); + const source = readRequiredText(record.source, "source"); + const port = record.port; + if ( + port !== undefined && + (!Number.isInteger(port) || Number(port) < 1 || Number(port) > 65_535) + ) { + throw new Error( + "Network observation port must be an integer from 1 through 65535.", + ); + } + const resolvedAddresses = record.resolvedAddresses; + if ( + resolvedAddresses !== undefined && + (!Array.isArray(resolvedAddresses) || + resolvedAddresses.some((item) => typeof item !== "string")) + ) { + throw new Error( + "Network observation resolvedAddresses must contain strings.", + ); + } + if ( + event === "connection-attempt" && + (typeof record.destination !== "string" || + typeof port !== "number" || + !record.protocol) + ) { + throw new Error( + "Connection observations require destination, port, and protocol.", + ); + } + if ( + event === "dns-resolution" && + (typeof record.hostname !== "string" || !Array.isArray(resolvedAddresses)) + ) { + throw new Error( + "DNS observations require a hostname and resolved addresses.", + ); + } + if ( + (event === "enforcement-state") !== + (disposition === "policy-enforced" || + disposition === "enforcement-unavailable") + ) { + throw new Error( + "Network observation event and disposition are inconsistent.", + ); + } + return { + observedAt, + event, + disposition, + source, + ...(typeof record.destination === "string" + ? { destination: record.destination } + : {}), + ...(typeof port === "number" ? { port } : {}), + ...(record.protocol === "tcp" || record.protocol === "udp" + ? { protocol: record.protocol } + : {}), + ...(typeof record.hostname === "string" + ? { hostname: record.hostname } + : {}), + ...(Array.isArray(resolvedAddresses) + ? { resolvedAddresses: resolvedAddresses as string[] } + : {}), + ...(typeof record.reason === "string" ? { reason: record.reason } : {}), + }; } function readRequiredText(value: unknown, field: string): string { - if (typeof value !== "string" || !value.trim()) { - throw new Error(`Network observation ${field} must be a non-empty string.`); - } - return value; + if (typeof value !== "string" || !value.trim()) { + throw new Error(`Network observation ${field} must be a non-empty string.`); + } + return value; } -function readEnum(value: unknown, allowed: T): T[number] { - if (typeof value !== "string" || !allowed.includes(value)) { - throw new Error(`Unsupported network observation value: ${String(value)}.`); - } - return value as T[number]; +function readEnum( + value: unknown, + allowed: T, +): T[number] { + if (typeof value !== "string" || !allowed.includes(value)) { + throw new Error(`Unsupported network observation value: ${String(value)}.`); + } + return value as T[number]; } diff --git a/src/server/labs/service.ts b/src/server/labs/service.ts index 36e745f15..5ae74c8eb 100644 --- a/src/server/labs/service.ts +++ b/src/server/labs/service.ts @@ -55,8 +55,7 @@ export class ProjectLabService { (process.env.PROJECT_LAB_RUNTIME_MODE as "auto" | "docker" | "dry-run" | undefined) ?? "auto", }), - private readonly networkEvidence: LabNetworkEvidenceRecorder = - createArtifactLabNetworkEvidenceRecorder(), + private readonly networkEvidence: LabNetworkEvidenceRecorder = createArtifactLabNetworkEvidenceRecorder(), ) {} async status(projectId: string): Promise { diff --git a/tests/integration/lab-network-evidence.test.ts b/tests/integration/lab-network-evidence.test.ts index 7225608b3..bd4e10b78 100644 --- a/tests/integration/lab-network-evidence.test.ts +++ b/tests/integration/lab-network-evidence.test.ts @@ -1,126 +1,129 @@ import { describe, expect, it, vi } from "vitest"; -import type { ArtifactServiceInstance, CreateArtifactInput } from "../../src/server/evidence"; +import type { + ArtifactServiceInstance, + CreateArtifactInput, +} from "../../src/server/evidence"; import { - buildLabNetworkEvidenceJsonl, - createArtifactLabNetworkEvidenceRecorder, - LAB_NETWORK_EVIDENCE_SCHEMA, - parseLabNetworkObservation, + buildLabNetworkEvidenceJsonl, + createArtifactLabNetworkEvidenceRecorder, + LAB_NETWORK_EVIDENCE_SCHEMA, + parseLabNetworkObservation, } from "../../src/server/labs/network-evidence"; describe("lab network evidence", () => { - it("persists correlated observations through the central artifact service", async () => { - const createArtifact = vi.fn(async (_input: CreateArtifactInput) => ({ - id: "artifact-network-1", - projectId: "project-1", - threadId: "thread-1", - name: "network-evidence-policy-1.jsonl", - kind: "log", - indexing: { status: "not_attempted" as const, reason: "disabled" }, - })); - const writer = { - createArtifact, - createFinding: vi.fn(), - } satisfies ArtifactServiceInstance; - const recorder = createArtifactLabNetworkEvidenceRecorder(writer); + it("persists correlated observations through the central artifact service", async () => { + const createArtifact = vi.fn(async (_input: CreateArtifactInput) => ({ + id: "artifact-network-1", + projectId: "project-1", + threadId: "thread-1", + name: "network-evidence-policy-1.jsonl", + kind: "log", + indexing: { status: "not_attempted" as const, reason: "disabled" }, + })); + const writer = { + createArtifact, + createFinding: vi.fn(), + } satisfies ArtifactServiceInstance; + const recorder = createArtifactLabNetworkEvidenceRecorder(writer); - await recorder.record({ - projectId: "project-1", - threadId: "thread-1", - taskId: "task-1", - toolRunId: "tool-run-1", - researchRunId: "research-run-1", - networkProfile: "approved-targets", - policyId: "policy-1", - observations: [ - { - observedAt: "2026-08-26T12:00:00.000Z", - event: "connection-attempt", - disposition: "denied", - destination: "203.0.113.8", - port: 443, - protocol: "tcp", - source: "external-egress-controller", - }, - ], - }); + await recorder.record({ + projectId: "project-1", + threadId: "thread-1", + taskId: "task-1", + toolRunId: "tool-run-1", + researchRunId: "research-run-1", + networkProfile: "approved-targets", + policyId: "policy-1", + observations: [ + { + observedAt: "2026-08-26T12:00:00.000Z", + event: "connection-attempt", + disposition: "denied", + destination: "203.0.113.8", + port: 443, + protocol: "tcp", + source: "external-egress-controller", + }, + ], + }); - expect(createArtifact).toHaveBeenCalledOnce(); - expect(createArtifact).toHaveBeenCalledWith( - expect.objectContaining({ - projectId: "project-1", - threadId: "thread-1", - taskId: "task-1", - toolRunId: "tool-run-1", - source: "network-observation", - indexForRag: false, - metadata: expect.objectContaining({ - policyId: "policy-1", - researchRunId: "research-run-1", - dispositions: ["denied"], - }), - }), - ); - const artifactInput = createArtifact.mock.calls[0]?.[0]; - expect(JSON.parse(String(artifactInput?.content).trim())).toMatchObject({ - schema: LAB_NETWORK_EVIDENCE_SCHEMA, - projectId: "project-1", - threadId: "thread-1", - toolRunId: "tool-run-1", - policyId: "policy-1", - disposition: "denied", - destination: "203.0.113.8", - port: 443, - }); - }); + expect(createArtifact).toHaveBeenCalledOnce(); + expect(createArtifact).toHaveBeenCalledWith( + expect.objectContaining({ + projectId: "project-1", + threadId: "thread-1", + taskId: "task-1", + toolRunId: "tool-run-1", + source: "network-observation", + indexForRag: false, + metadata: expect.objectContaining({ + policyId: "policy-1", + researchRunId: "research-run-1", + dispositions: ["denied"], + }), + }), + ); + const artifactInput = createArtifact.mock.calls[0]?.[0]; + expect(JSON.parse(String(artifactInput?.content).trim())).toMatchObject({ + schema: LAB_NETWORK_EVIDENCE_SCHEMA, + projectId: "project-1", + threadId: "thread-1", + toolRunId: "tool-run-1", + policyId: "policy-1", + disposition: "denied", + destination: "203.0.113.8", + port: 443, + }); + }); - it("rejects controller observations from another project or policy", () => { - const line = buildLabNetworkEvidenceJsonl({ - projectId: "project-1", - networkProfile: "approved-targets", - policyId: "policy-1", - observations: [ - { - observedAt: "2026-08-26T12:00:00.000Z", - event: "dns-resolution", - disposition: "allowed", - hostname: "target.example", - resolvedAddresses: ["192.0.2.10"], - source: "external-egress-controller", - }, - ], - }); - const parsed = JSON.parse(line); + it("rejects controller observations from another project or policy", () => { + const line = buildLabNetworkEvidenceJsonl({ + projectId: "project-1", + networkProfile: "approved-targets", + policyId: "policy-1", + observations: [ + { + observedAt: "2026-08-26T12:00:00.000Z", + event: "dns-resolution", + disposition: "allowed", + hostname: "target.example", + resolvedAddresses: ["192.0.2.10"], + source: "external-egress-controller", + }, + ], + }); + const parsed = JSON.parse(line); - expect(() => - parseLabNetworkObservation(parsed, { - projectId: "project-2", - policyId: "policy-1", - }), - ).toThrow(/active project and policy/); - expect(() => - parseLabNetworkObservation(parsed, { - projectId: "project-1", - policyId: "policy-2", - }), - ).toThrow(/active project and policy/); - }); + expect(() => + parseLabNetworkObservation(parsed, { + projectId: "project-2", + policyId: "policy-1", + }), + ).toThrow(/active project and policy/); + expect(() => + parseLabNetworkObservation(parsed, { + projectId: "project-1", + policyId: "policy-2", + }), + ).toThrow(/active project and policy/); + }); - it("rejects incomplete connection observations", () => { - expect(() => - parseLabNetworkObservation( - { - schema: LAB_NETWORK_EVIDENCE_SCHEMA, - projectId: "project-1", - policyId: "policy-1", - observedAt: "2026-08-26T12:00:00.000Z", - event: "connection-attempt", - disposition: "allowed", - destination: "192.0.2.10", - source: "external-egress-controller", - }, - { projectId: "project-1", policyId: "policy-1" }, - ), - ).toThrow(/destination, port, and protocol/); - }); + it("rejects incomplete connection observations", () => { + expect(() => + parseLabNetworkObservation( + { + schema: LAB_NETWORK_EVIDENCE_SCHEMA, + projectId: "project-1", + policyId: "policy-1", + observedAt: "2026-08-26T12:00:00.000Z", + event: "connection-attempt", + disposition: "allowed", + destination: "192.0.2.10", + source: "external-egress-controller", + }, + { projectId: "project-1", policyId: "policy-1" }, + ), + ).toThrow(/destination, port, and protocol/); + }); }); From 15a4cbc3a45e939f4a30c4540848bfde665c344e Mon Sep 17 00:00:00 2001 From: Dan Levy Date: Sun, 30 Aug 2026 18:13:26 -0400 Subject: [PATCH 3/4] fix(labs): fail closed around egress evidence --- docs/lab-runtime-hardening.md | 4 +- src/lib/ids.ts | 2 + src/mastra/tools/agent-lab-command.ts | 29 +- src/server/agent-lab/command-runner.ts | 66 ++- ...260830180000_network_evidence_receipts.sql | 22 + src/server/db/postgres-migrate.ts | 1 + src/server/labs/docker-plan.ts | 17 +- src/server/labs/network-evidence.ts | 466 ++++++++++-------- src/server/labs/network-profiles.ts | 89 +++- src/server/labs/runtime.ts | 68 +-- src/server/labs/service.ts | 201 +++++--- src/server/labs/types.ts | 4 + tests/integration/agent-lab-command.test.ts | 3 + .../integration/lab-network-evidence.test.ts | 332 ++++++++----- tests/integration/lab-runtime.test.ts | 84 ++-- tests/integration/project-lab.test.ts | 75 ++- .../live/lab-egress-enforcement-mode.test.ts | 116 ++++- 17 files changed, 1079 insertions(+), 500 deletions(-) create mode 100644 src/server/db/migrations/20260830180000_network_evidence_receipts.sql diff --git a/docs/lab-runtime-hardening.md b/docs/lab-runtime-hardening.md index 00ee11024..3eb6a81ec 100644 --- a/docs/lab-runtime-hardening.md +++ b/docs/lab-runtime-hardening.md @@ -103,9 +103,9 @@ Denied packets are rate-limited and logged with the `EXPLOIT_HUNTER_EGRESS_DENIE ### Durable network evidence -Lab start and restart now save the external controller's enforcement result through the central Artifact service as project-scoped JSONL. Each record uses the versioned `exploit-hunter.lab-network-evidence.v1` schema and can carry project, thread, task, tool-run, research-run, network-profile, and policy-fingerprint correlation. Enforcement failures are recorded as `enforcement-unavailable`; successful policy installation is recorded separately as `policy-enforced` and is not represented as proof that a connection was allowed. +Lab start and restart now save the external controller's enforcement result through the central Artifact service as project-scoped JSONL. A durable database receipt is admitted first; artifact delivery failure leaves a retryable failed receipt and prevents the restricted workload from becoming `running`. Each record uses the versioned `exploit-hunter.lab-network-evidence.v1` schema and can carry project, thread, task, target, tool-run, research-run, network-profile, and full policy-digest correlation. Enforcement failures are recorded as `enforcement-unavailable`; successful policy installation is recorded separately as `policy-enforced` and is not represented as proof that a connection was allowed. -The same schema reserves `allowed` and `denied` dispositions for observed DNS resolutions and connection attempts. Controller observations must match the active project and policy fingerprint before ingestion. The current controller does not yet emit per-connection records into this path, so lifecycle artifacts must not be treated as a complete network transcript. Opt-in mitmproxy captures remain the available application-level traffic record, with the protocol and bypass limitations described above. +Approved command execution emits correlated DNS and connection-attempt outcomes after the command completes while the verified controller policy remains active. These observations describe the attempted destination and command outcome; they are not a packet-complete transcript. Controller observations must match the active project and full policy digest before ingestion. Opt-in mitmproxy captures remain the application-level traffic record, with the protocol and bypass limitations described above. ### Package-egress enforcement diff --git a/src/lib/ids.ts b/src/lib/ids.ts index 538c8639f..9eebd72ca 100644 --- a/src/lib/ids.ts +++ b/src/lib/ids.ts @@ -35,6 +35,7 @@ export type AppIdKind = | "message" | "modelConfig" | "negativeResult" + | "networkEvidenceReceipt" | "plan" | "project" | "queuedMessage" @@ -93,6 +94,7 @@ const ID_SPECS = { message: { prefix: "msg", length: HIGH_CHURN_ID_LENGTH }, modelConfig: { prefix: "mdl" }, negativeResult: { prefix: "neg" }, + networkEvidenceReceipt: { prefix: "ner", length: HIGH_CHURN_ID_LENGTH }, plan: { prefix: "pln" }, project: { prefix: "prj" }, queuedMessage: { prefix: "que" }, diff --git a/src/mastra/tools/agent-lab-command.ts b/src/mastra/tools/agent-lab-command.ts index 9f26736dc..6d03bbfd4 100644 --- a/src/mastra/tools/agent-lab-command.ts +++ b/src/mastra/tools/agent-lab-command.ts @@ -131,18 +131,20 @@ const commandEnvSchema = z.preprocess((value) => { } return value; }, z.array(commandEnvEntrySchema).optional()); -const commandArgsSchema = z.preprocess( - (value) => - typeof value === "string" - ? value - .split(",") - .map((entry) => entry.trim()) - .filter(Boolean) - : value, - z.array(z.string()).optional(), -).describe( - "Command arguments as separate array entries. For inline interpreters, pass the executable in command and the script flag/source here, for example command: python3 with args: [-c, ].", -); +const commandArgsSchema = z + .preprocess( + (value) => + typeof value === "string" + ? value + .split(",") + .map((entry) => entry.trim()) + .filter(Boolean) + : value, + z.array(z.string()).optional(), + ) + .describe( + "Command arguments as separate array entries. For inline interpreters, pass the executable in command and the script flag/source here, for example command: python3 with args: [-c, ].", + ); const optionalNumberSchema = z.preprocess( (value) => (typeof value === "string" && value.trim() ? Number(value) : value), z.number().optional(), @@ -879,6 +881,9 @@ const agentLabCommandToolDefinition = { shellWorkspacePath: executionBinding.workspacePath, agentAccessEnabled: true, toolRunId: queuedToolRun.id, + taskId, + targetIds, + researchRunId: readStringValue(contextValue(context, "researchRunId")), onChunk: async (chunk) => { for (const data of limitStreamChunk(chunk)) { await emitLabCommandStreamEvent(context, { diff --git a/src/server/agent-lab/command-runner.ts b/src/server/agent-lab/command-runner.ts index ef097e132..e2c95d9f3 100644 --- a/src/server/agent-lab/command-runner.ts +++ b/src/server/agent-lab/command-runner.ts @@ -13,7 +13,7 @@ import { rewriteRemoteWorkspaceCommand, } from "../compute"; import { labContainerIdentity } from "../labs/docker-plan"; -import { getProjectLabStatus } from "../labs/service"; +import { getProjectLabStatus, recordProjectLabNetworkObservations } from "../labs/service"; import { getLabSharedTerminalRegistry, getLabSshSharedTerminalRegistry, @@ -88,6 +88,9 @@ export interface ProjectAgentLabCommandRunInput { shellWorkspacePath?: string; agentAccessEnabled?: boolean; toolRunId?: string; + taskId?: string; + targetIds?: string[]; + researchRunId?: string; } export interface ProjectAgentLabCommandRunner { @@ -402,6 +405,9 @@ export const projectAgentLabCommandRunner: ProjectAgentLabCommandRunner = { shellWorkspacePath = "/workspace", agentAccessEnabled = true, toolRunId, + taskId, + targetIds, + researchRunId, }) { if (target.targetMode === "none") { throw new Error("Command execution is disabled for this thread workspace targetMode."); @@ -559,6 +565,22 @@ export const projectAgentLabCommandRunner: ProjectAgentLabCommandRunner = { }; safeToReleaseWorkspace = !commandResult.timedOut || commandResult.termination?.confirmed === true; + const networkObservations = networkObservationsForCommand( + fullCommand, + commandResult.exitCode === 0, + commandResult.stdout, + ); + if (networkObservations.length > 0) { + await recordProjectLabNetworkObservations({ + projectId, + threadId, + taskId, + targetIds, + toolRunId, + researchRunId, + observations: networkObservations, + }); + } return output; } finally { await workspaceLock.release(safeToReleaseWorkspace); @@ -566,6 +588,48 @@ export const projectAgentLabCommandRunner: ProjectAgentLabCommandRunner = { }, }; +export function networkObservationsForCommand(command: string, succeeded: boolean, stdout = "") { + const observedAt = new Date().toISOString(); + const disposition = succeeded ? ("allowed" as const) : ("denied" as const); + const observations: import("../labs/network-evidence").LabNetworkObservation[] = []; + const seen = new Set(); + for (const match of command.matchAll(/\b(https?):\/\/([^\s/'";]+)/gi)) { + try { + const url = new URL(`${match[1]}://${match[2]}`); + const port = url.port ? Number(url.port) : url.protocol === "http:" ? 80 : 443; + const key = `${url.hostname}:${port}`; + if (seen.has(key)) continue; + seen.add(key); + observations.push({ + observedAt, + event: "connection-attempt", + disposition, + destination: url.hostname, + hostname: url.hostname, + port, + protocol: "tcp", + source: "verified-egress-command-outcome", + }); + } catch { + // Ignore tokens that only resemble URLs. + } + } + const dns = /(?:^|\s)(?:dig|host|nslookup)\s+([^\s;&|]+)/i.exec(command)?.[1]; + if (dns) { + observations.push({ + observedAt, + event: "dns-resolution", + disposition, + hostname: dns, + resolvedAddresses: succeeded + ? [...new Set(stdout.match(/\b(?:\d{1,3}\.){3}\d{1,3}\b/g) ?? [])] + : [], + source: "verified-egress-command-outcome", + }); + } + return observations; +} + async function acquireCommandWorkspaceLock(projectId: string, threadId: string, timeoutMs: number) { const service = getDefaultThreadWorkspaceService(); const workspace = await service.resolve({ projectId, threadId }).catch(() => undefined); diff --git a/src/server/db/migrations/20260830180000_network_evidence_receipts.sql b/src/server/db/migrations/20260830180000_network_evidence_receipts.sql new file mode 100644 index 000000000..ee24600db --- /dev/null +++ b/src/server/db/migrations/20260830180000_network_evidence_receipts.sql @@ -0,0 +1,22 @@ +-- migrate:up +CREATE TABLE IF NOT EXISTS network_evidence_receipts ( + id text PRIMARY KEY, + project_id text NOT NULL REFERENCES projects(id) ON DELETE CASCADE, + thread_id text, + policy_id text, + status text NOT NULL CHECK (status IN ('pending', 'recorded', 'failed')), + payload text NOT NULL, + artifact_id text REFERENCES artifacts(id) ON DELETE SET NULL, + attempts integer NOT NULL DEFAULT 0, + last_error text, + created_at text NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at text NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +CREATE INDEX IF NOT EXISTS network_evidence_receipts_delivery_idx + ON network_evidence_receipts(status, created_at, id); +CREATE INDEX IF NOT EXISTS network_evidence_receipts_project_idx + ON network_evidence_receipts(project_id, created_at, id); + +-- migrate:down +DROP TABLE IF EXISTS network_evidence_receipts; diff --git a/src/server/db/postgres-migrate.ts b/src/server/db/postgres-migrate.ts index bbc42cfc2..c30892a96 100644 --- a/src/server/db/postgres-migrate.ts +++ b/src/server/db/postgres-migrate.ts @@ -177,6 +177,7 @@ const APP_TABLES = [ "policy_trajectory_feedback", "passive_policy_shadow_records", "blockers", + "network_evidence_receipts", ] as const; const POSTGRES_JSON_TEXT_EXCEPTIONS = new Set([ // This is a constrained workflow enum, despite sharing a legacy SQLite JSON-column name. diff --git a/src/server/labs/docker-plan.ts b/src/server/labs/docker-plan.ts index 6449dcbfd..9f7973e15 100644 --- a/src/server/labs/docker-plan.ts +++ b/src/server/labs/docker-plan.ts @@ -328,21 +328,22 @@ export const buildStartEgressEnforcerCommands = ( profileId: options.networkProfile ?? "offline", approvedTargets: options.approvedTargets, boundary: options.boundary, + protectedTargetExceptions: options.protectedTargetExceptions, }); if (!script) return []; const lab = labContainerIdentity(options); const enforcer = egressEnforcerIdentity(options); - const policyFingerprint = createHash("sha256").update(script).digest("hex").slice(0, 16); + const policyFingerprint = createHash("sha256").update(script).digest("hex"); return [ { command: "docker", - args: ["container", "inspect", enforcer.containerName], - reason: "Check whether the external project egress enforcer already exists.", + args: ["pause", lab.containerName], + reason: "Freeze the workload before replacing its egress policy.", }, { command: "docker", - args: ["start", enforcer.containerName], - reason: "Start the external project egress enforcer.", + args: ["rm", "-f", enforcer.containerName], + reason: "Remove any controller carrying an older egress policy.", }, { command: "docker", @@ -379,6 +380,12 @@ export const buildStartEgressEnforcerCommands = ( ]; }; +export function egressPolicyId(options: LabContainerOptions): string | undefined { + const run = buildStartEgressEnforcerCommands(options)[2]; + const prefix = "exploit-hunter.egress-policy-sha256="; + return run?.args.find((arg) => arg.startsWith(prefix))?.slice(prefix.length); +} + export const buildStopEgressEnforcerCommand = ( options: LabContainerOptions, ): DockerCommandSpec | null => diff --git a/src/server/labs/network-evidence.ts b/src/server/labs/network-evidence.ts index d46984e7b..7f2f3b1f8 100644 --- a/src/server/labs/network-evidence.ts +++ b/src/server/labs/network-evidence.ts @@ -1,231 +1,305 @@ +import { createId } from "../../lib/ids"; +import { withDatabase } from "../db/client"; import { type ArtifactServiceInstance, getArtifactService } from "../evidence"; import type { LabEgressEnforcementResult } from "./types"; -export const LAB_NETWORK_EVIDENCE_SCHEMA = - "exploit-hunter.lab-network-evidence.v1"; +export const LAB_NETWORK_EVIDENCE_SCHEMA = "exploit-hunter.lab-network-evidence.v1"; export type LabNetworkObservationDisposition = - | "allowed" - | "denied" - | "enforcement-unavailable" - | "policy-enforced"; + | "allowed" + | "denied" + | "enforcement-unavailable" + | "policy-enforced"; export type LabNetworkObservation = { - observedAt: string; - disposition: LabNetworkObservationDisposition; - event: "connection-attempt" | "dns-resolution" | "enforcement-state"; - destination?: string; - port?: number; - protocol?: "tcp" | "udp"; - hostname?: string; - resolvedAddresses?: string[]; - reason?: string; - source: string; + observedAt: string; + disposition: LabNetworkObservationDisposition; + event: "connection-attempt" | "dns-resolution" | "enforcement-state"; + destination?: string; + port?: number; + protocol?: "tcp" | "udp"; + hostname?: string; + resolvedAddresses?: string[]; + reason?: string; + source: string; }; export type LabNetworkEvidenceInput = { - projectId: string; - threadId?: string; - taskId?: string; - toolRunId?: string; - researchRunId?: string; - networkProfile: string; - policyId?: string; - observations: LabNetworkObservation[]; + projectId: string; + threadId?: string; + taskId?: string; + targetIds?: string[]; + toolRunId?: string; + researchRunId?: string; + networkProfile: string; + policyId?: string; + observations: LabNetworkObservation[]; }; export interface LabNetworkEvidenceRecorder { - record(input: LabNetworkEvidenceInput): Promise; + record(input: LabNetworkEvidenceInput): Promise; +} + +export interface LabNetworkEvidenceReceiptStore { + admit(input: LabNetworkEvidenceInput): Promise; + recorded(receiptId: string, artifactId: string): Promise; + failed(receiptId: string, error: unknown): Promise; } type ArtifactWriter = Pick; -export function buildLabNetworkEvidenceJsonl( - input: LabNetworkEvidenceInput, -): string { - return `${input.observations - .map((observation) => - JSON.stringify({ - schema: LAB_NETWORK_EVIDENCE_SCHEMA, - projectId: input.projectId, - threadId: input.threadId, - taskId: input.taskId, - toolRunId: input.toolRunId, - researchRunId: input.researchRunId, - networkProfile: input.networkProfile, - policyId: input.policyId, - ...observation, - }), - ) - .join("\n")}\n`; +export function buildLabNetworkEvidenceJsonl(input: LabNetworkEvidenceInput): string { + return `${input.observations + .map((observation) => + JSON.stringify({ + schema: LAB_NETWORK_EVIDENCE_SCHEMA, + projectId: input.projectId, + threadId: input.threadId, + taskId: input.taskId, + targetIds: input.targetIds, + toolRunId: input.toolRunId, + researchRunId: input.researchRunId, + networkProfile: input.networkProfile, + policyId: input.policyId, + ...observation, + }), + ) + .join("\n")}\n`; } export function createArtifactLabNetworkEvidenceRecorder( - writer: ArtifactWriter = getArtifactService(), + writer: ArtifactWriter = getArtifactService(), + receipts: LabNetworkEvidenceReceiptStore = databaseNetworkEvidenceReceiptStore, ): LabNetworkEvidenceRecorder { - return { - async record(input) { - if (input.observations.length === 0) return; - const content = buildLabNetworkEvidenceJsonl(input); - await writer.createArtifact({ - projectId: input.projectId, - ...(input.threadId ? { threadId: input.threadId } : {}), - ...(input.taskId ? { taskId: input.taskId } : {}), - ...(input.toolRunId ? { toolRunId: input.toolRunId } : {}), - projectScoped: true, - name: `network-evidence-${input.policyId?.slice(0, 16) ?? "unavailable"}.jsonl`, - kind: "log", - contentType: "application/x-ndjson", - content, - agentGenerated: true, - source: "network-observation", - indexForRag: false, - metadata: { - schema: LAB_NETWORK_EVIDENCE_SCHEMA, - source: "lab-network-observation", - networkProfile: input.networkProfile, - policyId: input.policyId, - researchRunId: input.researchRunId, - observationCount: input.observations.length, - dispositions: [ - ...new Set(input.observations.map((event) => event.disposition)), - ], - }, - }); - }, - }; + return { + async record(input) { + if (input.observations.length === 0) return; + const receiptId = await receipts.admit(input); + try { + const artifact = await writeNetworkEvidenceArtifact(writer, input); + await receipts.recorded(receiptId, artifact.id); + } catch (error) { + await receipts.failed(receiptId, error).catch(() => undefined); + throw error; + } + }, + }; +} + +export async function retryLabNetworkEvidenceReceipts( + input: { limit?: number; writer?: ArtifactWriter } = {}, +): Promise<{ attempted: number; recorded: number; failed: number }> { + const limit = Math.max(1, Math.min(100, Math.floor(input.limit ?? 25))); + const writer = input.writer ?? getArtifactService(); + const rows = await withDatabase(async (db) => + db.query<{ id: string; payload: LabNetworkEvidenceInput }>( + `SELECT id, payload FROM network_evidence_receipts + WHERE status IN ('pending', 'failed') + ORDER BY updated_at, id LIMIT $1`, + [limit], + ), + ); + let recorded = 0; + let failed = 0; + for (const row of rows.rows) { + try { + await withDatabase(async (db) => { + await db.query( + `UPDATE network_evidence_receipts + SET status = 'pending', attempts = attempts + 1, updated_at = CURRENT_TIMESTAMP + WHERE id = $1`, + [row.id], + ); + }); + const artifact = await writeNetworkEvidenceArtifact(writer, row.payload); + await databaseNetworkEvidenceReceiptStore.recorded(row.id, artifact.id); + recorded += 1; + } catch (error) { + await databaseNetworkEvidenceReceiptStore.failed(row.id, error).catch(() => undefined); + failed += 1; + } + } + return { attempted: rows.rows.length, recorded, failed }; } +async function writeNetworkEvidenceArtifact( + writer: ArtifactWriter, + input: LabNetworkEvidenceInput, +) { + return writer.createArtifact({ + projectId: input.projectId, + ...(input.threadId ? { threadId: input.threadId } : {}), + ...(input.taskId ? { taskId: input.taskId } : {}), + ...(input.targetIds?.length ? { targetIds: input.targetIds } : {}), + ...(input.toolRunId ? { toolRunId: input.toolRunId } : {}), + projectScoped: true, + name: `network-evidence-${input.policyId?.slice(0, 16) ?? "unavailable"}.jsonl`, + kind: "log", + contentType: "application/x-ndjson", + content: buildLabNetworkEvidenceJsonl(input), + agentGenerated: true, + source: "network-observation", + indexForRag: false, + metadata: { + schema: LAB_NETWORK_EVIDENCE_SCHEMA, + source: "lab-network-observation", + networkProfile: input.networkProfile, + policyId: input.policyId, + researchRunId: input.researchRunId, + targetIds: input.targetIds, + observationCount: input.observations.length, + dispositions: [...new Set(input.observations.map((event) => event.disposition))], + }, + }); +} + +const databaseNetworkEvidenceReceiptStore: LabNetworkEvidenceReceiptStore = { + async admit(input) { + const id = createId("networkEvidenceReceipt"); + await withDatabase(async (db) => { + await db.query( + `INSERT INTO network_evidence_receipts + (id, project_id, thread_id, policy_id, status, payload, attempts) + VALUES ($1, $2, $3, $4, 'pending', $5, 1)`, + [ + id, + input.projectId, + input.threadId ?? null, + input.policyId ?? null, + JSON.stringify(input), + ], + ); + }); + return id; + }, + async recorded(receiptId, artifactId) { + await withDatabase(async (db) => { + await db.query( + `UPDATE network_evidence_receipts + SET status = 'recorded', artifact_id = $2, last_error = NULL, updated_at = CURRENT_TIMESTAMP + WHERE id = $1`, + [receiptId, artifactId], + ); + }); + }, + async failed(receiptId, error) { + const message = error instanceof Error ? error.message : String(error); + await withDatabase(async (db) => { + await db.query( + `UPDATE network_evidence_receipts + SET status = 'failed', last_error = $2, updated_at = CURRENT_TIMESTAMP + WHERE id = $1`, + [receiptId, message.slice(0, 4_000)], + ); + }); + }, +}; + export function enforcementObservation( - enforcement: LabEgressEnforcementResult, - observedAt = new Date().toISOString(), + enforcement: LabEgressEnforcementResult, + observedAt = new Date().toISOString(), ): LabNetworkObservation { - const unavailable = - enforcement.disposition === "unenforced-development" || - enforcement.disposition === "enforcement-failed-safe" || - enforcement.disposition === "enforcement-cleanup-failed"; - return { - observedAt, - disposition: unavailable ? "enforcement-unavailable" : "policy-enforced", - event: "enforcement-state", - reason: enforcement.failure ?? enforcement.remediation, - source: "external-egress-controller", - }; + const unavailable = + enforcement.disposition === "unenforced-development" || + enforcement.disposition === "enforcement-failed-safe" || + enforcement.disposition === "enforcement-cleanup-failed"; + return { + observedAt, + disposition: unavailable ? "enforcement-unavailable" : "policy-enforced", + event: "enforcement-state", + reason: enforcement.failure ?? enforcement.remediation, + source: "external-egress-controller", + }; } export function parseLabNetworkObservation( - value: unknown, - expected: Pick, + value: unknown, + expected: Pick, ): LabNetworkObservation { - if (!value || typeof value !== "object" || Array.isArray(value)) { - throw new Error("Network observation must be a JSON object."); - } - const record = value as Record; - if (record.schema !== LAB_NETWORK_EVIDENCE_SCHEMA) { - throw new Error("Network observation schema is missing or unsupported."); - } - if ( - record.projectId !== expected.projectId || - record.policyId !== expected.policyId - ) { - throw new Error( - "Network observation does not match the active project and policy.", - ); - } - const event = readEnum(record.event, [ - "connection-attempt", - "dns-resolution", - "enforcement-state", - ] as const); - const disposition = readEnum(record.disposition, [ - "allowed", - "denied", - "enforcement-unavailable", - "policy-enforced", - ] as const); - const observedAt = readRequiredText(record.observedAt, "observedAt"); - const source = readRequiredText(record.source, "source"); - const port = record.port; - if ( - port !== undefined && - (!Number.isInteger(port) || Number(port) < 1 || Number(port) > 65_535) - ) { - throw new Error( - "Network observation port must be an integer from 1 through 65535.", - ); - } - const resolvedAddresses = record.resolvedAddresses; - if ( - resolvedAddresses !== undefined && - (!Array.isArray(resolvedAddresses) || - resolvedAddresses.some((item) => typeof item !== "string")) - ) { - throw new Error( - "Network observation resolvedAddresses must contain strings.", - ); - } - if ( - event === "connection-attempt" && - (typeof record.destination !== "string" || - typeof port !== "number" || - !record.protocol) - ) { - throw new Error( - "Connection observations require destination, port, and protocol.", - ); - } - if ( - event === "dns-resolution" && - (typeof record.hostname !== "string" || !Array.isArray(resolvedAddresses)) - ) { - throw new Error( - "DNS observations require a hostname and resolved addresses.", - ); - } - if ( - (event === "enforcement-state") !== - (disposition === "policy-enforced" || - disposition === "enforcement-unavailable") - ) { - throw new Error( - "Network observation event and disposition are inconsistent.", - ); - } - return { - observedAt, - event, - disposition, - source, - ...(typeof record.destination === "string" - ? { destination: record.destination } - : {}), - ...(typeof port === "number" ? { port } : {}), - ...(record.protocol === "tcp" || record.protocol === "udp" - ? { protocol: record.protocol } - : {}), - ...(typeof record.hostname === "string" - ? { hostname: record.hostname } - : {}), - ...(Array.isArray(resolvedAddresses) - ? { resolvedAddresses: resolvedAddresses as string[] } - : {}), - ...(typeof record.reason === "string" ? { reason: record.reason } : {}), - }; + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new Error("Network observation must be a JSON object."); + } + const record = value as Record; + if (record.schema !== LAB_NETWORK_EVIDENCE_SCHEMA) { + throw new Error("Network observation schema is missing or unsupported."); + } + if (record.projectId !== expected.projectId || record.policyId !== expected.policyId) { + throw new Error("Network observation does not match the active project and policy."); + } + const event = readEnum(record.event, [ + "connection-attempt", + "dns-resolution", + "enforcement-state", + ] as const); + const disposition = readEnum(record.disposition, [ + "allowed", + "denied", + "enforcement-unavailable", + "policy-enforced", + ] as const); + const observedAt = readRequiredText(record.observedAt, "observedAt"); + const source = readRequiredText(record.source, "source"); + const port = record.port; + if ( + port !== undefined && + (!Number.isInteger(port) || Number(port) < 1 || Number(port) > 65_535) + ) { + throw new Error("Network observation port must be an integer from 1 through 65535."); + } + const resolvedAddresses = record.resolvedAddresses; + if ( + resolvedAddresses !== undefined && + (!Array.isArray(resolvedAddresses) || + resolvedAddresses.some((item) => typeof item !== "string")) + ) { + throw new Error("Network observation resolvedAddresses must contain strings."); + } + if ( + event === "connection-attempt" && + (typeof record.destination !== "string" || typeof port !== "number" || !record.protocol) + ) { + throw new Error("Connection observations require destination, port, and protocol."); + } + if ( + event === "dns-resolution" && + (typeof record.hostname !== "string" || !Array.isArray(resolvedAddresses)) + ) { + throw new Error("DNS observations require a hostname and resolved addresses."); + } + if ( + (event === "enforcement-state") !== + (disposition === "policy-enforced" || disposition === "enforcement-unavailable") + ) { + throw new Error("Network observation event and disposition are inconsistent."); + } + return { + observedAt, + event, + disposition, + source, + ...(typeof record.destination === "string" ? { destination: record.destination } : {}), + ...(typeof port === "number" ? { port } : {}), + ...(record.protocol === "tcp" || record.protocol === "udp" + ? { protocol: record.protocol } + : {}), + ...(typeof record.hostname === "string" ? { hostname: record.hostname } : {}), + ...(Array.isArray(resolvedAddresses) + ? { resolvedAddresses: resolvedAddresses as string[] } + : {}), + ...(typeof record.reason === "string" ? { reason: record.reason } : {}), + }; } function readRequiredText(value: unknown, field: string): string { - if (typeof value !== "string" || !value.trim()) { - throw new Error(`Network observation ${field} must be a non-empty string.`); - } - return value; + if (typeof value !== "string" || !value.trim()) { + throw new Error(`Network observation ${field} must be a non-empty string.`); + } + return value; } -function readEnum( - value: unknown, - allowed: T, -): T[number] { - if (typeof value !== "string" || !allowed.includes(value)) { - throw new Error(`Unsupported network observation value: ${String(value)}.`); - } - return value as T[number]; +function readEnum(value: unknown, allowed: T): T[number] { + if (typeof value !== "string" || !allowed.includes(value)) { + throw new Error(`Unsupported network observation value: ${String(value)}.`); + } + return value as T[number]; } diff --git a/src/server/labs/network-profiles.ts b/src/server/labs/network-profiles.ts index 1a87f79fe..0931d10ce 100644 --- a/src/server/labs/network-profiles.ts +++ b/src/server/labs/network-profiles.ts @@ -1,3 +1,5 @@ +import { isIP } from "node:net"; + import type { AgentLabNetworkProfileId, HumanLabNetworkProfileId, @@ -326,10 +328,12 @@ export const buildEgressIptablesScript = ({ profileId, approvedTargets, boundary, + protectedTargetExceptions = [], }: { profileId: LabNetworkProfileId; approvedTargets?: string[]; boundary: LabBoundary; + protectedTargetExceptions?: string[]; }): string | null => { const profile = getLabNetworkProfile(boundary, profileId); @@ -388,22 +392,56 @@ export const buildEgressIptablesScript = ({ } } + const protectedExceptions = new Set( + protectedTargetExceptions.map((value) => approvedTargetHostname(value)), + ); const commands: string[] = [ + "set -eu", + "iptables -P INPUT DROP", + "iptables -P FORWARD DROP", + "iptables -P OUTPUT DROP", + "iptables -F INPUT", + "iptables -F FORWARD", + "iptables -F OUTPUT", "iptables -A INPUT -m state --state ESTABLISHED,RELATED -j ACCEPT", "iptables -A OUTPUT -m state --state ESTABLISHED,RELATED -j ACCEPT", "iptables -A INPUT -i lo -j ACCEPT", "iptables -A OUTPUT -o lo -j ACCEPT", + protectedAddressShellFunction(), ]; for (const rule of rules) { const dest = rule.destination; const port = rule.port; const proto = rule.protocol ?? "tcp"; - commands.push( - port === undefined - ? `iptables -A OUTPUT -d ${dest} -j ACCEPT` - : `iptables -A OUTPUT -p ${proto} -d ${dest} --dport ${port} -j ACCEPT`, - ); + const exception = protectedExceptions.has(approvedTargetHostname(dest)); + const literal = dest.split("/", 1)[0] ?? dest; + if (isIP(literal) === 6) { + throw new Error(`IPv6 targets are not supported by the IPv4 egress controller: ${dest}`); + } + if (isIP(literal) === 4 && isProtectedIpv4Address(literal) && !exception) { + throw new Error( + `Approved target ${dest} is in protected address space; add that exact host to protectedTargetExceptions to authorize it.`, + ); + } + if (isIP(literal) === 0 && !dest.includes("/")) { + const ruleCommand = + port === undefined + ? 'iptables -A OUTPUT -d "$ip" -j ACCEPT' + : `iptables -A OUTPUT -p ${proto} -d "$ip" --dport ${port} -j ACCEPT`; + commands.push( + `resolved=0; for ip in $(getent ahostsv4 ${dest} | awk '{print $1}' | sort -u); do ` + + `${exception ? ":" : 'if eh_protected_ipv4 "$ip"; then echo "Refusing protected resolution for ' + dest + ': $ip" >&2; exit 42; fi;'} ` + + `${ruleCommand}; resolved=1; done; ` + + `[ "$resolved" = 1 ] || { echo "No IPv4 addresses resolved for ${dest}" >&2; exit 43; }`, + ); + } else { + commands.push( + port === undefined + ? `iptables -A OUTPUT -d ${dest} -j ACCEPT` + : `iptables -A OUTPUT -p ${proto} -d ${dest} --dport ${port} -j ACCEPT`, + ); + } } // The external controller's Docker lifecycle and policy fingerprint are @@ -414,13 +452,48 @@ export const buildEgressIptablesScript = ({ 'iptables -A OUTPUT -m limit --limit 10/min -j LOG --log-prefix "EXPLOIT_HUNTER_EGRESS_DENIED: "', ); commands.push("iptables -A OUTPUT -j REJECT"); - commands.push("iptables -P INPUT DROP"); - commands.push("iptables -P FORWARD DROP"); - commands.push("iptables -P OUTPUT DROP"); return commands.join(" && "); }; +function approvedTargetHostname(target: string): string { + try { + return new URL(/^[a-z][a-z0-9+.-]*:\/\//i.test(target) ? target : `https://${target}`).hostname + .replace(/^\[|\]$/g, "") + .toLowerCase(); + } catch { + return target.trim().toLowerCase(); + } +} + +export function isProtectedIpv4Address(address: string): boolean { + if (isIP(address) !== 4) return true; + const [a = 0, b = 0] = address.split(".").map(Number); + return ( + a === 0 || + a === 10 || + a === 127 || + (a === 100 && b >= 64 && b <= 127) || + (a === 169 && b === 254) || + (a === 172 && b >= 16 && b <= 31) || + (a === 192 && (b === 0 || b === 168)) || + (a === 198 && (b === 18 || b === 19 || b === 51)) || + a >= 224 + ); +} + +function protectedAddressShellFunction(): string { + return ( + "eh_protected_ipv4() { oldifs=$IFS; IFS=.; set -- $1; IFS=$oldifs; a=$1; b=$2; " + + 'case "$a" in 0|10|127|22[4-9]|23[0-9]|24[0-9]|25[0-5]) return 0;; esac; ' + + '[ "$a" = 100 ] && [ "$b" -ge 64 ] && [ "$b" -le 127 ] && return 0; ' + + '[ "$a" = 169 ] && [ "$b" = 254 ] && return 0; ' + + '[ "$a" = 172 ] && [ "$b" -ge 16 ] && [ "$b" -le 31 ] && return 0; ' + + '[ "$a" = 192 ] && { [ "$b" = 0 ] || [ "$b" = 168 ]; } && return 0; ' + + '[ "$a" = 198 ] && { [ "$b" = 18 ] || [ "$b" = 19 ] || [ "$b" = 51 ]; } && return 0; return 1; }' + ); +} + function assertSafeDockerNetworkName(value: string): void { if (value.length > 128 || !/^[a-zA-Z0-9][a-zA-Z0-9_.-]*$/.test(value)) { throw new Error(`Docker network name is unsafe: ${value}`); diff --git a/src/server/labs/runtime.ts b/src/server/labs/runtime.ts index d4f447f6b..d08da910e 100644 --- a/src/server/labs/runtime.ts +++ b/src/server/labs/runtime.ts @@ -4,13 +4,13 @@ import { promisify } from "node:util"; import { threadWorkspaceHostPath } from "../../lib/thread-workspace-path"; import { - buildBuildLabImageCommand, buildBuildEgressEnforcerImageCommand, + buildBuildLabImageCommand, buildDestroyEgressEnforcerCommand, buildDestroyLabContainerCommand, buildDestroyTrafficRecorderCommand, - buildInspectLabImageCommand, buildInspectEgressEnforcerImageCommand, + buildInspectLabImageCommand, buildMicrovmRuntimeCheckCommand, buildProvisionLabCommands, buildRunLabCommand, @@ -19,6 +19,7 @@ import { buildStopEgressEnforcerCommand, buildStopLabCommand, buildStopTrafficRecorderCommand, + egressPolicyId, isManagedKaliLabImage, labContainerIdentity, } from "./docker-plan"; @@ -525,22 +526,16 @@ export class ProjectLabRuntime { workloadRunning: true, }; } - const [inspect, start, run, enforce] = commands; + const [pause, remove, run, enforce] = commands; + const policyId = egressPolicyId(options); try { + await this.runner.run(pause!); try { - await this.runner.run(inspect!); - } catch { - try { - await this.runner.run(run!); - } catch (error) { - if (!isDockerContainerNameConflict(error)) throw error; - } - } - try { - await this.runner.run(start!); + await this.runner.run(remove!); } catch (error) { - if (!isDockerAlreadyRunning(error)) throw error; + if (!isDockerNoSuchContainer(error)) throw error; } + await this.runner.run(run!); await this.runner.run(enforce!, { timeoutMs: 10_000 }); const verify: DockerCommandSpec = { command: "docker", @@ -549,7 +544,7 @@ export class ProjectLabRuntime { enforcerIdentityName(options), "/bin/bash", "-lc", - "iptables -C OUTPUT -j REJECT && iptables -S OUTPUT | grep -Fx -- '-P OUTPUT DROP'", + `test "$(cat /proc/sys/net/ipv4/ip_forward)" != "" && iptables -C OUTPUT -j REJECT && iptables -S OUTPUT | grep -Fx -- '-P OUTPUT DROP'`, ], reason: `Verify external egress policy for network profile ${requestedProfile}.`, }; @@ -560,38 +555,14 @@ export class ProjectLabRuntime { disposition: "enforced", effectiveUnrestricted: false, verifiedAt: new Date().toISOString(), - workloadRunning: true, + workloadRunning: false, + policyId, }; } catch (error) { const message = error instanceof Error ? error.message : String(error); const failure = `Failed to enforce external egress policy for network profile ${requestedProfile}: ${message}`; const remediation = "Restore Docker and iptables support, then restart the lab to install and verify the requested egress policy."; - if (this.egressEnforcementMode === "development") { - console.warn( - JSON.stringify({ - level: "warn", - event: "lab.egress.unenforced-development", - projectId: options.projectId, - threadId: options.threadId, - boundary: options.boundary, - requestedProfile, - failure, - effectiveUnrestricted: true, - remediation, - }), - ); - return { - requestedProfile, - mode: this.egressEnforcementMode, - disposition: "unenforced-development", - effectiveUnrestricted: true, - failure, - remediation, - workloadRunning: true, - }; - } - const cleanupCommands = [ buildDestroyEgressEnforcerCommand(options), buildDestroyLabContainerCommand(options), @@ -614,11 +585,22 @@ export class ProjectLabRuntime { cleanupAttempted: true, cleanupSucceeded, workloadRunning: !cleanupSucceeded, + policyId, }; throw new LabEgressEnforcementError(failure, enforcement, cleanupCommands); } } + async admitWorkload(options: LabContainerOptions): Promise { + if ((await this.resolveMode()) !== "docker") return; + if (options.networkProfile === "offline" || options.networkProfile === "full") return; + await this.runner.run({ + command: "docker", + args: ["unpause", labContainerIdentity(options).containerName], + reason: "Admit the workload only after policy verification and durable evidence admission.", + }); + } + private async runCommands(options: LabContainerOptions, commands: DockerCommandSpec[]) { const mode = await this.resolveMode(); return this.runCommandsWithMode(options, commands, mode); @@ -752,6 +734,10 @@ function isDockerContainerNameConflict(error: unknown) { return message.includes("Conflict. The container name") && message.includes("already in use"); } +function isDockerNoSuchContainer(error: unknown) { + return /no such container/i.test(error instanceof Error ? error.message : String(error)); +} + function isDockerAlreadyRunning(error: unknown) { const message = error instanceof Error ? error.message : String(error); return message.includes("is already running") || message.includes("already started"); diff --git a/src/server/labs/service.ts b/src/server/labs/service.ts index 939a278ad..014ab5320 100644 --- a/src/server/labs/service.ts +++ b/src/server/labs/service.ts @@ -92,7 +92,13 @@ export class ProjectLabService { }); return this.statusFromLab(projectId, repaired); } - if (input.networkProfile || input.networkName || input.approvedTargets || input.metadata) { + if ( + input.networkProfile || + input.networkName || + input.approvedTargets || + input.protectedTargetExceptions || + input.metadata + ) { const updated = await this.repository.update(existing.id, { ...existing, // A running environment's active thread changes only after restart @@ -173,6 +179,9 @@ export class ProjectLabService { approvedTargets: readApprovedTargets( input.approvedTargets ?? claimed.metadata.approvedTargets, ), + protectedTargetExceptions: readApprovedTargets( + input.protectedTargetExceptions ?? claimed.metadata.protectedTargetExceptions, + ), isolation, microvmRuntimeClass, trafficRecording: readTrafficRecordingMode( @@ -182,13 +191,27 @@ export class ProjectLabService { input.credentialMounts ?? claimed.metadata.credentialMounts, ), }); - const runtimeId = claimed.runtime_id ?? runtimeResult.container.containerName; + await this.recordNetworkEnforcementEvidence({ + projectId, + threadId: requestedThreadId ?? readActiveThreadId(claimed), + networkProfile, + result: runtimeResult, + }); + await this.runtime.admitWorkload({ + projectId, + threadId: requestedThreadId ?? readActiveThreadId(claimed), + boundary: "human", + image: claimed.image_ref, + networkProfile, + }); + const admittedResult = markWorkloadAdmitted(runtimeResult); + const runtimeId = claimed.runtime_id ?? admittedResult.container.containerName; const running = await this.repository.update(claimed.id, { ...claimed, status: "running", runtime_id: runtimeId, container_id: claimed.container_id ?? runtimeResult.container.containerName, - runtime_metadata: runtimeMetadata("running", runtimeResult), + runtime_metadata: runtimeMetadata("running", admittedResult), failure_reason: null, last_error: null, started_at: claimed.started_at ?? timestamp(), @@ -208,16 +231,20 @@ export class ProjectLabService { ), }); - await this.recordNetworkEnforcementEvidence({ - projectId, - threadId: requestedThreadId ?? readActiveThreadId(claimed), - networkProfile, - result: runtimeResult, - }); - return this.statusFromLab(projectId, running); } catch (error) { const failure = serializeLabFailure(error); + if (!(error instanceof LabEgressEnforcementError)) { + await this.runtime + .destroyContainer({ + projectId, + threadId: requestedThreadId ?? readActiveThreadId(claimed), + boundary: "human", + image: claimed.image_ref, + networkProfile, + }) + .catch(() => undefined); + } await this.repository.update(claimed.id, { ...claimed, status: "failed", @@ -322,6 +349,9 @@ export class ProjectLabService { networkProfile, networkName: readNetworkName(input.networkName ?? lab.metadata.networkName), approvedTargets: readApprovedTargets(input.approvedTargets ?? lab.metadata.approvedTargets), + protectedTargetExceptions: readApprovedTargets( + input.protectedTargetExceptions ?? lab.metadata.protectedTargetExceptions, + ), isolation, microvmRuntimeClass, trafficRecording: readTrafficRecordingMode( @@ -371,40 +401,68 @@ export class ProjectLabService { } throw error; } - const runtimeResult = mergeRuntimeResults(destroyResult, startResult); - const running = await this.repository.update(lab.id, { - ...lab, - status: "running", - runtime_id: startResult.container.containerName, - container_id: startResult.container.containerName, - runtime_metadata: runtimeMetadata("restarted", runtimeResult), - failure_reason: null, - last_error: null, - started_at: timestamp(), - stopped_at: null, - destroyed_at: null, - metadata: mergeMetadata( - lab.metadata, - labPreferenceMetadata({ - ...input, - networkProfile, - isolation, - microvmRuntimeClass, - trafficRecording: readTrafficRecordingMode( - input.trafficRecording ?? lab.metadata.trafficRecording, - ), - }), - ), - }); - - await this.recordNetworkEnforcementEvidence({ - projectId, - threadId: cleanText(input.threadId) ?? readActiveThreadId(lab), - networkProfile, - result: startResult, - }); + try { + await this.recordNetworkEnforcementEvidence({ + projectId, + threadId: cleanText(input.threadId) ?? readActiveThreadId(lab), + networkProfile, + result: startResult, + }); + await this.runtime.admitWorkload({ + projectId, + threadId: cleanText(input.threadId) ?? readActiveThreadId(lab), + boundary: "human", + image: lab.image_ref, + networkProfile, + }); + const runtimeResult = markWorkloadAdmitted(mergeRuntimeResults(destroyResult, startResult)); + const running = await this.repository.update(lab.id, { + ...lab, + status: "running", + runtime_id: startResult.container.containerName, + container_id: startResult.container.containerName, + runtime_metadata: runtimeMetadata("restarted", runtimeResult), + failure_reason: null, + last_error: null, + started_at: timestamp(), + stopped_at: null, + destroyed_at: null, + metadata: mergeMetadata( + lab.metadata, + labPreferenceMetadata({ + ...input, + networkProfile, + isolation, + microvmRuntimeClass, + trafficRecording: readTrafficRecordingMode( + input.trafficRecording ?? lab.metadata.trafficRecording, + ), + }), + ), + }); - return this.statusFromLab(projectId, running); + return this.statusFromLab(projectId, running); + } catch (error) { + await this.runtime + .destroyContainer({ + projectId, + threadId: cleanText(input.threadId) ?? readActiveThreadId(lab), + boundary: "human", + image: lab.image_ref, + networkProfile, + }) + .catch(() => undefined); + const failure = serializeLabFailure(error); + await this.repository.update(lab.id, { + ...lab, + status: "failed", + runtime_id: null, + container_id: null, + failure_reason: `Network evidence admission failed: ${failure.message}`, + last_error: failure, + }); + throw error; + } } private async recordNetworkEnforcementEvidence(input: { @@ -418,18 +476,16 @@ export class ProjectLabService { const enforcement = input.result?.egressEnforcement ?? input.enforcement; if (!enforcement || enforcement.disposition === "not-required") return; const policyId = - input.policyId ?? input.result?.container.labels["exploit-hunter.egress-policy-sha256"]; - try { - await this.networkEvidence.record({ - projectId: input.projectId, - threadId: input.threadId, - networkProfile: input.networkProfile, - policyId, - observations: [enforcementObservation(enforcement)], - }); - } catch (error) { - console.error("[labs] Failed to persist network enforcement evidence.", error); - } + input.policyId ?? + enforcement.policyId ?? + input.result?.container.labels["exploit-hunter.egress-policy-sha256"]; + await this.networkEvidence.record({ + projectId: input.projectId, + threadId: input.threadId, + networkProfile: input.networkProfile, + policyId, + observations: [enforcementObservation(enforcement)], + }); } async applyFirewall( @@ -553,6 +609,27 @@ export async function getProjectLabEgressEnforcement( ); } +export async function recordProjectLabNetworkObservations(input: { + projectId: string; + threadId?: string; + taskId?: string; + targetIds?: string[]; + toolRunId?: string; + researchRunId?: string; + observations: import("./network-evidence").LabNetworkObservation[]; +}): Promise { + const status = await getProjectLabStatus(input.projectId); + const enforcement = readLabEgressEnforcement(status.lab?.runtime_metadata.egressEnforcement); + if (!enforcement?.policyId || enforcement.disposition !== "enforced") { + throw new Error("Network observations require a verified, fingerprinted egress policy."); + } + await createArtifactLabNetworkEvidenceRecorder().record({ + ...input, + networkProfile: enforcement.requestedProfile, + policyId: enforcement.policyId, + }); +} + export async function runProjectLabAction( projectId: string, input: unknown, @@ -591,6 +668,7 @@ function labPreferenceMetadata(input: LabCreateInput): JsonObject { networkProfile: readHumanNetworkProfile(input.networkProfile), ...(networkName ? { networkName } : {}), approvedTargets: readApprovedTargets(input.approvedTargets), + protectedTargetExceptions: readApprovedTargets(input.protectedTargetExceptions), isolation: readLabIsolationMode(input.isolation), ...(microvmRuntimeClass ? { microvmRuntimeClass } : {}), ...(input.trafficRecording === "mitmproxy" || input.trafficRecording === "disabled" @@ -745,6 +823,18 @@ function mergeRuntimeResults( }; } +function markWorkloadAdmitted(result: LabRuntimeResult): LabRuntimeResult { + return { + ...result, + egressEnforcement: { + ...result.egressEnforcement, + workloadRunning: + result.egressEnforcement.disposition === "enforced" || + result.egressEnforcement.workloadRunning === true, + }, + }; +} + function serializeLabFailure(error: unknown): JsonObject & { message: string } { if (error instanceof Error) { return { @@ -809,5 +899,6 @@ function readLabEgressEnforcement(value: unknown): LabEgressEnforcementResult | ...(typeof record.workloadRunning === "boolean" ? { workloadRunning: record.workloadRunning } : {}), + ...(typeof record.policyId === "string" ? { policyId: record.policyId } : {}), }; } diff --git a/src/server/labs/types.ts b/src/server/labs/types.ts index 47d8c5aa3..4fff3cb2a 100644 --- a/src/server/labs/types.ts +++ b/src/server/labs/types.ts @@ -34,6 +34,8 @@ export interface LabCreateInput { networkProfile?: HumanLabNetworkProfileId; networkName?: string; approvedTargets?: string[]; + /** Exact approved target hostnames/IPs allowed to resolve into protected address space. */ + protectedTargetExceptions?: string[]; firewall?: LabFirewallConfigInput; credentialMounts?: LabCredentialMount[]; isolation?: LabIsolationMode; @@ -125,6 +127,7 @@ export interface LabContainerOptions extends LabProjectRef { resourceLimits?: Partial; networkName?: string; approvedTargets?: string[]; + protectedTargetExceptions?: string[]; credentialMounts?: LabCredentialMount[]; isolation?: LabIsolationMode; microvmRuntimeClass?: string; @@ -165,6 +168,7 @@ export interface LabEgressEnforcementResult { cleanupAttempted?: boolean; cleanupSucceeded?: boolean; workloadRunning?: boolean; + policyId?: string; } export interface LabRuntimeResult { diff --git a/tests/integration/agent-lab-command.test.ts b/tests/integration/agent-lab-command.test.ts index b7cbc8303..01ad6a575 100644 --- a/tests/integration/agent-lab-command.test.ts +++ b/tests/integration/agent-lab-command.test.ts @@ -401,6 +401,9 @@ describe("agent lab command tool", () => { shellWorkspacePath: "/workspace", agentAccessEnabled: true, toolRunId: expect.any(String), + taskId: "task-1", + targetIds: ["target-1"], + researchRunId: undefined, env: undefined, onChunk: expect.any(Function), }); diff --git a/tests/integration/lab-network-evidence.test.ts b/tests/integration/lab-network-evidence.test.ts index bd4e10b78..44bffd62c 100644 --- a/tests/integration/lab-network-evidence.test.ts +++ b/tests/integration/lab-network-evidence.test.ts @@ -1,129 +1,227 @@ +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + import { describe, expect, it, vi } from "vitest"; +import { networkObservationsForCommand } from "../../src/server/agent-lab/command-runner"; +import { createSqlitePool, getDatabaseConfig } from "../../src/server/db/client"; -import type { - ArtifactServiceInstance, - CreateArtifactInput, -} from "../../src/server/evidence"; +import type { ArtifactServiceInstance, CreateArtifactInput } from "../../src/server/evidence"; import { - buildLabNetworkEvidenceJsonl, - createArtifactLabNetworkEvidenceRecorder, - LAB_NETWORK_EVIDENCE_SCHEMA, - parseLabNetworkObservation, + buildLabNetworkEvidenceJsonl, + createArtifactLabNetworkEvidenceRecorder, + LAB_NETWORK_EVIDENCE_SCHEMA, + parseLabNetworkObservation, } from "../../src/server/labs/network-evidence"; describe("lab network evidence", () => { - it("persists correlated observations through the central artifact service", async () => { - const createArtifact = vi.fn(async (_input: CreateArtifactInput) => ({ - id: "artifact-network-1", - projectId: "project-1", - threadId: "thread-1", - name: "network-evidence-policy-1.jsonl", - kind: "log", - indexing: { status: "not_attempted" as const, reason: "disabled" }, - })); - const writer = { - createArtifact, - createFinding: vi.fn(), - } satisfies ArtifactServiceInstance; - const recorder = createArtifactLabNetworkEvidenceRecorder(writer); + it("persists correlated observations through the central artifact service", async () => { + const createArtifact = vi.fn(async (_input: CreateArtifactInput) => ({ + id: "artifact-network-1", + projectId: "project-1", + threadId: "thread-1", + name: "network-evidence-policy-1.jsonl", + kind: "log", + indexing: { status: "not_attempted" as const, reason: "disabled" }, + })); + const writer = { + createArtifact, + createFinding: vi.fn(), + } satisfies ArtifactServiceInstance; + const receipts = { + admit: vi.fn(async () => "receipt-1"), + recorded: vi.fn(async () => undefined), + failed: vi.fn(async () => undefined), + }; + const recorder = createArtifactLabNetworkEvidenceRecorder(writer, receipts); + + await recorder.record({ + projectId: "project-1", + threadId: "thread-1", + taskId: "task-1", + toolRunId: "tool-run-1", + researchRunId: "research-run-1", + networkProfile: "approved-targets", + policyId: "policy-1", + observations: [ + { + observedAt: "2026-08-26T12:00:00.000Z", + event: "connection-attempt", + disposition: "denied", + destination: "203.0.113.8", + port: 443, + protocol: "tcp", + source: "external-egress-controller", + }, + ], + }); + + expect(createArtifact).toHaveBeenCalledOnce(); + expect(receipts.admit).toHaveBeenCalledOnce(); + expect(receipts.recorded).toHaveBeenCalledWith("receipt-1", "artifact-network-1"); + expect(createArtifact).toHaveBeenCalledWith( + expect.objectContaining({ + projectId: "project-1", + threadId: "thread-1", + taskId: "task-1", + toolRunId: "tool-run-1", + source: "network-observation", + indexForRag: false, + metadata: expect.objectContaining({ + policyId: "policy-1", + researchRunId: "research-run-1", + dispositions: ["denied"], + }), + }), + ); + const artifactInput = createArtifact.mock.calls[0]?.[0]; + expect(JSON.parse(String(artifactInput?.content).trim())).toMatchObject({ + schema: LAB_NETWORK_EVIDENCE_SCHEMA, + projectId: "project-1", + threadId: "thread-1", + toolRunId: "tool-run-1", + policyId: "policy-1", + disposition: "denied", + destination: "203.0.113.8", + port: 443, + }); + }); + + it("rejects controller observations from another project or policy", () => { + const line = buildLabNetworkEvidenceJsonl({ + projectId: "project-1", + networkProfile: "approved-targets", + policyId: "policy-1", + observations: [ + { + observedAt: "2026-08-26T12:00:00.000Z", + event: "dns-resolution", + disposition: "allowed", + hostname: "target.example", + resolvedAddresses: ["192.0.2.10"], + source: "external-egress-controller", + }, + ], + }); + const parsed = JSON.parse(line); + + expect(() => + parseLabNetworkObservation(parsed, { + projectId: "project-2", + policyId: "policy-1", + }), + ).toThrow(/active project and policy/); + expect(() => + parseLabNetworkObservation(parsed, { + projectId: "project-1", + policyId: "policy-2", + }), + ).toThrow(/active project and policy/); + }); - await recorder.record({ - projectId: "project-1", - threadId: "thread-1", - taskId: "task-1", - toolRunId: "tool-run-1", - researchRunId: "research-run-1", - networkProfile: "approved-targets", - policyId: "policy-1", - observations: [ - { - observedAt: "2026-08-26T12:00:00.000Z", - event: "connection-attempt", - disposition: "denied", - destination: "203.0.113.8", - port: 443, - protocol: "tcp", - source: "external-egress-controller", - }, - ], - }); + it("rejects incomplete connection observations", () => { + expect(() => + parseLabNetworkObservation( + { + schema: LAB_NETWORK_EVIDENCE_SCHEMA, + projectId: "project-1", + policyId: "policy-1", + observedAt: "2026-08-26T12:00:00.000Z", + event: "connection-attempt", + disposition: "allowed", + destination: "192.0.2.10", + source: "external-egress-controller", + }, + { projectId: "project-1", policyId: "policy-1" }, + ), + ).toThrow(/destination, port, and protocol/); + }); - expect(createArtifact).toHaveBeenCalledOnce(); - expect(createArtifact).toHaveBeenCalledWith( - expect.objectContaining({ - projectId: "project-1", - threadId: "thread-1", - taskId: "task-1", - toolRunId: "tool-run-1", - source: "network-observation", - indexForRag: false, - metadata: expect.objectContaining({ - policyId: "policy-1", - researchRunId: "research-run-1", - dispositions: ["denied"], - }), - }), - ); - const artifactInput = createArtifact.mock.calls[0]?.[0]; - expect(JSON.parse(String(artifactInput?.content).trim())).toMatchObject({ - schema: LAB_NETWORK_EVIDENCE_SCHEMA, - projectId: "project-1", - threadId: "thread-1", - toolRunId: "tool-run-1", - policyId: "policy-1", - disposition: "denied", - destination: "203.0.113.8", - port: 443, - }); - }); + it("projects actual command outcomes into DNS and connection observations", () => { + expect(networkObservationsForCommand("curl https://target.example:8443/health", true)).toEqual([ + expect.objectContaining({ + event: "connection-attempt", + disposition: "allowed", + destination: "target.example", + port: 8443, + }), + ]); + expect( + networkObservationsForCommand( + "dig target.example", + true, + "target.example. 60 IN A 198.51.100.9", + ), + ).toEqual([ + expect.objectContaining({ + event: "dns-resolution", + disposition: "allowed", + hostname: "target.example", + resolvedAddresses: ["198.51.100.9"], + }), + ]); + }); - it("rejects controller observations from another project or policy", () => { - const line = buildLabNetworkEvidenceJsonl({ - projectId: "project-1", - networkProfile: "approved-targets", - policyId: "policy-1", - observations: [ - { - observedAt: "2026-08-26T12:00:00.000Z", - event: "dns-resolution", - disposition: "allowed", - hostname: "target.example", - resolvedAddresses: ["192.0.2.10"], - source: "external-egress-controller", - }, - ], - }); - const parsed = JSON.parse(line); + it("retains a retryable receipt when artifact delivery is unavailable", async () => { + const previous = process.env.SQLITE_DATABASE_URL; + const directory = await mkdtemp(join(tmpdir(), "network-evidence-receipt-")); + process.env.SQLITE_DATABASE_URL = `sqlite://${join(directory, "app.sqlite")}`; + const pool = createSqlitePool(getDatabaseConfig()); + try { + await pool.query("INSERT INTO projects (id, name, slug) VALUES ($1, $2, $3)", [ + "project-receipt", + "Receipt project", + "receipt-project", + ]); + const recorder = createArtifactLabNetworkEvidenceRecorder({ + createArtifact: vi.fn(async () => { + throw new Error("artifact store unavailable"); + }), + }); - expect(() => - parseLabNetworkObservation(parsed, { - projectId: "project-2", - policyId: "policy-1", - }), - ).toThrow(/active project and policy/); - expect(() => - parseLabNetworkObservation(parsed, { - projectId: "project-1", - policyId: "policy-2", - }), - ).toThrow(/active project and policy/); - }); + await expect( + recorder.record({ + projectId: "project-receipt", + toolRunId: "run-1", + researchRunId: "research-1", + targetIds: ["target-1"], + networkProfile: "approved-targets", + policyId: "a".repeat(64), + observations: [ + { + observedAt: new Date().toISOString(), + event: "connection-attempt", + disposition: "denied", + destination: "203.0.113.8", + port: 443, + protocol: "tcp", + source: "verified-egress-command-outcome", + }, + ], + }), + ).rejects.toThrow("artifact store unavailable"); - it("rejects incomplete connection observations", () => { - expect(() => - parseLabNetworkObservation( - { - schema: LAB_NETWORK_EVIDENCE_SCHEMA, - projectId: "project-1", - policyId: "policy-1", - observedAt: "2026-08-26T12:00:00.000Z", - event: "connection-attempt", - disposition: "allowed", - destination: "192.0.2.10", - source: "external-egress-controller", - }, - { projectId: "project-1", policyId: "policy-1" }, - ), - ).toThrow(/destination, port, and protocol/); - }); + const receipts = await pool.query<{ + status: string; + policy_id: string; + payload: { toolRunId?: string; researchRunId?: string; targetIds?: string[] }; + }>("SELECT status, policy_id, payload FROM network_evidence_receipts"); + expect(receipts.rows).toEqual([ + expect.objectContaining({ + status: "failed", + policy_id: "a".repeat(64), + payload: expect.objectContaining({ + toolRunId: "run-1", + researchRunId: "research-1", + targetIds: ["target-1"], + }), + }), + ]); + } finally { + await pool.end(); + if (previous === undefined) delete process.env.SQLITE_DATABASE_URL; + else process.env.SQLITE_DATABASE_URL = previous; + await rm(directory, { recursive: true, force: true }); + } + }); }); diff --git a/tests/integration/lab-runtime.test.ts b/tests/integration/lab-runtime.test.ts index 25c06dec2..0b4374d23 100644 --- a/tests/integration/lab-runtime.test.ts +++ b/tests/integration/lab-runtime.test.ts @@ -206,9 +206,7 @@ describe("project lab Docker runtime primitives", () => { approvedTargets: ["tcp://files.corpo.internal:9090"], }); - expect(script).toContain( - "iptables -A OUTPUT -p tcp -d files.corpo.internal --dport 9090 -j ACCEPT", - ); + expect(script).toContain('iptables -A OUTPUT -p tcp -d "$ip" --dport 9090 -j ACCEPT'); expect(script).not.toContain( "iptables -A OUTPUT -p tcp -d files.corpo.internal --dport 443 -j ACCEPT", ); @@ -219,6 +217,7 @@ describe("project lab Docker runtime primitives", () => { boundary: "human", profileId: "approved-targets", approvedTargets: ["192.168.56.0/24"], + protectedTargetExceptions: ["192.168.56.0/24"], }); expect(script).toContain("iptables -A OUTPUT -d 192.168.56.0/24 -j ACCEPT"); @@ -237,20 +236,20 @@ describe("project lab Docker runtime primitives", () => { ...humanOptions, networkProfile: "approved-targets" as const, approvedTargets: ["tcp://host.docker.internal:31337"], + protectedTargetExceptions: ["host.docker.internal"], }; const workload = buildRunLabCommand(options); const script = buildEgressIptablesScript({ boundary: options.boundary, profileId: options.networkProfile, approvedTargets: options.approvedTargets, + protectedTargetExceptions: options.protectedTargetExceptions, }); expect(workload.args).toEqual( expect.arrayContaining(["--add-host", "host.docker.internal:host-gateway"]), ); - expect(script).toContain( - "iptables -A OUTPUT -p tcp -d host.docker.internal --dport 31337 -j ACCEPT", - ); + expect(script).toContain('iptables -A OUTPUT -p tcp -d "$ip" --dport 31337 -j ACCEPT'); expect(script).toContain("iptables -P OUTPUT DROP"); expect(() => buildEgressIptablesScript({ @@ -282,11 +281,31 @@ describe("project lab Docker runtime primitives", () => { egressEnforcerImage(options), ]), ); - expect(run.args.join(" ")).toMatch(/exploit-hunter\.egress-policy-sha256=[a-f0-9]{16}/); + expect(commands[0]?.args[0]).toBe("pause"); + expect(commands[1]?.args.slice(0, 2)).toEqual(["rm", "-f"]); + expect(run.args.join(" ")).toMatch(/exploit-hunter\.egress-policy-sha256=[a-f0-9]{64}/); expect(enforce.args.join(" ")).toContain("EXPLOIT_HUNTER_EGRESS_DENIED:"); expect(enforce.args).toContain("/bin/sh"); }); + it("rejects protected targets unless the exact host has a scoped exception", () => { + expect(() => + buildEgressIptablesScript({ + boundary: "agent", + profileId: "approved-targets", + approvedTargets: ["http://169.254.169.254:80"], + }), + ).toThrow(/protected address space/); + expect( + buildEgressIptablesScript({ + boundary: "agent", + profileId: "approved-targets", + approvedTargets: ["http://169.254.169.254:80"], + protectedTargetExceptions: ["169.254.169.254"], + }), + ).toContain("-d 169.254.169.254 --dport 80 -j ACCEPT"); + }); + it("derives an egress controller from the eval-selected workload image", () => { const options = { ...humanOptions, @@ -453,10 +472,7 @@ describe("project lab Docker runtime primitives", () => { ); }); - it("keeps a development workload usable with durable degraded enforcement state", async () => { - const warnings: string[] = []; - const originalWarn = console.warn; - console.warn = (message?: unknown) => warnings.push(String(message)); + it("fails a development workload closed when policy verification is unavailable", async () => { const runner: DockerCommandRunner = { async isAvailable() { return true; @@ -468,36 +484,25 @@ describe("project lab Docker runtime primitives", () => { return { stdout: "", stderr: "", exitCode: 0 }; }, }; - try { - const runtime = new ProjectLabRuntime({ - mode: "docker", - egressEnforcementMode: "development", - runner, - }); - const result = await runtime.start({ + const runtime = new ProjectLabRuntime({ + mode: "docker", + egressEnforcementMode: "development", + runner, + }); + await expect( + runtime.start({ projectId: "development-egress-failure", threadId: "thread-1", boundary: "human", networkProfile: "package-egress", - }); - - expect(result.egressEnforcement).toMatchObject({ - requestedProfile: "package-egress", - mode: "development", - disposition: "unenforced-development", - effectiveUnrestricted: true, - workloadRunning: true, - }); - expect(JSON.parse(warnings[0]!)).toMatchObject({ - event: "lab.egress.unenforced-development", - projectId: "development-egress-failure", - threadId: "thread-1", - requestedProfile: "package-egress", - effectiveUnrestricted: true, - }); - } finally { - console.warn = originalWarn; - } + }), + ).rejects.toMatchObject({ + enforcement: expect.objectContaining({ + disposition: "enforcement-failed-safe", + effectiveUnrestricted: false, + workloadRunning: false, + }), + }); }); it("distinguishes managed cleanup failure from successful fail-safe cleanup", async () => { @@ -579,8 +584,9 @@ describe("project lab Docker runtime primitives", () => { "container", "rm", "run", - "container", - "start", + "pause", + "rm", + "run", "exec", "exec", ]); diff --git a/tests/integration/project-lab.test.ts b/tests/integration/project-lab.test.ts index 35eb31f84..83ab56d28 100644 --- a/tests/integration/project-lab.test.ts +++ b/tests/integration/project-lab.test.ts @@ -132,6 +132,7 @@ describe("project lab lifecycle", () => { const service = new ProjectLabService( repository, new ProjectLabRuntime({ mode: "docker", runner }), + noopNetworkEvidence, ); const firstStart = service.start("project-1"); @@ -159,6 +160,7 @@ describe("project lab lifecycle", () => { const service = new ProjectLabService( repository, new ProjectLabRuntime({ mode: "docker", runner: new FailingDockerRunner() }), + noopNetworkEvidence, ); await expect(service.start("project-1")).rejects.toThrow("managed image build failed"); @@ -171,6 +173,49 @@ describe("project lab lifecycle", () => { }); }); + it("does not admit a workload when durable enforcement evidence cannot be recorded", async () => { + const repository = new InMemoryProjectLabRepository(); + const commands: DockerCommandSpec[] = []; + const runner: DockerCommandRunner = { + async isAvailable() { + return true; + }, + async run(command) { + commands.push(command); + return { stdout: "", stderr: "", exitCode: 0 }; + }, + }; + const service = new ProjectLabService( + repository, + new ProjectLabRuntime({ mode: "docker", runner }), + { + record: vi.fn(async () => { + throw new Error("evidence unavailable"); + }), + }, + ); + + await expect(service.start("project-1")).rejects.toThrow("evidence unavailable"); + await expect(service.status("project-1")).resolves.toMatchObject({ + status: "failed", + lab: { + runtime_id: null, + container_id: null, + failure_reason: "evidence unavailable", + }, + }); + expect(commands).not.toEqual( + expect.arrayContaining([ + expect.objectContaining({ args: expect.arrayContaining(["unpause"]) }), + ]), + ); + expect(commands).toEqual( + expect.arrayContaining([ + expect.objectContaining({ args: expect.arrayContaining(["rm", "-f"]) }), + ]), + ); + }); + it("releases an expired provisioning lease so automatic startup can resume", async () => { const repository = new InMemoryProjectLabRepository(); repository.rows.push( @@ -279,8 +324,8 @@ describe("project lab lifecycle", () => { "container", "start", "run", - "container", - "start", + "pause", + "rm", "run", "exec", ]); @@ -331,6 +376,7 @@ describe("project lab lifecycle", () => { networkProfile: "approved-targets", networkName: "fixture-network", approvedTargets: ["192.168.56.0/24"], + protectedTargetExceptions: ["192.168.56.0/24"], }); const commands = running.lab?.runtime_metadata.commands as Array<{ args: string[] }>; @@ -338,6 +384,7 @@ describe("project lab lifecycle", () => { networkProfile: "approved-targets", networkName: "fixture-network", approvedTargets: ["192.168.56.0/24"], + protectedTargetExceptions: ["192.168.56.0/24"], }); expect(commands).toEqual( expect.arrayContaining([ @@ -348,7 +395,7 @@ describe("project lab lifecycle", () => { ); }); - it("persists development egress degradation so reloads cannot report it as enforced", async () => { + it("fails closed in development and records enforcement-unavailable evidence", async () => { const repository = new InMemoryProjectLabRepository(); const runtime = new ProjectLabRuntime({ mode: "docker", @@ -360,16 +407,18 @@ describe("project lab lifecycle", () => { record: recordNetworkEvidence, }); - const started = await service.start("project-1", { networkProfile: "package-egress" }); + await expect(service.start("project-1", { networkProfile: "package-egress" })).rejects.toThrow( + /Failed to enforce external egress policy/, + ); const reloaded = await service.status("project-1"); - expect(started.status).toBe("running"); + expect(reloaded.status).toBe("failed"); expect(reloaded.lab?.runtime_metadata.egressEnforcement).toMatchObject({ requestedProfile: "package-egress", mode: "development", - disposition: "unenforced-development", - effectiveUnrestricted: true, - workloadRunning: true, + disposition: "enforcement-failed-safe", + effectiveUnrestricted: false, + workloadRunning: false, }); expect(recordNetworkEvidence).toHaveBeenCalledWith( expect.objectContaining({ @@ -392,7 +441,7 @@ describe("project lab lifecycle", () => { egressEnforcementMode: "managed", runner: egressFailureRunner(), }); - const service = new ProjectLabService(repository, runtime); + const service = new ProjectLabService(repository, runtime, noopNetworkEvidence); await expect(service.start("project-1", { networkProfile: "package-egress" })).rejects.toThrow( /Failed to enforce external egress policy/, @@ -475,7 +524,13 @@ describe("project lab lifecycle", () => { }); const buildTestLabService = (repository: ProjectLabRepository) => - new ProjectLabService(repository, new ProjectLabRuntime({ mode: "dry-run" })); + new ProjectLabService( + repository, + new ProjectLabRuntime({ mode: "dry-run" }), + noopNetworkEvidence, + ); + +const noopNetworkEvidence = { record: vi.fn(async () => undefined) }; const egressFailureRunner = (): DockerCommandRunner => ({ async isAvailable() { diff --git a/tests/live/lab-egress-enforcement-mode.test.ts b/tests/live/lab-egress-enforcement-mode.test.ts index 351bab62c..d5e799bdc 100644 --- a/tests/live/lab-egress-enforcement-mode.test.ts +++ b/tests/live/lab-egress-enforcement-mode.test.ts @@ -1,4 +1,5 @@ import { execFile } from "node:child_process"; +import { createServer } from "node:http"; import { promisify } from "node:util"; import { afterEach, describe, expect, it } from "vitest"; @@ -60,7 +61,7 @@ live("live lab egress enforcement failure modes", () => { ).rejects.toThrow(); }); - it("development mode leaves a conspicuously degraded but usable workload", async () => { + it("development mode also removes the workload when policy installation fails", async () => { const projectId = `live-development-egress-${Date.now()}`; projectIds.add(projectId); const options = { @@ -76,21 +77,108 @@ live("live lab egress enforcement failure modes", () => { runner: failPolicyRunner(), }); - const result = await runtime.start(options); - expect(result.egressEnforcement).toMatchObject({ - disposition: "unenforced-development", - effectiveUnrestricted: true, - workloadRunning: true, + await expect(runtime.start(options)).rejects.toMatchObject({ + enforcement: { + disposition: "enforcement-failed-safe", + effectiveUnrestricted: false, + workloadRunning: false, + }, }); - const usable = await execFileAsync("docker", [ - "exec", - labContainerIdentity(options).containerName, - "sh", - "-lc", - "printf usable", - ]); - expect(usable.stdout).toBe("usable"); + await expect( + execFileAsync("docker", [ + "container", + "inspect", + labContainerIdentity(options).containerName, + ]), + ).rejects.toThrow(); }); + + it("admits the approved endpoint, denies bypasses, and replaces the policy on profile change", async () => { + const first = createServer((_request, response) => response.end("first")); + const second = createServer((_request, response) => response.end("second")); + await Promise.all([ + new Promise((resolve) => first.listen(0, "0.0.0.0", resolve)), + new Promise((resolve) => second.listen(0, "0.0.0.0", resolve)), + ]); + const firstPort = (first.address() as { port: number }).port; + const secondPort = (second.address() as { port: number }).port; + const projectId = `live-egress-policy-${Date.now()}`; + projectIds.add(projectId); + const base = { + projectId, + threadId: "live-egress-thread", + boundary: "human" as const, + image: "alpine:3.20", + networkProfile: "approved-targets" as const, + protectedTargetExceptions: ["host.docker.internal"], + }; + const runtime = new ProjectLabRuntime({ mode: "docker", egressEnforcementMode: "managed" }); + const container = labContainerIdentity(base).containerName; + try { + const firstPolicy = await runtime.start({ + ...base, + approvedTargets: [`http://host.docker.internal:${firstPort}`], + }); + expect(firstPolicy.egressEnforcement.policyId).toMatch(/^[a-f0-9]{64}$/); + await runtime.admitWorkload({ + ...base, + approvedTargets: [`http://host.docker.internal:${firstPort}`], + }); + await expect( + execFileAsync("docker", [ + "exec", + container, + "wget", + "-qO-", + `http://host.docker.internal:${firstPort}`, + ]), + ).resolves.toMatchObject({ stdout: "first" }); + + for (const command of [ + ["nc", "-z", "-w", "2", "1.1.1.1", "80"], + ["nslookup", "example.com", "8.8.8.8"], + ["nc", "-z", "-w", "2", "169.254.169.254", "80"], + ["sh", "-lc", "http_proxy=http://1.1.1.1:8080 wget -qO- http://example.com"], + ]) { + await expect(execFileAsync("docker", ["exec", container, ...command])).rejects.toThrow(); + } + + const secondPolicy = await runtime.start({ + ...base, + approvedTargets: [`http://host.docker.internal:${secondPort}`], + }); + await runtime.admitWorkload({ + ...base, + approvedTargets: [`http://host.docker.internal:${secondPort}`], + }); + expect(secondPolicy.egressEnforcement.policyId).not.toBe( + firstPolicy.egressEnforcement.policyId, + ); + await expect( + execFileAsync("docker", [ + "exec", + container, + "wget", + "-qO-", + `http://host.docker.internal:${firstPort}`, + ]), + ).rejects.toThrow(); + await expect( + execFileAsync("docker", [ + "exec", + container, + "wget", + "-qO-", + `http://host.docker.internal:${secondPort}`, + ]), + ).resolves.toMatchObject({ stdout: "second" }); + } finally { + await Promise.all([ + new Promise((resolve) => first.close(() => resolve())), + new Promise((resolve) => second.close(() => resolve())), + ]); + } + }, 120_000); }); function failPolicyRunner(): DockerCommandRunner { From 51df9778915f762b7bcb26bbc097c68a2a384598 Mon Sep 17 00:00:00 2001 From: Dan Levy Date: Sun, 30 Aug 2026 18:40:06 -0400 Subject: [PATCH 4/4] fix(labs): close restricted network admission races --- docs/lab-runtime-hardening.md | 15 +- package.json | 1 + scripts/repair-lab-evidence.ts | 13 ++ src/lib/ids.ts | 2 + src/server/agent-lab/command-runner.ts | 151 +++++++++----- ...830190000_lab_runtime_cleanup_receipts.sql | 24 +++ src/server/db/postgres-migrate.ts | 1 + src/server/labs/cleanup-receipts.ts | 82 ++++++++ src/server/labs/docker-plan.ts | 64 ++++-- src/server/labs/network-evidence.ts | 158 +++++++++++---- src/server/labs/network-profiles.ts | 1 + src/server/labs/repository.ts | 14 ++ src/server/labs/runtime.ts | 71 +++++-- src/server/labs/service.ts | 186 +++++++++++++++--- src/server/labs/types.ts | 1 + .../agent-command-tool-run-lifecycle.test.ts | 2 +- .../integration/lab-cleanup-receipts.test.ts | 70 +++++++ .../integration/lab-network-evidence.test.ts | 136 +++++++++++-- tests/integration/lab-runtime.test.ts | 34 ++-- tests/integration/project-lab.test.ts | 29 ++- .../live/lab-egress-enforcement-mode.test.ts | 1 + 21 files changed, 867 insertions(+), 189 deletions(-) create mode 100644 scripts/repair-lab-evidence.ts create mode 100644 src/server/db/migrations/20260830190000_lab_runtime_cleanup_receipts.sql create mode 100644 src/server/labs/cleanup-receipts.ts create mode 100644 tests/integration/lab-cleanup-receipts.test.ts diff --git a/docs/lab-runtime-hardening.md b/docs/lab-runtime-hardening.md index 3eb6a81ec..62180b74b 100644 --- a/docs/lab-runtime-hardening.md +++ b/docs/lab-runtime-hardening.md @@ -86,14 +86,15 @@ The current Kali lab container is appropriate for bounded static analysis, passi When a network profile has egress rules defined, the lab runtime: -1. Starts a dedicated egress-controller sidecar in the lab's network namespace. -2. Grants `NET_ADMIN` only to that sidecar; the workload retains `--cap-drop ALL` and never receives `NET_ADMIN` or `NET_RAW`. -3. Applies the iptables policy through the controller, then records the network-profile and a SHA-256 policy fingerprint as Docker labels on the controller. -4. The script sets default DROP policies on INPUT, FORWARD, and OUTPUT chains, then adds ACCEPT rules for: +1. Creates the workload with `--network none`, starts it without a network, pauses it, and only then attaches the approved Docker network. This prevents the image entrypoint from having an unfiltered startup window. +2. Starts a dedicated egress-controller sidecar in the paused lab's network namespace. +3. Grants `NET_ADMIN` only to that sidecar; the workload retains `--cap-drop ALL` and never receives `NET_ADMIN` or `NET_RAW`. +4. Applies the iptables policy through the controller, then records the network-profile and a SHA-256 policy fingerprint as Docker labels on the controller. +5. The script sets default DROP policies on IPv4 and IPv6 INPUT, FORWARD, and OUTPUT chains. If IPv6 is active but `ip6tables` is unavailable, admission fails closed. It then adds IPv4 ACCEPT rules for: - Established/related connections (so response traffic is allowed). - Loopback traffic (localhost communication within the container). - Each egress rule destination:port/protocol. -4. A final REJECT rule on OUTPUT drops all other outbound traffic. +6. A final REJECT rule on OUTPUT drops all other outbound traffic. ### Approved-targets enforcement @@ -105,7 +106,7 @@ Denied packets are rate-limited and logged with the `EXPLOIT_HUNTER_EGRESS_DENIE Lab start and restart now save the external controller's enforcement result through the central Artifact service as project-scoped JSONL. A durable database receipt is admitted first; artifact delivery failure leaves a retryable failed receipt and prevents the restricted workload from becoming `running`. Each record uses the versioned `exploit-hunter.lab-network-evidence.v1` schema and can carry project, thread, task, target, tool-run, research-run, network-profile, and full policy-digest correlation. Enforcement failures are recorded as `enforcement-unavailable`; successful policy installation is recorded separately as `policy-enforced` and is not represented as proof that a connection was allowed. -Approved command execution emits correlated DNS and connection-attempt outcomes after the command completes while the verified controller policy remains active. These observations describe the attempted destination and command outcome; they are not a packet-complete transcript. Controller observations must match the active project and full policy digest before ingestion. Opt-in mitmproxy captures remain the application-level traffic record, with the protocol and bypass limitations described above. +Before a network-capable approved command starts, execution admits an intent receipt bound to the server-owned tool run and current policy snapshot. After execution, deltas from the controller's packet counters are persisted as grounded allowed/denied policy decisions; shell text and exit status are not treated as network facts. A failed finalization leaves the pre-execution receipt retryable and does not invite replay of the command. Counter deltas are not a packet-complete transcript and may not identify every destination. Controller observations must retain the same project and full policy digest through finalization. `pnpm labs:repair-evidence` retries artifact delivery and orphan-runtime cleanup receipts. ### Package-egress enforcement @@ -125,7 +126,7 @@ Network profile changes are bound to durable approvals. The `agentLabCommandTool ## Limitations -- iptables rules are applied after container start and are lost on container restart. The external controller re-applies rules on every start. +- iptables rules are lost on container recreation. Restricted workloads are recreated offline and paused while the external controller reapplies and verifies the rules on every start. - DNS resolution for approved targets uses the container's configured DNS resolver. The iptables rules match against resolved IP addresses at connection time, not domain names. This means the hostname-based allowlisting is resolved at the time each connection is made. - The firewall controller shares the lab network namespace, but the workload has no firewall administration capability. A host-level or dedicated-network firewall remains the stronger option against a Docker daemon or kernel compromise. - Namespace/cgroup isolation shares the host kernel with the lab workload. For labs where a kernel-level container escape is an unacceptable risk (for example, running untrusted exploit code or malware samples), use `microvm` isolation (see above) so the lab runs in its own guest kernel instead of the host's. diff --git a/package.json b/package.json index 9ecf121b5..48210cb8c 100644 --- a/package.json +++ b/package.json @@ -26,6 +26,7 @@ "db:reset": "tsx scripts/db-reset.ts", "db:new": "tsx scripts/db-new-migration.ts", "db:seed": "tsx scripts/db-seed.ts", + "labs:repair-evidence": "tsx scripts/repair-lab-evidence.ts", "setup": "tsx scripts/setup.ts", "embeddings:init": "tsx scripts/init-embedding-runtime.ts", "rag:reindex": "tsx scripts/reindex-rag.ts", diff --git a/scripts/repair-lab-evidence.ts b/scripts/repair-lab-evidence.ts new file mode 100644 index 000000000..3ae2334b0 --- /dev/null +++ b/scripts/repair-lab-evidence.ts @@ -0,0 +1,13 @@ +import { retryLabRuntimeCleanupReceipts } from "../src/server/labs/cleanup-receipts"; +import { retryLabNetworkEvidenceReceipts } from "../src/server/labs/network-evidence"; + +const limitArg = process.argv.find((value) => value.startsWith("--limit=")); +const parsedLimit = limitArg ? Number(limitArg.slice("--limit=".length)) : 25; +const limit = Number.isFinite(parsedLimit) ? parsedLimit : 25; + +const [evidence, cleanup] = await Promise.all([ + retryLabNetworkEvidenceReceipts({ limit }), + retryLabRuntimeCleanupReceipts({ limit }), +]); + +process.stdout.write(`${JSON.stringify({ evidence, cleanup })}\n`); diff --git a/src/lib/ids.ts b/src/lib/ids.ts index 9eebd72ca..46668dd45 100644 --- a/src/lib/ids.ts +++ b/src/lib/ids.ts @@ -36,6 +36,7 @@ export type AppIdKind = | "modelConfig" | "negativeResult" | "networkEvidenceReceipt" + | "labRuntimeCleanup" | "plan" | "project" | "queuedMessage" @@ -95,6 +96,7 @@ const ID_SPECS = { modelConfig: { prefix: "mdl" }, negativeResult: { prefix: "neg" }, networkEvidenceReceipt: { prefix: "ner", length: HIGH_CHURN_ID_LENGTH }, + labRuntimeCleanup: { prefix: "lrc", length: HIGH_CHURN_ID_LENGTH }, plan: { prefix: "pln" }, project: { prefix: "prj" }, queuedMessage: { prefix: "que" }, diff --git a/src/server/agent-lab/command-runner.ts b/src/server/agent-lab/command-runner.ts index e2c95d9f3..a0ddd0c9a 100644 --- a/src/server/agent-lab/command-runner.ts +++ b/src/server/agent-lab/command-runner.ts @@ -1,5 +1,6 @@ -import { spawn } from "node:child_process"; +import { execFile, spawn } from "node:child_process"; import { randomUUID } from "node:crypto"; +import { promisify } from "node:util"; import type { CommandResult, @@ -13,7 +14,11 @@ import { rewriteRemoteWorkspaceCommand, } from "../compute"; import { labContainerIdentity } from "../labs/docker-plan"; -import { getProjectLabStatus, recordProjectLabNetworkObservations } from "../labs/service"; +import { + completeProjectLabNetworkIntent, + getProjectLabStatus, + openProjectLabNetworkIntent, +} from "../labs/service"; import { getLabSharedTerminalRegistry, getLabSshSharedTerminalRegistry, @@ -23,6 +28,8 @@ import { LEAD_AGENT_ID, sharedTerminalSessionId } from "../terminal/shared-sessi import { getDefaultThreadWorkspaceService } from "../workspaces"; import type { ThreadTargetConfig } from "../workspaces/target-mode"; +const execFileAsync = promisify(execFile); + export type AgentLabCommandStreamChunk = { stream: "stdout" | "stderr"; data: string; @@ -532,6 +539,23 @@ export const projectAgentLabCommandRunner: ProjectAgentLabCommandRunner = { agentId, agentAccessEnabled, }); + const networkCapable = commandMayUseNetwork(fullCommand); + const networkIntent = networkCapable + ? await openProjectLabNetworkIntent({ + projectId, + threadId, + taskId, + targetIds, + toolRunId, + researchRunId, + command: fullCommand, + }) + : undefined; + const beforeCounters = networkIntent + ? await captureEgressControllerCounters(containerName, networkIntent.policyId).catch( + () => undefined, + ) + : undefined; const commandResult = await session.runCommandTurn({ actor: `agent:${agentId}`, command: buildSharedTerminalCommand(fullCommand, cwd, commandEnv(env)), @@ -565,20 +589,39 @@ export const projectAgentLabCommandRunner: ProjectAgentLabCommandRunner = { }; safeToReleaseWorkspace = !commandResult.timedOut || commandResult.termination?.confirmed === true; - const networkObservations = networkObservationsForCommand( - fullCommand, - commandResult.exitCode === 0, - commandResult.stdout, - ); - if (networkObservations.length > 0) { - await recordProjectLabNetworkObservations({ + if (networkIntent) { + const afterCounters = await captureEgressControllerCounters( + containerName, + networkIntent.policyId, + ).catch(() => undefined); + const networkObservations = networkObservationsFromControllerCounters( + beforeCounters, + afterCounters, + ); + await completeProjectLabNetworkIntent({ + intent: networkIntent, projectId, threadId, taskId, targetIds, toolRunId, researchRunId, - observations: networkObservations, + observations: + networkObservations.length > 0 + ? networkObservations + : [ + { + observedAt: new Date().toISOString(), + event: "enforcement-state", + disposition: "policy-enforced", + reason: "No attributable egress-controller counter delta was observed.", + source: "external-egress-controller-counters", + }, + ], + }).catch((error) => { + console.error( + `[agent-lab] Network evidence finalization is pending repair for ${networkIntent.receiptId}: ${error instanceof Error ? error.message : String(error)}`, + ); }); } return output; @@ -588,43 +631,61 @@ export const projectAgentLabCommandRunner: ProjectAgentLabCommandRunner = { }, }; -export function networkObservationsForCommand(command: string, succeeded: boolean, stdout = "") { - const observedAt = new Date().toISOString(); - const disposition = succeeded ? ("allowed" as const) : ("denied" as const); - const observations: import("../labs/network-evidence").LabNetworkObservation[] = []; - const seen = new Set(); - for (const match of command.matchAll(/\b(https?):\/\/([^\s/'";]+)/gi)) { - try { - const url = new URL(`${match[1]}://${match[2]}`); - const port = url.port ? Number(url.port) : url.protocol === "http:" ? 80 : 443; - const key = `${url.hostname}:${port}`; - if (seen.has(key)) continue; - seen.add(key); - observations.push({ - observedAt, - event: "connection-attempt", - disposition, - destination: url.hostname, - hostname: url.hostname, - port, - protocol: "tcp", - source: "verified-egress-command-outcome", - }); - } catch { - // Ignore tokens that only resemble URLs. - } +export const commandMayUseNetwork = (command: string) => + /(?:\bhttps?:\/\/|\b(?:curl|wget|nc|ncat|netcat|dig|host|nslookup|nmap|masscan|naabu|ssh|scp|sftp|ftp|telnet|openssl\s+s_client)\b)/i.test( + command, + ); + +type EgressCounterSnapshot = { + policyId: string; + capturedAt: string; + rules: Map; +}; + +async function captureEgressControllerCounters( + workloadContainer: string, + policyId: string, +): Promise { + const result = await execFileAsync("docker", [ + "exec", + `${workloadContainer}-egress-enforcer`, + "iptables-save", + "-c", + "-t", + "filter", + ]); + return parseEgressControllerCounters(result.stdout, policyId); +} + +export function parseEgressControllerCounters( + text: string, + policyId: string, +): EgressCounterSnapshot { + const rules = new Map(); + for (const line of text.split("\n")) { + const match = /^\[(\d+):\d+\]\s+(-A OUTPUT.*-j (?:ACCEPT|REJECT))$/.exec(line.trim()); + if (!match) continue; + rules.set(match[2]!, { packets: Number(match[1]), target: match[2]! }); } - const dns = /(?:^|\s)(?:dig|host|nslookup)\s+([^\s;&|]+)/i.exec(command)?.[1]; - if (dns) { + return { policyId, capturedAt: new Date().toISOString(), rules }; +} + +export function networkObservationsFromControllerCounters( + before: EgressCounterSnapshot | undefined, + after: EgressCounterSnapshot | undefined, +): import("../labs/network-evidence").LabNetworkObservation[] { + if (!before || !after || before.policyId !== after.policyId) return []; + const observations: import("../labs/network-evidence").LabNetworkObservation[] = []; + for (const [key, current] of after.rules) { + const delta = current.packets - (before.rules.get(key)?.packets ?? 0); + if (delta <= 0) continue; observations.push({ - observedAt, - event: "dns-resolution", - disposition, - hostname: dns, - resolvedAddresses: succeeded - ? [...new Set(stdout.match(/\b(?:\d{1,3}\.){3}\d{1,3}\b/g) ?? [])] - : [], - source: "verified-egress-command-outcome", + observedAt: after.capturedAt, + event: "policy-counter", + disposition: / -j ACCEPT$/.test(current.target) ? "allowed" : "denied", + packetCount: delta, + reason: current.target, + source: "external-egress-controller-counters", }); } return observations; diff --git a/src/server/db/migrations/20260830190000_lab_runtime_cleanup_receipts.sql b/src/server/db/migrations/20260830190000_lab_runtime_cleanup_receipts.sql new file mode 100644 index 000000000..56d831dcd --- /dev/null +++ b/src/server/db/migrations/20260830190000_lab_runtime_cleanup_receipts.sql @@ -0,0 +1,24 @@ +-- migrate:up +CREATE TABLE IF NOT EXISTS lab_runtime_cleanup_receipts ( + id text PRIMARY KEY, + project_id text NOT NULL REFERENCES projects(id) ON DELETE CASCADE, + lab_id text NOT NULL REFERENCES project_labs(id) ON DELETE CASCADE, + runtime_locator text NOT NULL, + status text NOT NULL CHECK (status IN ('pending', 'completed')), + payload text NOT NULL, + attempts integer NOT NULL DEFAULT 0, + last_error_message text, + created_at text NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at text NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +CREATE INDEX IF NOT EXISTS lab_runtime_cleanup_delivery_idx + ON lab_runtime_cleanup_receipts(status, updated_at, id); + +CREATE UNIQUE INDEX IF NOT EXISTS artifacts_network_evidence_receipt_name_idx + ON artifacts(project_id, name) + WHERE name LIKE 'network-evidence-ner%'; + +-- migrate:down +DROP TABLE IF EXISTS lab_runtime_cleanup_receipts; +DROP INDEX IF EXISTS artifacts_network_evidence_receipt_name_idx; diff --git a/src/server/db/postgres-migrate.ts b/src/server/db/postgres-migrate.ts index c30892a96..57dc6b7a2 100644 --- a/src/server/db/postgres-migrate.ts +++ b/src/server/db/postgres-migrate.ts @@ -178,6 +178,7 @@ const APP_TABLES = [ "passive_policy_shadow_records", "blockers", "network_evidence_receipts", + "lab_runtime_cleanup_receipts", ] as const; const POSTGRES_JSON_TEXT_EXCEPTIONS = new Set([ // This is a constrained workflow enum, despite sharing a legacy SQLite JSON-column name. diff --git a/src/server/labs/cleanup-receipts.ts b/src/server/labs/cleanup-receipts.ts new file mode 100644 index 000000000..b080296c2 --- /dev/null +++ b/src/server/labs/cleanup-receipts.ts @@ -0,0 +1,82 @@ +import { createId } from "../../lib/ids"; +import { withDatabase } from "../db/client"; +import { ProjectLabRuntime } from "./runtime"; +import type { LabContainerOptions } from "./types"; + +export async function recordLabRuntimeCleanupFailure(input: { + projectId: string; + labId: string; + runtimeLocator: string; + options: LabContainerOptions; + error: unknown; +}): Promise { + const id = createId("labRuntimeCleanup"); + const message = input.error instanceof Error ? input.error.message : String(input.error); + await withDatabase((db) => + db.query( + `INSERT INTO lab_runtime_cleanup_receipts + (id, project_id, lab_id, runtime_locator, status, payload, attempts, last_error_message) + VALUES ($1, $2, $3, $4, 'pending', $5, 1, $6)`, + [ + id, + input.projectId, + input.labId, + input.runtimeLocator, + JSON.stringify(input.options), + message.slice(0, 4_000), + ], + ), + ); + return id; +} + +export async function retryLabRuntimeCleanupReceipts( + input: { limit?: number; runtime?: ProjectLabRuntime } = {}, +): Promise<{ attempted: number; completed: number; failed: number }> { + const limit = Math.max(1, Math.min(100, Math.floor(input.limit ?? 25))); + const runtime = input.runtime ?? new ProjectLabRuntime({ mode: "docker" }); + const rows = await withDatabase((db) => + db.query<{ id: string; payload: LabContainerOptions; attempts: number }>( + `SELECT id, payload, attempts FROM lab_runtime_cleanup_receipts + WHERE status = 'pending' ORDER BY updated_at, id LIMIT $1`, + [limit], + ), + ); + let completed = 0; + let failed = 0; + for (const row of rows.rows) { + const claimed = await withDatabase((db) => + db.query<{ id: string }>( + `UPDATE lab_runtime_cleanup_receipts + SET attempts = attempts + 1, updated_at = CURRENT_TIMESTAMP + WHERE id = $1 AND status = 'pending' AND attempts = $2 + RETURNING id`, + [row.id, row.attempts], + ), + ); + if (!claimed.rows[0]) continue; + try { + await runtime.destroyContainer(row.payload); + await withDatabase((db) => + db.query( + `UPDATE lab_runtime_cleanup_receipts + SET status = 'completed', last_error_message = NULL, updated_at = CURRENT_TIMESTAMP + WHERE id = $1`, + [row.id], + ), + ); + completed += 1; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + await withDatabase((db) => + db.query( + `UPDATE lab_runtime_cleanup_receipts + SET last_error_message = $2, updated_at = CURRENT_TIMESTAMP WHERE id = $1`, + [row.id, message.slice(0, 4_000)], + ), + ); + failed += 1; + } + } + return { attempted: rows.rows.length, completed, failed }; +} diff --git a/src/server/labs/docker-plan.ts b/src/server/labs/docker-plan.ts index 9f7973e15..721c39b1c 100644 --- a/src/server/labs/docker-plan.ts +++ b/src/server/labs/docker-plan.ts @@ -238,6 +238,7 @@ export const buildRunLabCommand = (options: LabContainerOptions): DockerCommandS const microvmRuntimeClass = isolation === "microvm" ? resolveMicrovmRuntimeClass(options.microvmRuntimeClass) : null; const trafficRecording = trafficRecordingModeFor(options); + const restricted = isRestrictedNetworkProfile(networkProfile.id); assertSafeValue("image ref", image, SAFE_IMAGE_REF); assertSafeValue("user", user, SAFE_USER); @@ -253,8 +254,8 @@ export const buildRunLabCommand = (options: LabContainerOptions): DockerCommandS return { command: "docker", args: [ - "run", - "--detach", + restricted ? "create" : "run", + ...(restricted ? [] : ["--detach"]), "--name", identity.containerName, "--hostname", @@ -275,11 +276,13 @@ export const buildRunLabCommand = (options: LabContainerOptions): DockerCommandS : {}), }), ...dockerHardeningArgs(options.boundary, limits), - ...dockerNetworkArgs({ - boundary: options.boundary, - profileId: networkProfile.id, - networkName: options.networkName, - }), + ...(restricted + ? ["--network", "none"] + : dockerNetworkArgs({ + boundary: options.boundary, + profileId: networkProfile.id, + networkName: options.networkName, + })), ...(networkProfile.dockerNetworkMode === "none" ? [] : ["--add-host", "host.docker.internal:host-gateway"]), @@ -312,10 +315,48 @@ export const buildRunLabCommand = (options: LabContainerOptions): DockerCommandS "sleep", "infinity", ], - reason: "Create and start the persistent project lab container.", + reason: restricted + ? "Create the restricted project lab without a network interface." + : "Create and start the persistent project lab container.", }; }; +export const isRestrictedNetworkProfile = (profile: LabContainerOptions["networkProfile"]) => + profile === "approved-targets" || profile === "package-egress"; + +export const buildActivateRestrictedLabCommands = ( + options: LabContainerOptions, +): DockerCommandSpec[] => { + if (!isRestrictedNetworkProfile(options.networkProfile)) return []; + const identity = labContainerIdentity(options); + const profile = getLabNetworkProfile(options.boundary, options.networkProfile); + const network = dockerNetworkArgs({ + boundary: options.boundary, + profileId: options.networkProfile, + networkName: options.networkName, + })[1]; + if (!network) { + throw new Error(`Network profile ${profile.id} requires a named Docker network.`); + } + return [ + { + command: "docker", + args: ["start", identity.containerName], + reason: "Start the restricted workload while it still has no network interface.", + }, + { + command: "docker", + args: ["pause", identity.containerName], + reason: "Freeze the restricted workload before attaching its network.", + }, + { + command: "docker", + args: ["network", "connect", network, identity.containerName], + reason: "Attach the restricted workload network while the workload is paused.", + }, + ]; +}; + /** * Runs the firewall controller outside the workload container. It shares the * workload network namespace so rules apply before packets leave it, but only @@ -335,11 +376,6 @@ export const buildStartEgressEnforcerCommands = ( const enforcer = egressEnforcerIdentity(options); const policyFingerprint = createHash("sha256").update(script).digest("hex"); return [ - { - command: "docker", - args: ["pause", lab.containerName], - reason: "Freeze the workload before replacing its egress policy.", - }, { command: "docker", args: ["rm", "-f", enforcer.containerName], @@ -381,7 +417,7 @@ export const buildStartEgressEnforcerCommands = ( }; export function egressPolicyId(options: LabContainerOptions): string | undefined { - const run = buildStartEgressEnforcerCommands(options)[2]; + const run = buildStartEgressEnforcerCommands(options)[1]; const prefix = "exploit-hunter.egress-policy-sha256="; return run?.args.find((arg) => arg.startsWith(prefix))?.slice(prefix.length); } diff --git a/src/server/labs/network-evidence.ts b/src/server/labs/network-evidence.ts index 7f2f3b1f8..d3ecf2119 100644 --- a/src/server/labs/network-evidence.ts +++ b/src/server/labs/network-evidence.ts @@ -6,6 +6,7 @@ import type { LabEgressEnforcementResult } from "./types"; export const LAB_NETWORK_EVIDENCE_SCHEMA = "exploit-hunter.lab-network-evidence.v1"; export type LabNetworkObservationDisposition = + | "intended" | "allowed" | "denied" | "enforcement-unavailable" @@ -14,12 +15,18 @@ export type LabNetworkObservationDisposition = export type LabNetworkObservation = { observedAt: string; disposition: LabNetworkObservationDisposition; - event: "connection-attempt" | "dns-resolution" | "enforcement-state"; + event: + | "network-attempt-intent" + | "policy-counter" + | "connection-attempt" + | "dns-resolution" + | "enforcement-state"; destination?: string; port?: number; protocol?: "tcp" | "udp"; hostname?: string; resolvedAddresses?: string[]; + packetCount?: number; reason?: string; source: string; }; @@ -76,7 +83,7 @@ export function createArtifactLabNetworkEvidenceRecorder( if (input.observations.length === 0) return; const receiptId = await receipts.admit(input); try { - const artifact = await writeNetworkEvidenceArtifact(writer, input); + const artifact = await writeNetworkEvidenceArtifact(writer, input, receiptId); await receipts.recorded(receiptId, artifact.id); } catch (error) { await receipts.failed(receiptId, error).catch(() => undefined); @@ -86,15 +93,59 @@ export function createArtifactLabNetworkEvidenceRecorder( }; } +export async function openLabNetworkEvidenceIntent( + input: LabNetworkEvidenceInput, +): Promise { + if (input.observations.length === 0) { + throw new Error("A network evidence intent requires at least one observation."); + } + const id = createId("networkEvidenceReceipt"); + await withDatabase((db) => + db.query( + `INSERT INTO network_evidence_receipts + (id, project_id, thread_id, policy_id, status, payload, attempts) + VALUES ($1, $2, $3, $4, 'pending', $5, 0)`, + [id, input.projectId, input.threadId ?? null, input.policyId ?? null, JSON.stringify(input)], + ), + ); + return id; +} + +export async function completeLabNetworkEvidenceIntent( + receiptId: string, + input: LabNetworkEvidenceInput, + writer: ArtifactWriter = getArtifactService(), +): Promise { + const opened = await withDatabase((db) => + db.query<{ id: string }>( + `UPDATE network_evidence_receipts + SET payload = $2, attempts = 1, updated_at = CURRENT_TIMESTAMP + WHERE id = $1 AND status IN ('pending', 'failed') AND attempts = 0 + RETURNING id`, + [receiptId, JSON.stringify(input)], + ), + ); + if (!opened.rows[0]) { + throw new Error(`Network evidence intent ${receiptId} is not open for finalization.`); + } + try { + const artifact = await writeNetworkEvidenceArtifact(writer, input, receiptId); + await databaseNetworkEvidenceReceiptStore.recorded(receiptId, artifact.id); + } catch (error) { + await databaseNetworkEvidenceReceiptStore.failed(receiptId, error).catch(() => undefined); + throw error; + } +} + export async function retryLabNetworkEvidenceReceipts( input: { limit?: number; writer?: ArtifactWriter } = {}, ): Promise<{ attempted: number; recorded: number; failed: number }> { const limit = Math.max(1, Math.min(100, Math.floor(input.limit ?? 25))); const writer = input.writer ?? getArtifactService(); const rows = await withDatabase(async (db) => - db.query<{ id: string; payload: LabNetworkEvidenceInput }>( - `SELECT id, payload FROM network_evidence_receipts - WHERE status IN ('pending', 'failed') + db.query<{ id: string; payload: LabNetworkEvidenceInput; attempts: number }>( + `SELECT id, payload, attempts FROM network_evidence_receipts + WHERE status IN ('pending', 'failed') AND attempts > 0 ORDER BY updated_at, id LIMIT $1`, [limit], ), @@ -104,17 +155,20 @@ export async function retryLabNetworkEvidenceReceipts( for (const row of rows.rows) { try { await withDatabase(async (db) => { - await db.query( + const claimed = await db.query<{ id: string }>( `UPDATE network_evidence_receipts SET status = 'pending', attempts = attempts + 1, updated_at = CURRENT_TIMESTAMP - WHERE id = $1`, - [row.id], + WHERE id = $1 AND status IN ('pending', 'failed') AND attempts = $2 + RETURNING id`, + [row.id, row.attempts], ); + if (!claimed.rows[0]) throw new ReceiptAlreadyClaimedError(); }); - const artifact = await writeNetworkEvidenceArtifact(writer, row.payload); + const artifact = await writeNetworkEvidenceArtifact(writer, row.payload, row.id); await databaseNetworkEvidenceReceiptStore.recorded(row.id, artifact.id); recorded += 1; } catch (error) { + if (error instanceof ReceiptAlreadyClaimedError) continue; await databaseNetworkEvidenceReceiptStore.failed(row.id, error).catch(() => undefined); failed += 1; } @@ -125,34 +179,57 @@ export async function retryLabNetworkEvidenceReceipts( async function writeNetworkEvidenceArtifact( writer: ArtifactWriter, input: LabNetworkEvidenceInput, + receiptId: string, ) { - return writer.createArtifact({ - projectId: input.projectId, - ...(input.threadId ? { threadId: input.threadId } : {}), - ...(input.taskId ? { taskId: input.taskId } : {}), - ...(input.targetIds?.length ? { targetIds: input.targetIds } : {}), - ...(input.toolRunId ? { toolRunId: input.toolRunId } : {}), - projectScoped: true, - name: `network-evidence-${input.policyId?.slice(0, 16) ?? "unavailable"}.jsonl`, - kind: "log", - contentType: "application/x-ndjson", - content: buildLabNetworkEvidenceJsonl(input), - agentGenerated: true, - source: "network-observation", - indexForRag: false, - metadata: { - schema: LAB_NETWORK_EVIDENCE_SCHEMA, - source: "lab-network-observation", - networkProfile: input.networkProfile, - policyId: input.policyId, - researchRunId: input.researchRunId, - targetIds: input.targetIds, - observationCount: input.observations.length, - dispositions: [...new Set(input.observations.map((event) => event.disposition))], - }, - }); + const name = `network-evidence-${receiptId}.jsonl`; + const existing = await withDatabase((db) => + db.query<{ id: string }>( + "SELECT id FROM artifacts WHERE project_id = $1 AND name = $2 LIMIT 1", + [input.projectId, name], + ), + ); + if (existing.rows[0]) return existing.rows[0]; + try { + return await writer.createArtifact({ + projectId: input.projectId, + ...(input.threadId ? { threadId: input.threadId } : {}), + ...(input.taskId ? { taskId: input.taskId } : {}), + ...(input.targetIds?.length ? { targetIds: input.targetIds } : {}), + ...(input.toolRunId ? { toolRunId: input.toolRunId } : {}), + projectScoped: true, + name, + kind: "log", + contentType: "application/x-ndjson", + content: buildLabNetworkEvidenceJsonl(input), + agentGenerated: true, + source: "network-observation", + indexForRag: false, + metadata: { + schema: LAB_NETWORK_EVIDENCE_SCHEMA, + receiptId, + source: "lab-network-observation", + networkProfile: input.networkProfile, + policyId: input.policyId, + researchRunId: input.researchRunId, + targetIds: input.targetIds, + observationCount: input.observations.length, + dispositions: [...new Set(input.observations.map((event) => event.disposition))], + }, + }); + } catch (error) { + const raced = await withDatabase((db) => + db.query<{ id: string }>( + "SELECT id FROM artifacts WHERE project_id = $1 AND name = $2 LIMIT 1", + [input.projectId, name], + ), + ); + if (raced.rows[0]) return raced.rows[0]; + throw error; + } } +class ReceiptAlreadyClaimedError extends Error {} + const databaseNetworkEvidenceReceiptStore: LabNetworkEvidenceReceiptStore = { async admit(input) { const id = createId("networkEvidenceReceipt"); @@ -189,7 +266,7 @@ const databaseNetworkEvidenceReceiptStore: LabNetworkEvidenceReceiptStore = { `UPDATE network_evidence_receipts SET status = 'failed', last_error = $2, updated_at = CURRENT_TIMESTAMP WHERE id = $1`, - [receiptId, message.slice(0, 4_000)], + [receiptId, JSON.stringify({ message: message.slice(0, 4_000) })], ); }); }, @@ -227,11 +304,14 @@ export function parseLabNetworkObservation( throw new Error("Network observation does not match the active project and policy."); } const event = readEnum(record.event, [ + "network-attempt-intent", + "policy-counter", "connection-attempt", "dns-resolution", "enforcement-state", ] as const); const disposition = readEnum(record.disposition, [ + "intended", "allowed", "denied", "enforcement-unavailable", @@ -247,6 +327,10 @@ export function parseLabNetworkObservation( throw new Error("Network observation port must be an integer from 1 through 65535."); } const resolvedAddresses = record.resolvedAddresses; + const packetCount = record.packetCount; + if (packetCount !== undefined && (!Number.isInteger(packetCount) || Number(packetCount) < 1)) { + throw new Error("Network observation packetCount must be a positive integer."); + } if ( resolvedAddresses !== undefined && (!Array.isArray(resolvedAddresses) || @@ -272,6 +356,9 @@ export function parseLabNetworkObservation( ) { throw new Error("Network observation event and disposition are inconsistent."); } + if ((event === "network-attempt-intent") !== (disposition === "intended")) { + throw new Error("Network intent observations require the intended disposition."); + } return { observedAt, event, @@ -286,6 +373,7 @@ export function parseLabNetworkObservation( ...(Array.isArray(resolvedAddresses) ? { resolvedAddresses: resolvedAddresses as string[] } : {}), + ...(typeof packetCount === "number" ? { packetCount } : {}), ...(typeof record.reason === "string" ? { reason: record.reason } : {}), }; } diff --git a/src/server/labs/network-profiles.ts b/src/server/labs/network-profiles.ts index 0931d10ce..3c57a2e84 100644 --- a/src/server/labs/network-profiles.ts +++ b/src/server/labs/network-profiles.ts @@ -403,6 +403,7 @@ export const buildEgressIptablesScript = ({ "iptables -F INPUT", "iptables -F FORWARD", "iptables -F OUTPUT", + "if command -v ip6tables >/dev/null 2>&1; then ip6tables -P INPUT DROP; ip6tables -P FORWARD DROP; ip6tables -P OUTPUT DROP; ip6tables -F INPUT; ip6tables -F FORWARD; ip6tables -F OUTPUT; elif ip -6 addr show scope global | grep -q inet6; then echo 'IPv6 is active but ip6tables is unavailable' >&2; exit 44; fi", "iptables -A INPUT -m state --state ESTABLISHED,RELATED -j ACCEPT", "iptables -A OUTPUT -m state --state ESTABLISHED,RELATED -j ACCEPT", "iptables -A INPUT -i lo -j ACCEPT", diff --git a/src/server/labs/repository.ts b/src/server/labs/repository.ts index a4cd30aa0..2fd1ed089 100644 --- a/src/server/labs/repository.ts +++ b/src/server/labs/repository.ts @@ -64,6 +64,20 @@ export class SqliteProjectLabRepository implements ProjectLabRepository { return result.rows[0] ? normalizeProjectLabRow(result.rows[0]) : null; } + async claimRestart(labId: string): Promise { + const result = await this.db.query( + `UPDATE project_labs + SET status = 'provisioning', + failure_reason = NULL, + last_error = NULL, + updated_at = now() + WHERE id = $1 AND status = 'running' + RETURNING *`, + [labId], + ); + return result.rows[0] ? normalizeProjectLabRow(result.rows[0]) : null; + } + async releaseStaleProvisioning( labId: string, staleBefore: string, diff --git a/src/server/labs/runtime.ts b/src/server/labs/runtime.ts index d08da910e..26f48c792 100644 --- a/src/server/labs/runtime.ts +++ b/src/server/labs/runtime.ts @@ -4,6 +4,7 @@ import { promisify } from "node:util"; import { threadWorkspaceHostPath } from "../../lib/thread-workspace-path"; import { + buildActivateRestrictedLabCommands, buildBuildEgressEnforcerImageCommand, buildBuildLabImageCommand, buildDestroyEgressEnforcerCommand, @@ -21,6 +22,7 @@ import { buildStopTrafficRecorderCommand, egressPolicyId, isManagedKaliLabImage, + isRestrictedNetworkProfile, labContainerIdentity, } from "./docker-plan"; import { DEFAULT_KALI_LAB_IMAGE, resolveMicrovmRuntimeClass } from "./hardening"; @@ -170,20 +172,30 @@ export class ProjectLabRuntime { const identity = labContainerIdentity(options); if (mode === "dry-run") { + const restrictedLifecycle = isRestrictedNetworkProfile(options.networkProfile) + ? [ + buildDestroyEgressEnforcerCommand(options), + buildDestroyLabContainerCommand(options), + buildRunLabCommand(options), + ...buildActivateRestrictedLabCommands(options), + ].filter((command): command is DockerCommandSpec => Boolean(command)) + : [ + { + command: "docker" as const, + args: ["container", "inspect", identity.containerName], + reason: "Check whether the persistent project lab container already exists.", + }, + { + command: "docker" as const, + args: ["start", identity.containerName], + reason: "Start an existing persistent project lab container.", + }, + buildRunLabCommand(options), + ]; const planned = [ ...(microvmCheckCommand ? [microvmCheckCommand] : []), ...commands, - { - command: "docker" as const, - args: ["container", "inspect", identity.containerName], - reason: "Check whether the persistent project lab container already exists.", - }, - { - command: "docker" as const, - args: ["start", identity.containerName], - reason: "Start an existing persistent project lab container.", - }, - buildRunLabCommand(options), + ...restrictedLifecycle, ...buildStartEgressEnforcerCommands(options), ...buildStartTrafficRecorderCommands(options), ]; @@ -194,6 +206,38 @@ export class ProjectLabRuntime { await this.runner.run(command); } + if (isRestrictedNetworkProfile(options.networkProfile)) { + const destroyCommands = [ + buildDestroyEgressEnforcerCommand(options), + buildDestroyLabContainerCommand(options), + ] + .filter((command): command is DockerCommandSpec => Boolean(command)) + .map((command) => ({ + ...command, + reason: `Remove stale restricted runtime before safe recreation: ${command.reason}`, + })); + for (const command of destroyCommands) { + try { + await this.runner.run(command); + } catch (error) { + if (!isDockerNoSuchContainer(error)) throw error; + } + } + const createCommand = buildRunLabCommand(options); + const activationCommands = buildActivateRestrictedLabCommands(options); + await this.runner.run(createCommand); + for (const command of activationCommands) await this.runner.run(command); + const egressEnforcement = await this.startEgressEnforcer(options); + await this.startTrafficRecorder(options); + return { + mode, + dryRun: false, + commands: [...commands, ...destroyCommands, createCommand, ...activationCommands], + container: identity, + egressEnforcement, + }; + } + const inspectCommand: DockerCommandSpec = { command: "docker", args: ["container", "inspect", identity.containerName], @@ -526,10 +570,9 @@ export class ProjectLabRuntime { workloadRunning: true, }; } - const [pause, remove, run, enforce] = commands; + const [remove, run, enforce] = commands; const policyId = egressPolicyId(options); try { - await this.runner.run(pause!); try { await this.runner.run(remove!); } catch (error) { @@ -544,7 +587,7 @@ export class ProjectLabRuntime { enforcerIdentityName(options), "/bin/bash", "-lc", - `test "$(cat /proc/sys/net/ipv4/ip_forward)" != "" && iptables -C OUTPUT -j REJECT && iptables -S OUTPUT | grep -Fx -- '-P OUTPUT DROP'`, + `test "$(cat /proc/sys/net/ipv4/ip_forward)" != "" && iptables -C OUTPUT -j REJECT && iptables -S OUTPUT | grep -Fx -- '-P OUTPUT DROP' && if command -v ip6tables >/dev/null 2>&1; then ip6tables -S OUTPUT | grep -Fx -- '-P OUTPUT DROP'; else ! ip -6 addr show scope global | grep -q inet6; fi`, ], reason: `Verify external egress policy for network profile ${requestedProfile}.`, }; diff --git a/src/server/labs/service.ts b/src/server/labs/service.ts index 014ab5320..2157aa819 100644 --- a/src/server/labs/service.ts +++ b/src/server/labs/service.ts @@ -1,4 +1,8 @@ +import { createHash } from "node:crypto"; + import type { JsonObject, ProjectLabRow } from "../db/types"; +import { recordLabRuntimeCleanupFailure } from "./cleanup-receipts"; +import { labContainerIdentity } from "./docker-plan"; import { DEFAULT_KALI_LAB_IMAGE, PREVIOUS_KALI_LAB_IMAGE, @@ -6,9 +10,11 @@ import { UPSTREAM_KALI_LAB_IMAGE, } from "./hardening"; import { + completeLabNetworkEvidenceIntent, createArtifactLabNetworkEvidenceRecorder, enforcementObservation, type LabNetworkEvidenceRecorder, + openLabNetworkEvidenceIntent, } from "./network-evidence"; import { buildUfwCommandPlan, normalizeLabFirewallConfig } from "./network-profiles"; import { sanitizeJsonObject, withProjectLabRepository } from "./repository"; @@ -234,22 +240,40 @@ export class ProjectLabService { return this.statusFromLab(projectId, running); } catch (error) { const failure = serializeLabFailure(error); + const cleanupOptions = { + projectId, + threadId: requestedThreadId ?? readActiveThreadId(claimed), + boundary: "human" as const, + image: claimed.image_ref, + networkProfile, + }; + let cleanupReceiptId: string | undefined; if (!(error instanceof LabEgressEnforcementError)) { - await this.runtime - .destroyContainer({ + try { + await this.runtime.destroyContainer(cleanupOptions); + } catch (cleanupError) { + cleanupReceiptId = await recordLabRuntimeCleanupFailure({ projectId, - threadId: requestedThreadId ?? readActiveThreadId(claimed), - boundary: "human", - image: claimed.image_ref, - networkProfile, - }) - .catch(() => undefined); + labId: claimed.id, + runtimeLocator: labContainerIdentity(cleanupOptions).containerName, + options: cleanupOptions, + error: cleanupError, + }); + } + } else if (error.enforcement.cleanupSucceeded === false) { + cleanupReceiptId = await recordLabRuntimeCleanupFailure({ + projectId, + labId: claimed.id, + runtimeLocator: labContainerIdentity(cleanupOptions).containerName, + options: cleanupOptions, + error, + }); } await this.repository.update(claimed.id, { ...claimed, status: "failed", - runtime_id: null, - container_id: null, + runtime_id: cleanupReceiptId ? labContainerIdentity(cleanupOptions).containerName : null, + container_id: cleanupReceiptId ? labContainerIdentity(cleanupOptions).containerName : null, runtime_metadata: error instanceof LabEgressEnforcementError ? mergeMetadata(claimed.runtime_metadata, { @@ -261,8 +285,14 @@ export class ProjectLabService { reason: command.reason, })), egressEnforcement: error.enforcement as unknown as JsonObject, + ...(cleanupReceiptId ? { cleanupPending: true, cleanupReceiptId } : {}), }) - : claimed.runtime_metadata, + : cleanupReceiptId + ? mergeMetadata(claimed.runtime_metadata, { + cleanupPending: true, + cleanupReceiptId, + }) + : claimed.runtime_metadata, failure_reason: failure.message, last_error: failure, metadata: @@ -317,10 +347,22 @@ export class ProjectLabService { async restart(projectId: string, input: LabCreateInput = {}): Promise { const created = await this.create(projectId, input); - const lab = created.lab; + let lab = created.lab; if (!lab) { return created; } + if (lab.status === "running") { + const claimed = await this.repository.claimRestart(lab.id); + if (!claimed) { + return this.statusFromLab( + projectId, + (await this.repository.findByProjectId(projectId)) ?? lab, + ); + } + lab = claimed; + } else if (lab.status === "provisioning") { + return this.statusFromLab(projectId, lab); + } const destroyResult = await this.runtime.destroyContainer({ projectId, @@ -443,21 +485,34 @@ export class ProjectLabService { return this.statusFromLab(projectId, running); } catch (error) { - await this.runtime - .destroyContainer({ + const cleanupOptions = { + projectId, + threadId: cleanText(input.threadId) ?? readActiveThreadId(lab), + boundary: "human" as const, + image: lab.image_ref, + networkProfile, + }; + let cleanupReceiptId: string | undefined; + try { + await this.runtime.destroyContainer(cleanupOptions); + } catch (cleanupError) { + cleanupReceiptId = await recordLabRuntimeCleanupFailure({ projectId, - threadId: cleanText(input.threadId) ?? readActiveThreadId(lab), - boundary: "human", - image: lab.image_ref, - networkProfile, - }) - .catch(() => undefined); + labId: lab.id, + runtimeLocator: labContainerIdentity(cleanupOptions).containerName, + options: cleanupOptions, + error: cleanupError, + }); + } const failure = serializeLabFailure(error); await this.repository.update(lab.id, { ...lab, status: "failed", - runtime_id: null, - container_id: null, + runtime_id: cleanupReceiptId ? labContainerIdentity(cleanupOptions).containerName : null, + container_id: cleanupReceiptId ? labContainerIdentity(cleanupOptions).containerName : null, + runtime_metadata: cleanupReceiptId + ? mergeMetadata(lab.runtime_metadata, { cleanupPending: true, cleanupReceiptId }) + : lab.runtime_metadata, failure_reason: `Network evidence admission failed: ${failure.message}`, last_error: failure, }); @@ -609,25 +664,96 @@ export async function getProjectLabEgressEnforcement( ); } -export async function recordProjectLabNetworkObservations(input: { +export type ProjectLabNetworkIntent = { + receiptId: string; + policyId: string; + networkProfile: string; +}; + +export async function openProjectLabNetworkIntent(input: { projectId: string; threadId?: string; taskId?: string; targetIds?: string[]; toolRunId?: string; researchRunId?: string; - observations: import("./network-evidence").LabNetworkObservation[]; -}): Promise { + command: string; +}): Promise { const status = await getProjectLabStatus(input.projectId); const enforcement = readLabEgressEnforcement(status.lab?.runtime_metadata.egressEnforcement); - if (!enforcement?.policyId || enforcement.disposition !== "enforced") { - throw new Error("Network observations require a verified, fingerprinted egress policy."); + const policyId = networkPolicySnapshotId(enforcement); + if (!policyId) { + throw new Error("Network execution requires a verified, fingerprinted egress policy."); } - await createArtifactLabNetworkEvidenceRecorder().record({ + const networkProfile = enforcement!.requestedProfile; + const receiptId = await openLabNetworkEvidenceIntent({ ...input, - networkProfile: enforcement.requestedProfile, - policyId: enforcement.policyId, + networkProfile, + policyId, + observations: [ + { + observedAt: new Date().toISOString(), + event: "network-attempt-intent", + disposition: "intended", + reason: `approved-command-sha256:${createHash("sha256").update(input.command).digest("hex")}`, + source: "approved-command-boundary", + }, + ], }); + return { receiptId, policyId, networkProfile }; +} + +export async function completeProjectLabNetworkIntent(input: { + intent: ProjectLabNetworkIntent; + projectId: string; + threadId?: string; + taskId?: string; + targetIds?: string[]; + toolRunId?: string; + researchRunId?: string; + observations: import("./network-evidence").LabNetworkObservation[]; +}): Promise { + const status = await getProjectLabStatus(input.projectId); + const enforcement = readLabEgressEnforcement(status.lab?.runtime_metadata.egressEnforcement); + const policyUnchanged = + status.status === "running" && networkPolicySnapshotId(enforcement) === input.intent.policyId; + await completeLabNetworkEvidenceIntent(input.intent.receiptId, { + projectId: input.projectId, + threadId: input.threadId, + taskId: input.taskId, + targetIds: input.targetIds, + toolRunId: input.toolRunId, + researchRunId: input.researchRunId, + networkProfile: input.intent.networkProfile, + policyId: input.intent.policyId, + observations: policyUnchanged + ? input.observations + : [ + { + observedAt: new Date().toISOString(), + event: "enforcement-state", + disposition: "enforcement-unavailable", + reason: "The active egress policy changed before command observation finalization.", + source: "policy-snapshot-cas", + }, + ], + }); +} + +function networkPolicySnapshotId( + enforcement: LabEgressEnforcementResult | undefined, +): string | undefined { + if (!enforcement) return undefined; + if (enforcement.disposition === "enforced" && enforcement.policyId) { + return enforcement.policyId; + } + if (enforcement.disposition === "not-required" && enforcement.requestedProfile === "offline") { + return "network-none:offline:v1"; + } + if (enforcement.disposition === "not-required" && enforcement.requestedProfile === "full") { + return "unrestricted:full:v1"; + } + return undefined; } export async function runProjectLabAction( diff --git a/src/server/labs/types.ts b/src/server/labs/types.ts index 4fff3cb2a..7b2f18597 100644 --- a/src/server/labs/types.ts +++ b/src/server/labs/types.ts @@ -66,6 +66,7 @@ export interface ProjectLabRepository { findByProjectId(projectId: string): Promise; create(projectId: string, input: NormalizedLabCreateInput): Promise; claimProvisioning(labId: string, staleBefore: string): Promise; + claimRestart(labId: string): Promise; releaseStaleProvisioning(labId: string, staleBefore: string): Promise; update(labId: string, input: Partial): Promise; } diff --git a/tests/integration/agent-command-tool-run-lifecycle.test.ts b/tests/integration/agent-command-tool-run-lifecycle.test.ts index fe4579998..7412f2491 100644 --- a/tests/integration/agent-command-tool-run-lifecycle.test.ts +++ b/tests/integration/agent-command-tool-run-lifecycle.test.ts @@ -317,7 +317,7 @@ describe("SQLite-backed approved command Tool Run lifecycle", () => { }); await expect(toolRunCount()).resolves.toBe(beforeMismatchRunCount); expect(projectAgentLabCommandRunner.run).toHaveBeenCalledTimes(7); - }, 15_000); + }, 30_000); it("counts a consumed autonomous approval when its command exits nonzero", async () => { await withDatabase(async (db) => { diff --git a/tests/integration/lab-cleanup-receipts.test.ts b/tests/integration/lab-cleanup-receipts.test.ts new file mode 100644 index 000000000..64b55c6c9 --- /dev/null +++ b/tests/integration/lab-cleanup-receipts.test.ts @@ -0,0 +1,70 @@ +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { afterEach, describe, expect, it } from "vitest"; +import { createSqlitePool, getDatabaseConfig } from "../../src/server/db/client"; +import { + recordLabRuntimeCleanupFailure, + retryLabRuntimeCleanupReceipts, +} from "../../src/server/labs/cleanup-receipts"; +import { ProjectLabRuntime } from "../../src/server/labs/runtime"; + +describe("lab runtime cleanup receipts", () => { + const directories: string[] = []; + const previous = process.env.SQLITE_DATABASE_URL; + + afterEach(async () => { + if (previous === undefined) delete process.env.SQLITE_DATABASE_URL; + else process.env.SQLITE_DATABASE_URL = previous; + await Promise.all( + directories.splice(0).map((path) => rm(path, { recursive: true, force: true })), + ); + }); + + it("retains the runtime locator until an operator retry completes cleanup", async () => { + const directory = await mkdtemp(join(tmpdir(), "lab-cleanup-receipt-")); + directories.push(directory); + process.env.SQLITE_DATABASE_URL = `sqlite://${join(directory, "app.sqlite")}`; + const pool = createSqlitePool(getDatabaseConfig()); + try { + await pool.query("INSERT INTO projects (id, name, slug) VALUES ($1, $2, $3)", [ + "cleanup-project", + "Cleanup project", + "cleanup-project", + ]); + const lab = await pool.query<{ id: string }>( + `INSERT INTO project_labs (project_id, status, image_ref, profile, runtime) + VALUES ($1, 'failed', $2, 'default', 'docker') RETURNING id`, + ["cleanup-project", "alpine:3.20"], + ); + await recordLabRuntimeCleanupFailure({ + projectId: "cleanup-project", + labId: lab.rows[0]!.id, + runtimeLocator: "exploit-hunter-human-cleanup-project-lab", + options: { + projectId: "cleanup-project", + boundary: "human", + image: "alpine:3.20", + networkProfile: "package-egress", + }, + error: new Error("docker temporarily unavailable"), + }); + + await expect( + retryLabRuntimeCleanupReceipts({ runtime: new ProjectLabRuntime({ mode: "dry-run" }) }), + ).resolves.toEqual({ attempted: 1, completed: 1, failed: 0 }); + const receipts = await pool.query<{ status: string; runtime_locator: string }>( + "SELECT status, runtime_locator FROM lab_runtime_cleanup_receipts", + ); + expect(receipts.rows).toEqual([ + { + status: "completed", + runtime_locator: "exploit-hunter-human-cleanup-project-lab", + }, + ]); + } finally { + await pool.end(); + } + }); +}); diff --git a/tests/integration/lab-network-evidence.test.ts b/tests/integration/lab-network-evidence.test.ts index 44bffd62c..a6fe502d9 100644 --- a/tests/integration/lab-network-evidence.test.ts +++ b/tests/integration/lab-network-evidence.test.ts @@ -3,18 +3,94 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { describe, expect, it, vi } from "vitest"; -import { networkObservationsForCommand } from "../../src/server/agent-lab/command-runner"; +import { + networkObservationsFromControllerCounters, + parseEgressControllerCounters, +} from "../../src/server/agent-lab/command-runner"; import { createSqlitePool, getDatabaseConfig } from "../../src/server/db/client"; -import type { ArtifactServiceInstance, CreateArtifactInput } from "../../src/server/evidence"; +import { + type ArtifactServiceInstance, + type CreateArtifactInput, + createArtifactService, +} from "../../src/server/evidence"; import { buildLabNetworkEvidenceJsonl, + completeLabNetworkEvidenceIntent, createArtifactLabNetworkEvidenceRecorder, LAB_NETWORK_EVIDENCE_SCHEMA, + openLabNetworkEvidenceIntent, parseLabNetworkObservation, + retryLabNetworkEvidenceReceipts, } from "../../src/server/labs/network-evidence"; describe("lab network evidence", () => { + it("keeps pre-execution intent durable but ineligible for delivery until finalized", async () => { + const previous = process.env.SQLITE_DATABASE_URL; + const directory = await mkdtemp(join(tmpdir(), "network-evidence-intent-")); + process.env.SQLITE_DATABASE_URL = `sqlite://${join(directory, "app.sqlite")}`; + const pool = createSqlitePool(getDatabaseConfig()); + try { + await pool.query("INSERT INTO projects (id, name, slug) VALUES ($1, $2, $3)", [ + "intent-project", + "Intent project", + "intent-project", + ]); + const base = { + projectId: "intent-project", + toolRunId: "tool-run-intent", + networkProfile: "package-egress", + policyId: "b".repeat(64), + }; + const receiptId = await openLabNetworkEvidenceIntent({ + ...base, + observations: [ + { + observedAt: new Date().toISOString(), + event: "network-attempt-intent", + disposition: "intended", + source: "approved-command-boundary", + }, + ], + }); + + await expect(retryLabNetworkEvidenceReceipts()).resolves.toEqual({ + attempted: 0, + recorded: 0, + failed: 0, + }); + await completeLabNetworkEvidenceIntent( + receiptId, + { + ...base, + observations: [ + { + observedAt: new Date().toISOString(), + event: "policy-counter", + disposition: "denied", + packetCount: 1, + source: "external-egress-controller-counters", + }, + ], + }, + createArtifactService({ storage: null }), + ); + const receipt = await pool.query<{ status: string; artifact_id: string | null }>( + "SELECT status, artifact_id FROM network_evidence_receipts WHERE id = $1", + [receiptId], + ); + expect(receipt.rows[0]).toMatchObject({ + status: "recorded", + artifact_id: expect.any(String), + }); + } finally { + await pool.end(); + if (previous === undefined) delete process.env.SQLITE_DATABASE_URL; + else process.env.SQLITE_DATABASE_URL = previous; + await rm(directory, { recursive: true, force: true }); + } + }); + it("persists correlated observations through the central artifact service", async () => { const createArtifact = vi.fn(async (_input: CreateArtifactInput) => ({ id: "artifact-network-1", @@ -137,27 +213,28 @@ describe("lab network evidence", () => { ).toThrow(/destination, port, and protocol/); }); - it("projects actual command outcomes into DNS and connection observations", () => { - expect(networkObservationsForCommand("curl https://target.example:8443/health", true)).toEqual([ + it("projects only controller-grounded policy counter deltas", () => { + const before = parseEgressControllerCounters( + "[2:100] -A OUTPUT -p tcp -d 192.0.2.10/32 --dport 443 -j ACCEPT\n[1:60] -A OUTPUT -j REJECT", + "policy-1", + ); + const after = parseEgressControllerCounters( + "[5:250] -A OUTPUT -p tcp -d 192.0.2.10/32 --dport 443 -j ACCEPT\n[4:240] -A OUTPUT -j REJECT", + "policy-1", + ); + + expect(networkObservationsFromControllerCounters(before, after)).toEqual([ expect.objectContaining({ - event: "connection-attempt", + event: "policy-counter", disposition: "allowed", - destination: "target.example", - port: 8443, + packetCount: 3, + source: "external-egress-controller-counters", }), - ]); - expect( - networkObservationsForCommand( - "dig target.example", - true, - "target.example. 60 IN A 198.51.100.9", - ), - ).toEqual([ expect.objectContaining({ - event: "dns-resolution", - disposition: "allowed", - hostname: "target.example", - resolvedAddresses: ["198.51.100.9"], + event: "policy-counter", + disposition: "denied", + packetCount: 3, + source: "external-egress-controller-counters", }), ]); }); @@ -217,6 +294,27 @@ describe("lab network evidence", () => { }), }), ]); + + const repairs = await Promise.all([ + retryLabNetworkEvidenceReceipts({ + writer: createArtifactService({ storage: null }), + }), + retryLabNetworkEvidenceReceipts({ + writer: createArtifactService({ storage: null }), + }), + ]); + expect(repairs.reduce((sum, result) => sum + result.recorded, 0)).toBe(1); + await expect(retryLabNetworkEvidenceReceipts()).resolves.toEqual({ + attempted: 0, + recorded: 0, + failed: 0, + }); + const repaired = await pool.query<{ status: string; artifact_id: string | null }>( + "SELECT status, artifact_id FROM network_evidence_receipts", + ); + expect(repaired.rows).toEqual([ + expect.objectContaining({ status: "recorded", artifact_id: expect.any(String) }), + ]); } finally { await pool.end(); if (previous === undefined) delete process.env.SQLITE_DATABASE_URL; diff --git a/tests/integration/lab-runtime.test.ts b/tests/integration/lab-runtime.test.ts index 0b4374d23..b99e1a202 100644 --- a/tests/integration/lab-runtime.test.ts +++ b/tests/integration/lab-runtime.test.ts @@ -265,8 +265,8 @@ describe("project lab Docker runtime primitives", () => { const lab = labContainerIdentity(options); const workload = buildRunLabCommand(options); const commands = buildStartEgressEnforcerCommands(options); - const run = commands[2]!; - const enforce = commands[3]!; + const run = commands[1]!; + const enforce = commands[2]!; expect(workload.args).not.toEqual(expect.arrayContaining(["NET_ADMIN", "NET_RAW"])); expect(run.args).toEqual( @@ -281,8 +281,9 @@ describe("project lab Docker runtime primitives", () => { egressEnforcerImage(options), ]), ); - expect(commands[0]?.args[0]).toBe("pause"); - expect(commands[1]?.args.slice(0, 2)).toEqual(["rm", "-f"]); + expect(workload.args.slice(0, 3)).toEqual(["create", "--name", lab.containerName]); + expect(workload.args).toEqual(expect.arrayContaining(["--network", "none"])); + expect(commands[0]?.args.slice(0, 2)).toEqual(["rm", "-f"]); expect(run.args.join(" ")).toMatch(/exploit-hunter\.egress-policy-sha256=[a-f0-9]{64}/); expect(enforce.args.join(" ")).toContain("EXPLOIT_HUNTER_EGRESS_DENIED:"); expect(enforce.args).toContain("/bin/sh"); @@ -513,8 +514,8 @@ describe("project lab Docker runtime primitives", () => { async run(command) { if ( command.reason.startsWith("Apply external egress policy") || - command.reason.startsWith("Remove the external project egress enforcer") || - command.reason.startsWith("Remove the persistent project lab container") + command.reason === "Remove the external project egress enforcer." || + command.reason === "Remove the persistent project lab container." ) { throw new Error("synthetic enforcement or cleanup failure"); } @@ -571,25 +572,16 @@ describe("project lab Docker runtime primitives", () => { "image", "image", "volume", - "container", - "container", "rm", - "run", - ]); - expect(executed.map((command) => command.args[0])).toEqual([ - "image", - "image", - "volume", - "container", - "container", "rm", - "run", + "create", + "start", "pause", - "rm", - "run", - "exec", - "exec", + "network", ]); + expect(executed.map((command) => command.args[0])).toEqual( + expect.arrayContaining(["create", "start", "pause", "network", "exec"]), + ); }); it("recreates stale existing containers when docker start fails", async () => { diff --git a/tests/integration/project-lab.test.ts b/tests/integration/project-lab.test.ts index 83ab56d28..2faae179c 100644 --- a/tests/integration/project-lab.test.ts +++ b/tests/integration/project-lab.test.ts @@ -321,16 +321,33 @@ describe("project lab lifecycle", () => { "image", "build", "volume", - "container", + "rm", + "rm", + "create", "start", - "run", "pause", + "network", "rm", "run", "exec", ]); }); + it("serializes concurrent restarts behind one durable lifecycle claim", async () => { + const repository = new InMemoryProjectLabRepository(); + const service = buildTestLabService(repository); + await service.start("project-1"); + + const [first, second] = await Promise.all([ + service.restart("project-1", { networkProfile: "package-egress" }), + service.restart("project-1", { networkProfile: "approved-targets" }), + ]); + + expect([first.status, second.status]).toContain("running"); + expect(repository.rows).toHaveLength(1); + await expect(service.status("project-1")).resolves.toMatchObject({ status: "running" }); + }); + it("recreates a running managed environment when its active thread changes", async () => { const repository = new InMemoryProjectLabRepository(); const service = buildTestLabService(repository); @@ -389,7 +406,7 @@ describe("project lab lifecycle", () => { expect(commands).toEqual( expect.arrayContaining([ expect.objectContaining({ - args: expect.arrayContaining(["--network", "fixture-network"]), + args: ["network", "connect", "fixture-network", expect.stringContaining("project-1")], }), ]), ); @@ -599,6 +616,12 @@ class InMemoryProjectLabRepository implements ProjectLabRepository { }); } + async claimRestart(labId: string): Promise { + const row = this.rows.find((candidate) => candidate.id === labId); + if (!row || row.status !== "running") return null; + return this.update(labId, { ...row, status: "provisioning" }); + } + async releaseStaleProvisioning( labId: string, staleBefore: string, diff --git a/tests/live/lab-egress-enforcement-mode.test.ts b/tests/live/lab-egress-enforcement-mode.test.ts index d5e799bdc..2beff7063 100644 --- a/tests/live/lab-egress-enforcement-mode.test.ts +++ b/tests/live/lab-egress-enforcement-mode.test.ts @@ -136,6 +136,7 @@ live("live lab egress enforcement failure modes", () => { for (const command of [ ["nc", "-z", "-w", "2", "1.1.1.1", "80"], + ["nc", "-6", "-z", "-w", "2", "2606:4700:4700::1111", "53"], ["nslookup", "example.com", "8.8.8.8"], ["nc", "-z", "-w", "2", "169.254.169.254", "80"], ["sh", "-lc", "http_proxy=http://1.1.1.1:8080 wget -qO- http://example.com"],