From 2fec7a4854b03361f8b8fbfd83464ae98b88e847 Mon Sep 17 00:00:00 2001 From: Dan Levy Date: Wed, 26 Aug 2026 23:11:16 -0400 Subject: [PATCH 1/4] Require microVMs for high-risk lab workloads --- docs/lab-runtime-hardening.md | 1 + src/server/labs/docker-plan.ts | 17 +++++++-- src/server/labs/hardening.ts | 23 +++++++++++- src/server/labs/service.ts | 50 ++++++++++++++++++++++++--- src/server/labs/types.ts | 3 ++ tests/integration/lab-runtime.test.ts | 41 ++++++++++++++++++++++ tests/integration/project-lab.test.ts | 43 +++++++++++++++++++++++ 7 files changed, 169 insertions(+), 9 deletions(-) diff --git a/docs/lab-runtime-hardening.md b/docs/lab-runtime-hardening.md index 72406af3f..eb5c0dd7a 100644 --- a/docs/lab-runtime-hardening.md +++ b/docs/lab-runtime-hardening.md @@ -28,6 +28,7 @@ The default `container` isolation mode above shares the host kernel with every l ### Enabling it - Per-lab: pass `isolation: "microvm"` (and optionally `microvmRuntimeClass`) to the lab create/start/restart API. The choice is persisted in lab metadata, so subsequent starts reuse it. +- Workload class: selecting `workloadClass: "untrusted-code"` or `workloadClass: "malware-analysis"` requires and persists MicroVM isolation. An explicit request to run either class with `isolation: "container"` is rejected; omitting isolation selects `microvm` and then follows the same fail-closed runtime-availability check. Containers carry both isolation and workload-class labels for external inspection. - Host-wide default: set `PROJECT_LAB_ISOLATION_MODE=microvm`, and optionally `PROJECT_LAB_MICROVM_RUNTIME_CLASS` to override the default runtime class. - Runtime class names map directly to `docker run --runtime ` and must already be registered in the Docker daemon's `runtimes` config (`/etc/docker/daemon.json`) by a Kata Containers install: - `kata-fc` (default) — Firecracker VMM. Smallest device model and strongest isolation, but only virtio net/block/vsock devices are available to the guest. diff --git a/src/server/labs/docker-plan.ts b/src/server/labs/docker-plan.ts index 6449dcbfd..834063607 100644 --- a/src/server/labs/docker-plan.ts +++ b/src/server/labs/docker-plan.ts @@ -16,7 +16,7 @@ import { LAB_HARDENING_LABELS, LAB_HOME_PATH, LAB_WORKSPACE_PATH, - resolveLabIsolationMode, + resolveLabIsolationForWorkload, resolveMicrovmRuntimeClass, SAFE_MICROVM_RUNTIME_CLASS, } from "./hardening"; @@ -180,6 +180,8 @@ export const labContainerIdentity = (options: LabContainerOptions): LabContainer home: `${namePrefix}-home`, workspace: `${namePrefix}-workspace`, }; + const workloadClass = options.workloadClass ?? "standard"; + const isolation = resolveLabIsolationForWorkload(options.isolation, workloadClass); assertSafeValue("container name", containerName, SAFE_DOCKER_NAME); assertSafeValue("hostname", hostname, SAFE_DOCKER_NAME); @@ -197,6 +199,8 @@ export const labContainerIdentity = (options: LabContainerOptions): LabContainer "exploit-hunter.boundary": options.boundary, "exploit-hunter.thread-id": safeThreadWorkspaceSegment(options.threadId, "default-thread"), "exploit-hunter.workspace-layout": "thread-bind-v1", + "exploit-hunter.isolation": isolation, + "exploit-hunter.workload-class": workloadClass, }, }; }; @@ -234,7 +238,10 @@ export const buildRunLabCommand = (options: LabContainerOptions): DockerCommandS const user = options.user ?? DEFAULT_KALI_LAB_USER; const networkProfile = getLabNetworkProfile(options.boundary, options.networkProfile); const limits = mergeResourceLimits(options); - const isolation = resolveLabIsolationMode(options.isolation); + const isolation = resolveLabIsolationForWorkload( + options.isolation, + options.workloadClass ?? "standard", + ); const microvmRuntimeClass = isolation === "microvm" ? resolveMicrovmRuntimeClass(options.microvmRuntimeClass) : null; const trafficRecording = trafficRecordingModeFor(options); @@ -269,6 +276,7 @@ export const buildRunLabCommand = (options: LabContainerOptions): DockerCommandS "exploit-hunter.network-profile": networkProfile.id, "exploit-hunter.image-ref": image, "exploit-hunter.isolation": isolation, + "exploit-hunter.workload-class": options.workloadClass ?? "standard", "exploit-hunter.traffic-recording": trafficRecording, ...(microvmRuntimeClass ? { "exploit-hunter.microvm-runtime-class": microvmRuntimeClass } @@ -506,7 +514,10 @@ export const buildStartLabCommands = (options: LabContainerOptions): DockerComma export const buildMicrovmRuntimeCheckCommand = ( options: LabContainerOptions, ): DockerCommandSpec | null => { - if (resolveLabIsolationMode(options.isolation) !== "microvm") { + if ( + resolveLabIsolationForWorkload(options.isolation, options.workloadClass ?? "standard") !== + "microvm" + ) { return null; } const runtimeClass = resolveMicrovmRuntimeClass(options.microvmRuntimeClass); diff --git a/src/server/labs/hardening.ts b/src/server/labs/hardening.ts index 3b28d57cd..9ecc1a512 100644 --- a/src/server/labs/hardening.ts +++ b/src/server/labs/hardening.ts @@ -1,4 +1,9 @@ -import type { LabBoundary, LabIsolationMode, LabResourceLimits } from "./types"; +import type { + LabBoundary, + LabIsolationMode, + LabResourceLimits, + LabWorkloadClass, +} from "./types"; export const PREVIOUS_KALI_LAB_IMAGE = "exploit-hunter/kali-workspace:latest"; export const DEFAULT_KALI_LAB_IMAGE = "exploit-hunter/kali-workspace:iptables"; @@ -97,6 +102,22 @@ export const resolveLabIsolationMode = (isolation?: LabIsolationMode): LabIsolat : DEFAULT_LAB_ISOLATION_MODE; }; +export class HighRiskIsolationRequiredError extends Error { + constructor(readonly workloadClass: Exclude) { + super(`Workload class "${workloadClass}" requires MicroVM isolation.`); + this.name = "HighRiskIsolationRequiredError"; + } +} + +export const resolveLabIsolationForWorkload = ( + isolation: LabIsolationMode | undefined, + workloadClass: LabWorkloadClass, +): LabIsolationMode => { + if (workloadClass === "standard") return resolveLabIsolationMode(isolation); + if (isolation === "container") throw new HighRiskIsolationRequiredError(workloadClass); + return "microvm"; +}; + export const resolveMicrovmRuntimeClass = (runtimeClass?: string): string => runtimeClass?.trim() || process.env.PROJECT_LAB_MICROVM_RUNTIME_CLASS?.trim() || diff --git a/src/server/labs/service.ts b/src/server/labs/service.ts index 4366b59e1..e034cfcd7 100644 --- a/src/server/labs/service.ts +++ b/src/server/labs/service.ts @@ -2,6 +2,7 @@ import type { JsonObject, ProjectLabRow } from "../db/types"; import { DEFAULT_KALI_LAB_IMAGE, PREVIOUS_KALI_LAB_IMAGE, + resolveLabIsolationForWorkload, resolveLabIsolationMode, UPSTREAM_KALI_LAB_IMAGE, } from "./hardening"; @@ -17,6 +18,7 @@ import type { LabFirewallConfigInput, LabIsolationMode, LabRuntimeResult, + LabWorkloadClass, NormalizedLabCreateInput, ProjectLabRepository, ProjectLabStatusView, @@ -86,7 +88,13 @@ export class ProjectLabService { }); return this.statusFromLab(projectId, repaired); } - if (input.networkProfile || input.metadata) { + if ( + input.networkProfile || + input.isolation || + input.workloadClass || + input.microvmRuntimeClass || + input.metadata + ) { const updated = await this.repository.update(existing.id, { ...existing, // A running environment's active thread changes only after restart @@ -132,7 +140,12 @@ export class ProjectLabService { const requestedThreadId = cleanText(input.threadId); if (lab.status === "running") { const activeThreadId = cleanText(readMetadataText(lab.metadata.activeThreadId)); - if (requestedThreadId && activeThreadId !== requestedThreadId) { + if ( + (requestedThreadId && activeThreadId !== requestedThreadId) || + input.isolation || + input.workloadClass || + input.microvmRuntimeClass + ) { return this.restart(projectId, input); } return created; @@ -152,7 +165,13 @@ export class ProjectLabService { const networkProfile = readHumanNetworkProfile( input.networkProfile ?? claimed.metadata.networkProfile, ); - const isolation = readLabIsolationMode(input.isolation ?? claimed.metadata.isolation); + const workloadClass = readLabWorkloadClass( + input.workloadClass ?? claimed.metadata.workloadClass, + ); + const isolation = resolveLabIsolationForWorkload( + readOptionalLabIsolationMode(input.isolation ?? claimed.metadata.isolation), + workloadClass, + ); const microvmRuntimeClass = readMicrovmRuntimeClass( input.microvmRuntimeClass ?? claimed.metadata.microvmRuntimeClass, ); @@ -167,6 +186,7 @@ export class ProjectLabService { input.approvedTargets ?? claimed.metadata.approvedTargets, ), isolation, + workloadClass, microvmRuntimeClass, trafficRecording: readTrafficRecordingMode( input.trafficRecording ?? claimed.metadata.trafficRecording, @@ -193,6 +213,7 @@ export class ProjectLabService { ...input, networkProfile, isolation, + workloadClass, microvmRuntimeClass, trafficRecording: readTrafficRecordingMode( input.trafficRecording ?? claimed.metadata.trafficRecording, @@ -285,7 +306,11 @@ export class ProjectLabService { const networkProfile = readHumanNetworkProfile( input.networkProfile ?? lab.metadata.networkProfile, ); - const isolation = readLabIsolationMode(input.isolation ?? lab.metadata.isolation); + const workloadClass = readLabWorkloadClass(input.workloadClass ?? lab.metadata.workloadClass); + const isolation = resolveLabIsolationForWorkload( + readOptionalLabIsolationMode(input.isolation ?? lab.metadata.isolation), + workloadClass, + ); const microvmRuntimeClass = readMicrovmRuntimeClass( input.microvmRuntimeClass ?? lab.metadata.microvmRuntimeClass, ); @@ -301,6 +326,7 @@ export class ProjectLabService { input.approvedTargets ?? lab.metadata.approvedTargets, ), isolation, + workloadClass, microvmRuntimeClass, trafficRecording: readTrafficRecordingMode( input.trafficRecording ?? lab.metadata.trafficRecording, @@ -358,6 +384,7 @@ export class ProjectLabService { ...input, networkProfile, isolation, + workloadClass, microvmRuntimeClass, trafficRecording: readTrafficRecordingMode( input.trafficRecording ?? lab.metadata.trafficRecording, @@ -518,6 +545,7 @@ function labDefaults(input: LabCreateInput): NormalizedLabCreateInput { function labPreferenceMetadata(input: LabCreateInput): JsonObject { const microvmRuntimeClass = readMicrovmRuntimeClass(input.microvmRuntimeClass); + const workloadClass = readLabWorkloadClass(input.workloadClass); return mergeMetadata( mergeMetadata( sanitizeJsonObject(input.metadata), @@ -526,7 +554,11 @@ function labPreferenceMetadata(input: LabCreateInput): JsonObject { { networkProfile: readHumanNetworkProfile(input.networkProfile), approvedTargets: readApprovedTargets(input.approvedTargets), - isolation: readLabIsolationMode(input.isolation), + isolation: resolveLabIsolationForWorkload( + readOptionalLabIsolationMode(input.isolation), + workloadClass, + ), + workloadClass, ...(microvmRuntimeClass ? { microvmRuntimeClass } : {}), ...(input.trafficRecording === "mitmproxy" || input.trafficRecording === "disabled" ? { trafficRecording: input.trafficRecording } @@ -561,6 +593,14 @@ function readLabIsolationMode(value: unknown): LabIsolationMode { return resolveLabIsolationMode(value === "microvm" || value === "container" ? value : undefined); } +function readOptionalLabIsolationMode(value: unknown): LabIsolationMode | undefined { + return value === "microvm" || value === "container" ? value : undefined; +} + +function readLabWorkloadClass(value: unknown): LabWorkloadClass { + return value === "untrusted-code" || value === "malware-analysis" ? value : "standard"; +} + function readTrafficRecordingMode(value: unknown): TrafficRecordingMode { if (value === "mitmproxy" || value === "disabled") { return value; diff --git a/src/server/labs/types.ts b/src/server/labs/types.ts index 18dc81bd5..09489839d 100644 --- a/src/server/labs/types.ts +++ b/src/server/labs/types.ts @@ -36,6 +36,7 @@ export interface LabCreateInput { firewall?: LabFirewallConfigInput; credentialMounts?: LabCredentialMount[]; isolation?: LabIsolationMode; + workloadClass?: LabWorkloadClass; microvmRuntimeClass?: string; trafficRecording?: TrafficRecordingMode; metadata?: JsonObject; @@ -75,6 +76,7 @@ export type LabBoundary = "human" | "agent"; * for kernel-level isolation between the lab workload and the Docker host. */ export type LabIsolationMode = "container" | "microvm"; +export type LabWorkloadClass = "standard" | "untrusted-code" | "malware-analysis"; export type HumanLabNetworkProfileId = | "offline" @@ -130,6 +132,7 @@ export interface LabContainerOptions extends LabProjectRef { approvedTargets?: string[]; credentialMounts?: LabCredentialMount[]; isolation?: LabIsolationMode; + workloadClass?: LabWorkloadClass; microvmRuntimeClass?: string; trafficRecording?: TrafficRecordingMode; } diff --git a/tests/integration/lab-runtime.test.ts b/tests/integration/lab-runtime.test.ts index 79819d335..62d9b6641 100644 --- a/tests/integration/lab-runtime.test.ts +++ b/tests/integration/lab-runtime.test.ts @@ -774,6 +774,23 @@ describe("microVM isolation", () => { expect(command.args[command.args.indexOf("--runtime") + 1]).toBe("kata-qemu"); }); + it("labels high-risk workload classes on their MicroVM runtime", () => { + const command = buildRunLabCommand({ + ...humanOptions, + networkProfile: "offline", + isolation: "microvm", + workloadClass: "malware-analysis", + }); + + expect(command.args).toEqual( + expect.arrayContaining([ + "--runtime", + DEFAULT_MICROVM_RUNTIME_CLASS, + "exploit-hunter.workload-class=malware-analysis", + ]), + ); + }); + it("rejects unsafe microVM runtime class names", () => { expect(() => buildRunLabCommand({ @@ -831,6 +848,30 @@ describe("microVM isolation", () => { ).rejects.toBeInstanceOf(MicrovmRuntimeUnavailableError); }); + it("fails closed for high-risk workloads when no MicroVM runtime is registered", async () => { + const runner: DockerCommandRunner = { + async isAvailable() { + return true; + }, + async run(command) { + if (command.args[0] === "info") { + return { stdout: JSON.stringify({ runc: {} }), stderr: "", exitCode: 0 }; + } + throw new Error("must not provision a high-risk workload without a MicroVM runtime"); + }, + }; + const runtime = new ProjectLabRuntime({ mode: "docker", runner }); + + await expect( + runtime.start({ + projectId: "malware-analysis-missing-runtime", + boundary: "human", + networkProfile: "offline", + workloadClass: "malware-analysis", + }), + ).rejects.toBeInstanceOf(MicrovmRuntimeUnavailableError); + }); + it("starts the lab once the requested microVM runtime is registered with the Docker daemon", async () => { const executed: DockerCommandSpec[] = []; const runner: DockerCommandRunner = { diff --git a/tests/integration/project-lab.test.ts b/tests/integration/project-lab.test.ts index 6c7e0c45a..3279242f9 100644 --- a/tests/integration/project-lab.test.ts +++ b/tests/integration/project-lab.test.ts @@ -48,6 +48,49 @@ describe("project lab lifecycle", () => { expect(created.lab?.image_ref).toBe(DEFAULT_KALI_LAB_IMAGE); }); + it("requires MicroVM isolation for high-risk workload classes", async () => { + const service = buildTestLabService(new InMemoryProjectLabRepository()); + + const created = await service.create("project-1", { + workloadClass: "malware-analysis", + }); + expect(created.lab?.metadata).toMatchObject({ + workloadClass: "malware-analysis", + isolation: "microvm", + }); + + const rejectingService = buildTestLabService(new InMemoryProjectLabRepository()); + await expect( + rejectingService.create("project-1", { + workloadClass: "untrusted-code", + isolation: "container", + }), + ).rejects.toThrow(/requires MicroVM isolation/); + }); + + it("restarts a running lab when it is reclassified as high risk", async () => { + const service = buildTestLabService(new InMemoryProjectLabRepository()); + await service.start("project-1"); + + const reclassified = await service.start("project-1", { + workloadClass: "untrusted-code", + }); + + expect(reclassified.lab?.metadata).toMatchObject({ + workloadClass: "untrusted-code", + isolation: "microvm", + }); + expect(reclassified.lab?.runtime_metadata).toMatchObject({ + state: "restarted", + container: { + labels: { + "exploit-hunter.isolation": "microvm", + "exploit-hunter.workload-class": "untrusted-code", + }, + }, + }); + }); + it("repairs legacy simulated image refs before starting existing labs", async () => { const repository = new InMemoryProjectLabRepository(); repository.rows.push( From 3e3119ed9070b6f21b30c93a2df87eea4d8db8a7 Mon Sep 17 00:00:00 2001 From: Dan Levy Date: Thu, 27 Aug 2026 00:32:14 -0400 Subject: [PATCH 2/4] Format microVM isolation changes --- src/server/labs/hardening.ts | 7 +------ src/server/labs/service.ts | 9 +-------- src/server/labs/types.ts | 6 +----- tests/integration/lab-runtime.test.ts | 2 +- 4 files changed, 4 insertions(+), 20 deletions(-) diff --git a/src/server/labs/hardening.ts b/src/server/labs/hardening.ts index 9ecc1a512..eac087be9 100644 --- a/src/server/labs/hardening.ts +++ b/src/server/labs/hardening.ts @@ -1,9 +1,4 @@ -import type { - LabBoundary, - LabIsolationMode, - LabResourceLimits, - LabWorkloadClass, -} from "./types"; +import type { LabBoundary, LabIsolationMode, LabResourceLimits, LabWorkloadClass } from "./types"; export const PREVIOUS_KALI_LAB_IMAGE = "exploit-hunter/kali-workspace:latest"; export const DEFAULT_KALI_LAB_IMAGE = "exploit-hunter/kali-workspace:iptables"; diff --git a/src/server/labs/service.ts b/src/server/labs/service.ts index e034cfcd7..da4a167a7 100644 --- a/src/server/labs/service.ts +++ b/src/server/labs/service.ts @@ -3,7 +3,6 @@ import { DEFAULT_KALI_LAB_IMAGE, PREVIOUS_KALI_LAB_IMAGE, resolveLabIsolationForWorkload, - resolveLabIsolationMode, UPSTREAM_KALI_LAB_IMAGE, } from "./hardening"; import { buildUfwCommandPlan, normalizeLabFirewallConfig } from "./network-profiles"; @@ -322,9 +321,7 @@ export class ProjectLabService { boundary: "human", image: lab.image_ref, networkProfile, - approvedTargets: readApprovedTargets( - input.approvedTargets ?? lab.metadata.approvedTargets, - ), + approvedTargets: readApprovedTargets(input.approvedTargets ?? lab.metadata.approvedTargets), isolation, workloadClass, microvmRuntimeClass, @@ -589,10 +586,6 @@ function readApprovedTargets(value: unknown): string[] { ].slice(0, 64); } -function readLabIsolationMode(value: unknown): LabIsolationMode { - return resolveLabIsolationMode(value === "microvm" || value === "container" ? value : undefined); -} - function readOptionalLabIsolationMode(value: unknown): LabIsolationMode | undefined { return value === "microvm" || value === "container" ? value : undefined; } diff --git a/src/server/labs/types.ts b/src/server/labs/types.ts index 09489839d..eb61fdea4 100644 --- a/src/server/labs/types.ts +++ b/src/server/labs/types.ts @@ -78,11 +78,7 @@ export type LabBoundary = "human" | "agent"; export type LabIsolationMode = "container" | "microvm"; export type LabWorkloadClass = "standard" | "untrusted-code" | "malware-analysis"; -export type HumanLabNetworkProfileId = - | "offline" - | "approved-targets" - | "package-egress" - | "full"; +export type HumanLabNetworkProfileId = "offline" | "approved-targets" | "package-egress" | "full"; export type AgentLabNetworkProfileId = "offline" | "approved-targets" | "package-egress" | "full"; diff --git a/tests/integration/lab-runtime.test.ts b/tests/integration/lab-runtime.test.ts index 62d9b6641..1f69a1359 100644 --- a/tests/integration/lab-runtime.test.ts +++ b/tests/integration/lab-runtime.test.ts @@ -12,11 +12,11 @@ import { buildStartTrafficRecorderCommands, buildUfwCommandPlan, DEFAULT_KALI_LAB_IMAGE, - egressEnforcerImage, DEFAULT_MICROVM_RUNTIME_CLASS, type DockerCommandRunner, type DockerCommandSpec, dockerNetworkArgs, + egressEnforcerImage, HUMAN_LAB_NETWORK_PROFILES, HUMAN_LAB_PACKAGE_CAPABILITIES, labContainerIdentity, From 6221a447f767f44930c09c1cb83cb7ac7aabccbd Mon Sep 17 00:00:00 2001 From: Dan Levy Date: Thu, 27 Aug 2026 01:46:41 -0400 Subject: [PATCH 3/4] Attest MicroVM isolation after lab start --- docs/lab-runtime-hardening.md | 6 +- src/server/labs/docker-plan.ts | 23 ++ src/server/labs/runtime.ts | 296 +++++++++++++++++- src/server/labs/service.ts | 125 ++++++-- src/server/labs/types.ts | 29 ++ .../lab-attestation-persistence.test.ts | 98 ++++++ tests/integration/lab-runtime.test.ts | 232 +++++++++++++- tests/integration/project-lab.test.ts | 277 ++++++++++++++-- 8 files changed, 1021 insertions(+), 65 deletions(-) create mode 100644 tests/integration/lab-attestation-persistence.test.ts diff --git a/docs/lab-runtime-hardening.md b/docs/lab-runtime-hardening.md index eb5c0dd7a..9868dc00b 100644 --- a/docs/lab-runtime-hardening.md +++ b/docs/lab-runtime-hardening.md @@ -37,7 +37,11 @@ The default `container` isolation mode above shares the host kernel with every l ### Fail-closed behavior -Because isolation strength is a security property, the runtime never silently downgrades from `microvm` to plain containers. Before starting a lab with `isolation: "microvm"`, it runs `docker info --format '{{json .Runtimes}}'` and refuses to start (throwing `MicrovmRuntimeUnavailableError`) if the requested runtime class is not registered with the Docker daemon. In `dry-run` mode the availability check is included in the planned command list but is never executed or enforced. +Because isolation strength is a security property, the runtime never silently downgrades from `microvm` to plain containers. Before starting a lab with `isolation: "microvm"`, it runs `docker info --format '{{json .Runtimes}}'` and refuses to start (throwing `MicrovmRuntimeUnavailableError`) if the requested runtime class is not registered with the Docker daemon. + +Registration is only a preflight. After every MicroVM start path, the runtime also asks the external Docker daemon for the running container's ID, state, selected OCI runtime, isolation label, and workload-class label. The lab is trusted as running only when those observed facts match the request. Missing, malformed, stopped, or mismatched observations trigger fail-safe removal of the recorder, egress controller, and workload. The failed attestation and cleanup outcome remain in the lab's durable runtime metadata. + +`dry-run` mode records an attestation as `planned`, never `verified`. The project lab service refuses to persist a MicroVM lab as running until it receives a verified external attestation. This check proves what the Docker daemon selected; it does not independently prove guest-kernel health or cover ongoing namespace, mount, privilege, device, socket, firewall, or listener integrity. ### Host setup diff --git a/src/server/labs/docker-plan.ts b/src/server/labs/docker-plan.ts index 834063607..7470ceb57 100644 --- a/src/server/labs/docker-plan.ts +++ b/src/server/labs/docker-plan.ts @@ -529,6 +529,29 @@ export const buildMicrovmRuntimeCheckCommand = ( }; }; +export const buildMicrovmStartAttestationCommand = ( + options: LabContainerOptions, +): DockerCommandSpec | null => { + if ( + resolveLabIsolationForWorkload(options.isolation, options.workloadClass ?? "standard") !== + "microvm" + ) { + return null; + } + return { + command: "docker", + args: [ + "container", + "inspect", + "--format", + "{{json .Id}}\n{{json .State.Running}}\n{{json .HostConfig.Runtime}}\n{{json .Config.Labels}}", + labContainerIdentity(options).containerName, + ], + reason: + "Attest the running lab isolation boundary from the external Docker daemon before trusting it.", + }; +}; + export const buildStopLabCommand = (options: LabContainerOptions): DockerCommandSpec => ({ command: "docker", args: ["stop", labContainerIdentity(options).containerName], diff --git a/src/server/labs/runtime.ts b/src/server/labs/runtime.ts index d4f447f6b..6d8ea8a60 100644 --- a/src/server/labs/runtime.ts +++ b/src/server/labs/runtime.ts @@ -12,6 +12,7 @@ import { buildInspectLabImageCommand, buildInspectEgressEnforcerImageCommand, buildMicrovmRuntimeCheckCommand, + buildMicrovmStartAttestationCommand, buildProvisionLabCommands, buildRunLabCommand, buildStartEgressEnforcerCommands, @@ -33,6 +34,7 @@ import type { LabFirewallConfigInput, LabRuntimeMode, LabRuntimeResult, + LabRuntimeStartAttestation, LabTerminalCommandInput, LabTerminalCommandResult, } from "./types"; @@ -44,12 +46,22 @@ export class MicrovmRuntimeUnavailableError extends Error { super( `MicroVM isolation was requested but the "${runtimeClass}" Docker runtime is not registered on this host` + `${cause ? ` (${cause})` : ""}. Install and register Kata Containers (see docs/lab-runtime-hardening.md) ` + - `or set PROJECT_LAB_ISOLATION_MODE=container.`, + `before starting this workload; container fallback is never automatic.`, ); this.name = "MicrovmRuntimeUnavailableError"; } } +export class LabRuntimeAttestationError extends Error { + constructor( + readonly attestation: LabRuntimeStartAttestation, + readonly commands: DockerCommandSpec[], + ) { + super(attestation.failure ?? "The external runtime start attestation failed."); + this.name = "LabRuntimeAttestationError"; + } +} + export interface DockerCommandRunner { isAvailable(): Promise; run( @@ -116,7 +128,13 @@ export class DryRunDockerCommandRunner implements DockerCommandRunner { async run(command: DockerCommandSpec) { this.commands.push(command); - return { stdout: "", stderr: "", exitCode: 0, timedOut: false, killed: false }; + return { + stdout: "", + stderr: "", + exitCode: 0, + timedOut: false, + killed: false, + }; } } @@ -154,6 +172,11 @@ export class ProjectLabRuntime { } async start(options: LabContainerOptions): Promise { + const result = await this.startUnattested(options); + return this.attestStartedRuntime(options, result); + } + + private async startUnattested(options: LabContainerOptions): Promise { await mkdir(threadWorkspaceHostPath(options.projectId, options.threadId ?? "default-thread"), { recursive: true, }); @@ -310,6 +333,182 @@ export class ProjectLabRuntime { } } + private async attestStartedRuntime( + options: LabContainerOptions, + result: LabRuntimeResult, + ): Promise { + const command = buildMicrovmStartAttestationCommand(options); + if (!command) return result; + + const identity = labContainerIdentity(options); + const expectedRuntimeClass = resolveMicrovmRuntimeClass(options.microvmRuntimeClass); + const expectedWorkloadClass = options.workloadClass ?? "standard"; + const planned: LabRuntimeStartAttestation = { + observer: "docker-daemon", + disposition: "planned", + containerName: identity.containerName, + expectedRuntimeClass, + expectedIsolation: "microvm", + expectedWorkloadClass, + workloadRunning: result.mode === "docker", + }; + if (result.mode === "dry-run") { + return { + ...result, + commands: [...result.commands, command], + startAttestation: planned, + }; + } + + let observed: ReturnType = { + failure: "The Docker daemon did not return a start attestation.", + }; + try { + const response = await this.runner.run(command, { timeoutMs: 5_000 }); + observed = + response.exitCode === 0 + ? parseDockerStartObservation(response.stdout) + : { + failure: `Docker inspect exited with code ${response.exitCode}.`, + }; + } catch (error) { + observed = { + failure: `Docker inspect failed: ${error instanceof Error ? error.message : String(error)}`, + }; + } + + const failures = [ + observed.failure, + !observed.containerId ? "The container ID was missing." : null, + observed.running !== true ? "The container was not observed running." : null, + observed.runtimeClass !== expectedRuntimeClass + ? `Expected runtime ${expectedRuntimeClass}, observed ${observed.runtimeClass ?? "none"}.` + : null, + observed.labels?.["exploit-hunter.isolation"] !== "microvm" + ? "The observed isolation label was not microvm." + : null, + observed.labels?.["exploit-hunter.workload-class"] !== expectedWorkloadClass + ? `The observed workload class did not match ${expectedWorkloadClass}.` + : null, + ].filter((value): value is string => Boolean(value)); + + if (failures.length === 0) { + return { + ...result, + commands: [...result.commands, command], + startAttestation: { + ...planned, + disposition: "verified", + containerId: observed.containerId, + observedRuntimeClass: observed.runtimeClass, + observedIsolation: observed.labels?.["exploit-hunter.isolation"], + observedWorkloadClass: observed.labels?.["exploit-hunter.workload-class"], + running: true, + observedAt: new Date().toISOString(), + workloadRunning: true, + }, + }; + } + + const cleanupCommands = [ + buildDestroyTrafficRecorderCommand(options), + buildDestroyEgressEnforcerCommand(options), + buildDestroyLabContainerCommand(options), + ].filter((cleanup): cleanup is DockerCommandSpec => cleanup !== null); + let cleanupSucceeded = true; + let workloadCleanupSucceeded = false; + const workloadCleanup = cleanupCommands.at(-1); + const cleanupOutcomes: NonNullable = []; + for (const cleanup of cleanupCommands) { + let commandSucceeded = false; + try { + const outcome = await this.runner.run(cleanup); + commandSucceeded = outcome.exitCode === 0; + cleanupOutcomes.push({ + reason: cleanup.reason, + succeeded: commandSucceeded, + exitCode: outcome.exitCode, + ...(outcome.timedOut ? { timedOut: true } : {}), + ...(outcome.killed ? { killed: true } : {}), + ...(outcome.stderr.trim() ? { diagnostic: outcome.stderr.trim().slice(0, 2_000) } : {}), + }); + } catch (error) { + commandSucceeded = false; + cleanupOutcomes.push({ + reason: cleanup.reason, + succeeded: false, + diagnostic: (error instanceof Error ? error.message : String(error)).slice(0, 2_000), + }); + } + cleanupSucceeded &&= commandSucceeded; + if (cleanup === workloadCleanup) workloadCleanupSucceeded = commandSucceeded; + } + const verifyWorkloadRemoval: DockerCommandSpec = { + command: "docker", + args: ["container", "inspect", identity.containerName], + reason: "Verify from the Docker daemon that the failed-attestation workload no longer exists.", + }; + let workloadRemovalVerified = false; + let workloadRemovalFailure = "Workload removal could not be independently verified."; + try { + const outcome = await this.runner.run(verifyWorkloadRemoval, { timeoutMs: 5_000 }); + const diagnostic = `${outcome.stderr}\n${outcome.stdout}`.trim(); + workloadRemovalVerified = + outcome.exitCode !== 0 && isDockerContainerMissingMessage(diagnostic); + cleanupOutcomes.push({ + reason: verifyWorkloadRemoval.reason, + succeeded: workloadRemovalVerified, + exitCode: outcome.exitCode, + ...(outcome.timedOut ? { timedOut: true } : {}), + ...(outcome.killed ? { killed: true } : {}), + ...(diagnostic ? { diagnostic: diagnostic.slice(0, 2_000) } : {}), + }); + if (!workloadRemovalVerified && diagnostic) { + workloadRemovalFailure = `Workload removal verification failed: ${diagnostic.slice(0, 2_000)}`; + } + } catch (error) { + workloadRemovalVerified = isDockerContainerMissing(error); + cleanupOutcomes.push({ + reason: verifyWorkloadRemoval.reason, + succeeded: workloadRemovalVerified, + diagnostic: (error instanceof Error ? error.message : String(error)).slice(0, 2_000), + }); + if (!workloadRemovalVerified) { + workloadRemovalFailure = `Workload removal verification failed: ${error instanceof Error ? error.message : String(error)}`; + } + } + workloadCleanupSucceeded = workloadRemovalVerified; + cleanupSucceeded &&= workloadRemovalVerified; + const attestation: LabRuntimeStartAttestation = { + ...planned, + disposition: "failed", + ...(observed.containerId ? { containerId: observed.containerId } : {}), + ...(observed.runtimeClass ? { observedRuntimeClass: observed.runtimeClass } : {}), + ...(observed.labels?.["exploit-hunter.isolation"] + ? { observedIsolation: observed.labels["exploit-hunter.isolation"] } + : {}), + ...(observed.labels?.["exploit-hunter.workload-class"] + ? { + observedWorkloadClass: observed.labels["exploit-hunter.workload-class"], + } + : {}), + ...(typeof observed.running === "boolean" ? { running: observed.running } : {}), + observedAt: new Date().toISOString(), + failure: `External runtime start attestation failed: ${failures.join(" ")}${workloadRemovalVerified ? "" : ` ${workloadRemovalFailure}`}`, + cleanupAttempted: true, + cleanupSucceeded, + workloadCleanupSucceeded, + workloadRunning: !workloadCleanupSucceeded, + cleanupOutcomes, + }; + throw new LabRuntimeAttestationError(attestation, [ + ...result.commands, + command, + ...cleanupCommands, + verifyWorkloadRemoval, + ]); + } + private async startExistingAfterRunConflict( options: LabContainerOptions, identity: ReturnType, @@ -318,6 +517,23 @@ export class ProjectLabRuntime { if (!isDockerContainerNameConflict(error)) { throw error; } + if (buildMicrovmStartAttestationCommand(options)) { + const destroyCommand = buildDestroyLabContainerCommand(options); + const runCommand = buildRunLabCommand(options); + const destroyResult = await this.runner.run(destroyCommand); + if (destroyResult.exitCode !== 0) { + throw new Error( + "Refusing to start an existing container after a MicroVM name conflict because the incompatible container could not be removed.", + ); + } + const runResult = await this.runner.run(runCommand); + if (runResult.exitCode !== 0) { + throw new Error( + "Refusing to recover a MicroVM name conflict because the replacement container did not start.", + ); + } + return [destroyCommand, runCommand]; + } const startCommand: DockerCommandSpec = { command: "docker", args: ["start", identity.containerName], @@ -703,18 +919,35 @@ export class ProjectLabRuntime { const image = options.image ?? DEFAULT_KALI_LAB_IMAGE; const identity = labContainerIdentity(options); const expectedThreadId = identity.labels["exploit-hunter.thread-id"]; - const expectedLabels = `${image}|thread-bind-v1|${expectedThreadId}`; + const expectedIsolation = identity.labels["exploit-hunter.isolation"]; + const expectedWorkloadClass = identity.labels["exploit-hunter.workload-class"]; + const requiresMicrovm = expectedIsolation === "microvm"; + const expectedRuntimeClass = + requiresMicrovm ? resolveMicrovmRuntimeClass(options.microvmRuntimeClass) : ""; + const expectedLabels = requiresMicrovm + ? [ + image, + "thread-bind-v1", + expectedThreadId, + expectedRuntimeClass, + expectedIsolation, + expectedWorkloadClass, + ].join("|") + : `${image}|thread-bind-v1|${expectedThreadId}`; const inspectImageLabelCommand: DockerCommandSpec = { command: "docker", args: [ "container", "inspect", "--format", - '{{ index .Config.Labels "exploit-hunter.image-ref" }}|{{ index .Config.Labels "exploit-hunter.workspace-layout" }}|{{ index .Config.Labels "exploit-hunter.thread-id" }}', + requiresMicrovm + ? '{{ index .Config.Labels "exploit-hunter.image-ref" }}|{{ index .Config.Labels "exploit-hunter.workspace-layout" }}|{{ index .Config.Labels "exploit-hunter.thread-id" }}|{{ .HostConfig.Runtime }}|{{ index .Config.Labels "exploit-hunter.isolation" }}|{{ index .Config.Labels "exploit-hunter.workload-class" }}' + : '{{ index .Config.Labels "exploit-hunter.image-ref" }}|{{ index .Config.Labels "exploit-hunter.workspace-layout" }}|{{ index .Config.Labels "exploit-hunter.thread-id" }}', identity.containerName, ], - reason: - "Check whether the existing project lab container uses the configured image and thread workspace.", + reason: requiresMicrovm + ? "Check whether the existing project lab container matches the image, workspace, workload, and isolation boundary before starting it." + : "Check whether the existing project lab container uses the configured image and thread workspace.", }; try { @@ -734,6 +967,45 @@ export class ProjectLabRuntime { } } +function parseDockerStartObservation(stdout: string): { + containerId?: string; + running?: boolean; + runtimeClass?: string; + labels?: Record; + failure?: string; +} { + const lines = stdout.trimEnd().split("\n"); + if (lines.length !== 4) { + return { + failure: "The Docker daemon returned a malformed start attestation.", + }; + } + try { + const containerId = JSON.parse(lines[0] ?? "null"); + const running = JSON.parse(lines[1] ?? "null"); + const runtimeClass = JSON.parse(lines[2] ?? "null"); + const rawLabels = JSON.parse(lines[3] ?? "null"); + const labels = + rawLabels && typeof rawLabels === "object" && !Array.isArray(rawLabels) + ? Object.fromEntries( + Object.entries(rawLabels).filter( + (entry): entry is [string, string] => typeof entry[1] === "string", + ), + ) + : undefined; + return { + ...(typeof containerId === "string" && containerId ? { containerId } : {}), + ...(typeof running === "boolean" ? { running } : {}), + ...(typeof runtimeClass === "string" && runtimeClass ? { runtimeClass } : {}), + ...(labels ? { labels } : {}), + }; + } catch { + return { + failure: "The Docker daemon returned invalid JSON for the start attestation.", + }; + } +} + export function resolveLabEgressEnforcementMode(env: NodeJS.ProcessEnv): LabEgressEnforcementMode { const configured = env.PROJECT_LAB_EGRESS_ENFORCEMENT_MODE?.trim().toLowerCase(); if (configured === "development" || configured === "managed") return configured; @@ -757,6 +1029,14 @@ function isDockerAlreadyRunning(error: unknown) { return message.includes("is already running") || message.includes("already started"); } +function isDockerContainerMissing(error: unknown) { + return isDockerContainerMissingMessage(error instanceof Error ? error.message : String(error)); +} + +function isDockerContainerMissingMessage(message: string) { + return /no such (?:container|object)/i.test(message); +} + function spawnStreamingDockerCommand( command: DockerCommandSpec, options: { @@ -771,7 +1051,9 @@ function spawnStreamingDockerCommand( timedOut: boolean; killed: boolean; }>((resolve) => { - const child = spawn(command.command, command.args, { stdio: ["ignore", "pipe", "pipe"] }); + const child = spawn(command.command, command.args, { + stdio: ["ignore", "pipe", "pipe"], + }); const stdout: string[] = []; const stderr: string[] = []; let timedOut = false; diff --git a/src/server/labs/service.ts b/src/server/labs/service.ts index da4a167a7..c45a0d008 100644 --- a/src/server/labs/service.ts +++ b/src/server/labs/service.ts @@ -7,7 +7,11 @@ import { } from "./hardening"; import { buildUfwCommandPlan, normalizeLabFirewallConfig } from "./network-profiles"; import { sanitizeJsonObject, withProjectLabRepository } from "./repository"; -import { LabEgressEnforcementError, ProjectLabRuntime } from "./runtime"; +import { + LabEgressEnforcementError, + LabRuntimeAttestationError, + ProjectLabRuntime, +} from "./runtime"; import type { HumanLabNetworkProfileId, LabActionInput, @@ -194,6 +198,7 @@ export class ProjectLabService { input.credentialMounts ?? claimed.metadata.credentialMounts, ), }); + requireVerifiedMicrovmStart(isolation, runtimeResult); const runtimeId = claimed.runtime_id ?? runtimeResult.container.containerName; const running = await this.repository.update(claimed.id, { ...claimed, @@ -227,29 +232,52 @@ export class ProjectLabService { await this.repository.update(claimed.id, { ...claimed, status: "failed", - runtime_id: null, - container_id: null, + runtime_id: + error instanceof LabRuntimeAttestationError && error.attestation.workloadRunning + ? error.attestation.containerName + : null, + container_id: + error instanceof LabRuntimeAttestationError && error.attestation.workloadRunning + ? (error.attestation.containerId ?? error.attestation.containerName) + : null, runtime_metadata: - error instanceof LabEgressEnforcementError + error instanceof LabRuntimeAttestationError ? mergeMetadata(claimed.runtime_metadata, { provider: "docker", - state: "egress-enforcement-failed", + state: error.attestation.workloadRunning + ? "isolation-cleanup-failed" + : "isolation-attestation-failed", commands: error.commands.map((command) => ({ command: command.command, args: command.args, reason: command.reason, })), - egressEnforcement: error.enforcement as unknown as JsonObject, + startAttestation: error.attestation as unknown as JsonObject, }) - : claimed.runtime_metadata, + : error instanceof LabEgressEnforcementError + ? mergeMetadata(claimed.runtime_metadata, { + provider: "docker", + state: "egress-enforcement-failed", + commands: error.commands.map((command) => ({ + command: command.command, + args: command.args, + reason: command.reason, + })), + egressEnforcement: error.enforcement as unknown as JsonObject, + }) + : claimed.runtime_metadata, failure_reason: failure.message, last_error: failure, metadata: - error instanceof LabEgressEnforcementError + error instanceof LabRuntimeAttestationError ? mergeMetadata(claimed.metadata, { - egressEnforcement: error.enforcement as unknown as JsonObject, + startAttestation: error.attestation as unknown as JsonObject, }) - : claimed.metadata, + : error instanceof LabEgressEnforcementError + ? mergeMetadata(claimed.metadata, { + egressEnforcement: error.enforcement as unknown as JsonObject, + }) + : claimed.metadata, }); throw error; } @@ -332,34 +360,58 @@ export class ProjectLabService { input.credentialMounts ?? lab.metadata.credentialMounts, ), }); + requireVerifiedMicrovmStart(isolation, startResult); } catch (error) { const failure = serializeLabFailure(error); await this.repository.update(lab.id, { ...lab, status: "failed", - runtime_id: null, - container_id: null, + runtime_id: + error instanceof LabRuntimeAttestationError && error.attestation.workloadRunning + ? error.attestation.containerName + : null, + container_id: + error instanceof LabRuntimeAttestationError && error.attestation.workloadRunning + ? (error.attestation.containerId ?? error.attestation.containerName) + : null, runtime_metadata: - error instanceof LabEgressEnforcementError + error instanceof LabRuntimeAttestationError ? mergeMetadata(lab.runtime_metadata, { provider: "docker", - state: "egress-enforcement-failed", + state: error.attestation.workloadRunning + ? "isolation-cleanup-failed" + : "isolation-attestation-failed", commands: error.commands.map((command) => ({ command: command.command, args: command.args, reason: command.reason, })), - egressEnforcement: error.enforcement as unknown as JsonObject, + startAttestation: error.attestation as unknown as JsonObject, }) - : lab.runtime_metadata, + : error instanceof LabEgressEnforcementError + ? mergeMetadata(lab.runtime_metadata, { + provider: "docker", + state: "egress-enforcement-failed", + commands: error.commands.map((command) => ({ + command: command.command, + args: command.args, + reason: command.reason, + })), + egressEnforcement: error.enforcement as unknown as JsonObject, + }) + : lab.runtime_metadata, failure_reason: failure.message, last_error: failure, metadata: - error instanceof LabEgressEnforcementError + error instanceof LabRuntimeAttestationError ? mergeMetadata(lab.metadata, { - egressEnforcement: error.enforcement as unknown as JsonObject, + startAttestation: error.attestation as unknown as JsonObject, }) - : lab.metadata, + : error instanceof LabEgressEnforcementError + ? mergeMetadata(lab.metadata, { + egressEnforcement: error.enforcement as unknown as JsonObject, + }) + : lab.metadata, }); throw error; } @@ -674,6 +726,9 @@ function runtimeMetadata(state: string, result: LabRuntimeResult): JsonObject { dryRun: result.dryRun, state, egressEnforcement: result.egressEnforcement as unknown as JsonObject, + ...(result.startAttestation + ? { startAttestation: result.startAttestation as unknown as JsonObject } + : {}), container: { name: result.container.containerName, hostname: result.container.hostname, @@ -704,6 +759,14 @@ function mergeRuntimeResults( } function serializeLabFailure(error: unknown): JsonObject & { message: string } { + if (error instanceof LabRuntimeAttestationError) { + return { + name: error.name, + message: error.message, + ...(error.stack ? { stack: error.stack } : {}), + startAttestation: error.attestation as unknown as JsonObject, + }; + } if (error instanceof Error) { return { name: error.name, @@ -717,6 +780,30 @@ function serializeLabFailure(error: unknown): JsonObject & { message: string } { }; } +function requireVerifiedMicrovmStart(isolation: LabIsolationMode, result: LabRuntimeResult): void { + if (isolation !== "microvm" || result.startAttestation?.disposition === "verified") return; + const attestation = result.startAttestation ?? { + observer: "docker-daemon" as const, + disposition: "planned" as const, + containerName: result.container.containerName, + expectedRuntimeClass: "unknown", + expectedIsolation: "microvm" as const, + expectedWorkloadClass: "standard" as const, + workloadRunning: false, + }; + throw new LabRuntimeAttestationError( + { + ...attestation, + disposition: "failed", + failure: + "MicroVM lab startup remained a dry-run plan and was not externally verified; refusing to mark it running.", + cleanupAttempted: false, + workloadRunning: false, + }, + result.commands, + ); +} + function timestamp(): string { return new Date().toISOString(); } diff --git a/src/server/labs/types.ts b/src/server/labs/types.ts index eb61fdea4..0bb6d3526 100644 --- a/src/server/labs/types.ts +++ b/src/server/labs/types.ts @@ -169,12 +169,41 @@ export interface LabEgressEnforcementResult { workloadRunning?: boolean; } +export interface LabRuntimeStartAttestation { + observer: "docker-daemon"; + disposition: "planned" | "verified" | "failed"; + containerName: string; + containerId?: string; + expectedRuntimeClass: string; + observedRuntimeClass?: string; + expectedIsolation: "microvm"; + observedIsolation?: string; + expectedWorkloadClass: LabWorkloadClass; + observedWorkloadClass?: string; + running?: boolean; + observedAt?: string; + failure?: string; + cleanupAttempted?: boolean; + cleanupSucceeded?: boolean; + workloadCleanupSucceeded?: boolean; + workloadRunning?: boolean; + cleanupOutcomes?: Array<{ + reason: string; + succeeded: boolean; + exitCode?: number; + timedOut?: boolean; + killed?: boolean; + diagnostic?: string; + }>; +} + export interface LabRuntimeResult { mode: LabRuntimeMode; dryRun: boolean; commands: DockerCommandSpec[]; container: LabContainerIdentity; egressEnforcement: LabEgressEnforcementResult; + startAttestation?: LabRuntimeStartAttestation; } export interface LabTerminalCommandInput extends LabProjectRef { diff --git a/tests/integration/lab-attestation-persistence.test.ts b/tests/integration/lab-attestation-persistence.test.ts new file mode 100644 index 000000000..3c478cc92 --- /dev/null +++ b/tests/integration/lab-attestation-persistence.test.ts @@ -0,0 +1,98 @@ +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { afterEach, beforeEach, describe, expect, it } from "vitest"; + +import { getProjectStore } from "../../src/server/chat/projectAdapter"; +import { withDatabase } from "../../src/server/db/client"; +import { + type DockerCommandRunner, + ProjectLabRuntime, + ProjectLabService, + SqliteProjectLabRepository, +} from "../../src/server/labs"; + +describe("lab start attestation persistence", () => { + let databaseRoot: string; + let previousDatabaseUrl: string | undefined; + + beforeEach(async () => { + previousDatabaseUrl = process.env.EH_APP_DB_URL; + databaseRoot = await mkdtemp(join(tmpdir(), "exploit-hunter-lab-attestation-")); + process.env.EH_APP_DB_URL = `sqlite://${join(databaseRoot, "app.sqlite")}`; + }); + + afterEach(async () => { + if (previousDatabaseUrl === undefined) delete process.env.EH_APP_DB_URL; + else process.env.EH_APP_DB_URL = previousDatabaseUrl; + await rm(databaseRoot, { recursive: true, force: true }); + }); + + it("round-trips externally verified MicroVM facts through the durable repository", async () => { + const project = await (await getProjectStore()).createProject({ name: "Attested lab" }); + const started = await withDatabase((db) => + new ProjectLabService( + new SqliteProjectLabRepository(db), + new ProjectLabRuntime({ mode: "docker", runner: attestedDockerRunner() }), + ).start(project.id, { + workloadClass: "malware-analysis", + networkProfile: "full", + }), + ); + + const reloaded = await withDatabase((db) => + new ProjectLabService( + new SqliteProjectLabRepository(db), + new ProjectLabRuntime({ mode: "dry-run" }), + ).status(project.id), + ); + + expect(reloaded).toEqual(started); + expect(reloaded).toMatchObject({ + status: "running", + lab: { + runtime_metadata: { + startAttestation: { + observer: "docker-daemon", + disposition: "verified", + containerId: "durable-container-id", + expectedRuntimeClass: "kata-fc", + observedRuntimeClass: "kata-fc", + observedWorkloadClass: "malware-analysis", + running: true, + }, + }, + }, + }); + }); +}); + +function attestedDockerRunner(): DockerCommandRunner { + return { + async isAvailable() { + return true; + }, + async run(command) { + if (command.args[0] === "info") { + return { stdout: JSON.stringify({ "kata-fc": {}, runc: {} }), stderr: "", exitCode: 0 }; + } + if (command.reason.startsWith("Attest the running lab isolation boundary")) { + return { + stdout: [ + JSON.stringify("durable-container-id"), + JSON.stringify(true), + JSON.stringify("kata-fc"), + JSON.stringify({ + "exploit-hunter.isolation": "microvm", + "exploit-hunter.workload-class": "malware-analysis", + }), + ].join("\n"), + stderr: "", + exitCode: 0, + }; + } + return { stdout: "", stderr: "", exitCode: 0 }; + }, + }; +} diff --git a/tests/integration/lab-runtime.test.ts b/tests/integration/lab-runtime.test.ts index 1f69a1359..0900fc3fa 100644 --- a/tests/integration/lab-runtime.test.ts +++ b/tests/integration/lab-runtime.test.ts @@ -20,6 +20,7 @@ import { HUMAN_LAB_NETWORK_PROFILES, HUMAN_LAB_PACKAGE_CAPABILITIES, labContainerIdentity, + LabRuntimeAttestationError, MicrovmRuntimeUnavailableError, ProjectLabRuntime, trafficRecorderIdentity, @@ -230,7 +231,10 @@ describe("project lab Docker runtime primitives", () => { }); it("enforces restricted egress from an external capability-scoped sidecar", () => { - const options = { ...humanOptions, networkProfile: "package-egress" as const }; + const options = { + ...humanOptions, + networkProfile: "package-egress" as const, + }; const lab = labContainerIdentity(options); const workload = buildRunLabCommand(options); const commands = buildStartEgressEnforcerCommands(options); @@ -616,7 +620,10 @@ describe("project lab Docker runtime primitives", () => { timeoutMs: 1234, env: { AGENT_SECURITY_SAFE: "true" }, }); - const identity = labContainerIdentity({ projectId: "agent-shell-project", boundary: "agent" }); + const identity = labContainerIdentity({ + projectId: "agent-shell-project", + boundary: "agent", + }); const execCommand = executed[0]?.command; expect(result.stdout).toBe("ok\n"); @@ -680,7 +687,12 @@ describe("project lab Docker runtime primitives", () => { port: 443, protocol: "tcp" as const, }, - { action: "block" as const, host: "bad.example", port: 80, protocol: "tcp" as const }, + { + action: "block" as const, + host: "bad.example", + port: 80, + protocol: "tcp" as const, + }, ], }; @@ -740,7 +752,10 @@ describe("project lab Docker runtime primitives", () => { describe("microVM isolation", () => { it("stays on the plain container boundary by default, with no --runtime flag or check command", () => { - const command = buildRunLabCommand({ ...humanOptions, networkProfile: "offline" }); + const command = buildRunLabCommand({ + ...humanOptions, + networkProfile: "offline", + }); expect(command.args).not.toEqual(expect.arrayContaining(["--runtime"])); expect(command.args).toEqual(expect.arrayContaining(["exploit-hunter.isolation=container"])); @@ -821,6 +836,12 @@ describe("microVM isolation", () => { }); expect(result.mode).toBe("dry-run"); + expect(result.startAttestation).toMatchObject({ + observer: "docker-daemon", + disposition: "planned", + expectedRuntimeClass: DEFAULT_MICROVM_RUNTIME_CLASS, + workloadRunning: false, + }); expect(result.commands[0]?.args).toEqual(["info", "--format", "{{json .Runtimes}}"]); }); @@ -831,7 +852,11 @@ describe("microVM isolation", () => { }, async run(command) { if (command.args[0] === "info") { - return { stdout: JSON.stringify({ runc: {} }), stderr: "", exitCode: 0 }; + return { + stdout: JSON.stringify({ runc: {} }), + stderr: "", + exitCode: 0, + }; } throw new Error("must not provision a lab when the microVM runtime check fails"); }, @@ -855,7 +880,11 @@ describe("microVM isolation", () => { }, async run(command) { if (command.args[0] === "info") { - return { stdout: JSON.stringify({ runc: {} }), stderr: "", exitCode: 0 }; + return { + stdout: JSON.stringify({ runc: {} }), + stderr: "", + exitCode: 0, + }; } throw new Error("must not provision a high-risk workload without a MicroVM runtime"); }, @@ -881,10 +910,25 @@ describe("microVM isolation", () => { async run(command) { executed.push(command); if (command.args[0] === "info") { - return { stdout: JSON.stringify({ "kata-fc": {}, runc: {} }), stderr: "", exitCode: 0 }; + return { + stdout: JSON.stringify({ "kata-fc": {}, runc: {} }), + stderr: "", + exitCode: 0, + }; } if (command.args.includes('{{ index .Config.Labels "exploit-hunter.image-ref" }}')) { - return { stdout: `${DEFAULT_KALI_LAB_IMAGE}\n`, stderr: "", exitCode: 0 }; + return { + stdout: `${DEFAULT_KALI_LAB_IMAGE}\n`, + stderr: "", + exitCode: 0, + }; + } + if (command.reason.startsWith("Attest the running lab isolation boundary")) { + return { + stdout: dockerStartObservation("kata-fc", "standard"), + stderr: "", + exitCode: 0, + }; } return { stdout: "", stderr: "", exitCode: 0 }; }, @@ -899,6 +943,178 @@ describe("microVM isolation", () => { }); expect(result.mode).toBe("docker"); + expect(result.startAttestation).toMatchObject({ + observer: "docker-daemon", + disposition: "verified", + containerId: "container-id", + observedRuntimeClass: "kata-fc", + observedIsolation: "microvm", + running: true, + }); expect(executed[0]?.args).toEqual(["info", "--format", "{{json .Runtimes}}"]); }); + + it("recreates a stopped plain container before starting a reclassified high-risk workload", async () => { + const executed: DockerCommandSpec[] = []; + const runner: DockerCommandRunner = { + async isAvailable() { + return true; + }, + async run(command) { + executed.push(command); + if (command.args[0] === "info") { + return { stdout: JSON.stringify({ "kata-fc": {}, runc: {} }), stderr: "", exitCode: 0 }; + } + if (command.reason.startsWith("Check whether the existing project lab container matches")) { + return { + stdout: `${DEFAULT_KALI_LAB_IMAGE}|thread-bind-v1|default-thread|runc|container|standard`, + stderr: "", + exitCode: 0, + }; + } + if (command.reason.startsWith("Attest the running lab isolation boundary")) { + return { + stdout: dockerStartObservation("kata-fc", "untrusted-code"), + stderr: "", + exitCode: 0, + }; + } + return { stdout: "", stderr: "", exitCode: 0 }; + }, + }; + const runtime = new ProjectLabRuntime({ mode: "docker", runner }); + + await expect( + runtime.start({ + projectId: "reclassified-project", + boundary: "human", + networkProfile: "offline", + workloadClass: "untrusted-code", + }), + ).resolves.toMatchObject({ startAttestation: { disposition: "verified" } }); + + const destroyIndex = executed.findIndex((command) => + command.reason.includes("Remove the project lab container"), + ); + const runIndex = executed.findIndex((command) => command.args[0] === "run"); + expect(destroyIndex).toBeGreaterThan(-1); + expect(runIndex).toBeGreaterThan(destroyIndex); + expect(executed.some((command) => command.args[0] === "start")).toBe(false); + }); + + it("never starts an existing container while recovering a MicroVM name conflict", async () => { + const executed: DockerCommandSpec[] = []; + let runAttempts = 0; + const runner: DockerCommandRunner = { + async isAvailable() { + return true; + }, + async run(command) { + executed.push(command); + if (command.args[0] === "info") { + return { stdout: JSON.stringify({ "kata-fc": {}, runc: {} }), stderr: "", exitCode: 0 }; + } + if (command.reason === "Check whether the persistent project lab container already exists.") { + throw new Error("No such container"); + } + if (command.args[0] === "run") { + runAttempts += 1; + if (runAttempts === 1) { + throw new Error( + 'docker: Error response from daemon: Conflict. The container name "/lab" is already in use.', + ); + } + } + if (command.reason.startsWith("Attest the running lab isolation boundary")) { + return { + stdout: dockerStartObservation("kata-fc", "malware-analysis"), + stderr: "", + exitCode: 0, + }; + } + return { stdout: "", stderr: "", exitCode: 0 }; + }, + }; + const runtime = new ProjectLabRuntime({ mode: "docker", runner }); + + await expect( + runtime.start({ + projectId: "microvm-name-conflict", + boundary: "human", + networkProfile: "offline", + workloadClass: "malware-analysis", + }), + ).resolves.toMatchObject({ startAttestation: { disposition: "verified" } }); + expect(runAttempts).toBe(2); + expect(executed.some((command) => command.args[0] === "start")).toBe(false); + }); + + it("removes the workload when the external daemon observes a container runtime mismatch", async () => { + const executed: DockerCommandSpec[] = []; + const runner: DockerCommandRunner = { + async isAvailable() { + return true; + }, + async run(command) { + executed.push(command); + if (command.args[0] === "info") { + return { + stdout: JSON.stringify({ "kata-fc": {}, runc: {} }), + stderr: "", + exitCode: 0, + }; + } + if (command.reason.startsWith("Attest the running lab isolation boundary")) { + return { + stdout: dockerStartObservation("runc", "malware-analysis"), + stderr: "", + exitCode: 0, + }; + } + if (command.reason.includes("Remove the external project egress enforcer")) { + return { stdout: "", stderr: "sidecar cleanup failed", exitCode: 1 }; + } + if (command.reason.startsWith("Verify from the Docker daemon")) { + throw new Error("No such container"); + } + return { stdout: "", stderr: "", exitCode: 0 }; + }, + }; + const runtime = new ProjectLabRuntime({ mode: "docker", runner }); + + await expect( + runtime.start({ + projectId: "microvm-mismatch-project", + boundary: "human", + networkProfile: "package-egress", + workloadClass: "malware-analysis", + }), + ).rejects.toMatchObject({ + name: LabRuntimeAttestationError.name, + attestation: { + disposition: "failed", + expectedRuntimeClass: "kata-fc", + observedRuntimeClass: "runc", + cleanupAttempted: true, + cleanupSucceeded: false, + workloadCleanupSucceeded: true, + workloadRunning: false, + }, + }); + expect( + executed.some((command) => command.reason.includes("Remove the project lab container")), + ).toBe(true); + }); }); + +function dockerStartObservation(runtimeClass: string, workloadClass: string): string { + return [ + JSON.stringify("container-id"), + JSON.stringify(true), + JSON.stringify(runtimeClass), + JSON.stringify({ + "exploit-hunter.isolation": "microvm", + "exploit-hunter.workload-class": workloadClass, + }), + ].join("\n"); +} diff --git a/tests/integration/project-lab.test.ts b/tests/integration/project-lab.test.ts index 3279242f9..263225a7c 100644 --- a/tests/integration/project-lab.test.ts +++ b/tests/integration/project-lab.test.ts @@ -29,8 +29,13 @@ describe("project lab lifecycle", () => { const repository = new InMemoryProjectLabRepository(); const service = buildTestLabService(repository); - const first = await service.create("project-1", { imageRef: "lab:test", profile: "web" }); - const second = await service.create("project-1", { imageRef: "ignored:test" }); + const first = await service.create("project-1", { + imageRef: "lab:test", + profile: "web", + }); + const second = await service.create("project-1", { + imageRef: "ignored:test", + }); expect(repository.createCount).toBe(1); expect(first.status).toBe("stopped"); @@ -68,27 +73,141 @@ describe("project lab lifecycle", () => { ).rejects.toThrow(/requires MicroVM isolation/); }); - it("restarts a running lab when it is reclassified as high risk", async () => { - const service = buildTestLabService(new InMemoryProjectLabRepository()); + it("refuses to represent a dry-run high-risk workload as externally verified", async () => { + const repository = new InMemoryProjectLabRepository(); + const service = buildTestLabService(repository); await service.start("project-1"); - const reclassified = await service.start("project-1", { - workloadClass: "untrusted-code", + await expect( + service.start("project-1", { + workloadClass: "untrusted-code", + }), + ).rejects.toThrow(/was not externally verified/); + + await expect(service.status("project-1")).resolves.toMatchObject({ + status: "failed", + lab: { + runtime_id: null, + container_id: null, + runtime_metadata: { + state: "isolation-attestation-failed", + startAttestation: { + disposition: "failed", + observer: "docker-daemon", + workloadRunning: false, + }, + }, + }, }); + }); - expect(reclassified.lab?.metadata).toMatchObject({ - workloadClass: "untrusted-code", - isolation: "microvm", + it("persists a verified external start attestation for a high-risk workload", async () => { + const repository = new InMemoryProjectLabRepository(); + const service = new ProjectLabService( + repository, + new ProjectLabRuntime({ + mode: "docker", + runner: microvmAttestationRunner("kata-fc"), + }), + ); + + const started = await service.start("project-1", { + workloadClass: "malware-analysis", + networkProfile: "full", }); - expect(reclassified.lab?.runtime_metadata).toMatchObject({ - state: "restarted", - container: { - labels: { - "exploit-hunter.isolation": "microvm", - "exploit-hunter.workload-class": "untrusted-code", + + expect(started).toMatchObject({ + status: "running", + lab: { + runtime_metadata: { + startAttestation: { + disposition: "verified", + observer: "docker-daemon", + expectedRuntimeClass: "kata-fc", + observedRuntimeClass: "kata-fc", + observedWorkloadClass: "malware-analysis", + running: true, + }, + }, + }, + }); + expect(await service.status("project-1")).toEqual(started); + }); + + it("persists failed external attestation evidence and clears runtime identity", async () => { + const repository = new InMemoryProjectLabRepository(); + const service = new ProjectLabService( + repository, + new ProjectLabRuntime({ + mode: "docker", + runner: microvmAttestationRunner("runc"), + }), + ); + + await expect( + service.start("project-1", { + workloadClass: "malware-analysis", + networkProfile: "full", + }), + ).rejects.toThrow(/Expected runtime kata-fc, observed runc/); + + await expect(service.status("project-1")).resolves.toMatchObject({ + status: "failed", + lab: { + runtime_id: null, + container_id: null, + runtime_metadata: { + state: "isolation-attestation-failed", + startAttestation: { + disposition: "failed", + expectedRuntimeClass: "kata-fc", + observedRuntimeClass: "runc", + cleanupAttempted: true, + cleanupSucceeded: true, + workloadRunning: false, + }, + }, + }, + }); + }); + + it("preserves the cleanup target when workload removal cannot be externally verified", async () => { + const repository = new InMemoryProjectLabRepository(); + const service = new ProjectLabService( + repository, + new ProjectLabRuntime({ + mode: "docker", + runner: microvmAttestationRunner("runc", { + verificationError: "Docker daemon unavailable", + }), + }), + ); + + await expect( + service.start("project-1", { + workloadClass: "malware-analysis", + networkProfile: "full", + }), + ).rejects.toThrow(/Expected runtime kata-fc, observed runc/); + + const failed = await service.status("project-1"); + expect(failed).toMatchObject({ + status: "failed", + lab: { + container_id: "container-id", + runtime_metadata: { + state: "isolation-cleanup-failed", + startAttestation: { + workloadCleanupSucceeded: false, + workloadRunning: true, + }, }, }, }); + const attestation = failed.lab?.runtime_metadata.startAttestation as { + containerName: string; + }; + expect(failed.lab?.runtime_id).toBe(attestation.containerName); }); it("repairs legacy simulated image refs before starting existing labs", async () => { @@ -194,14 +313,19 @@ describe("project lab lifecycle", () => { const [first, second] = await Promise.all([firstStart, secondStart]); expect(first.status).toBe("running"); expect(["provisioning", "running"]).toContain(second.status); - await expect(service.status("project-1")).resolves.toMatchObject({ status: "running" }); + await expect(service.status("project-1")).resolves.toMatchObject({ + status: "running", + }); }); it("persists a failed lifecycle state when Docker provisioning fails", async () => { const repository = new InMemoryProjectLabRepository(); const service = new ProjectLabService( repository, - new ProjectLabRuntime({ mode: "docker", runner: new FailingDockerRunner() }), + new ProjectLabRuntime({ + mode: "docker", + runner: new FailingDockerRunner(), + }), ); await expect(service.start("project-1")).rejects.toThrow("managed image build failed"); @@ -235,7 +359,9 @@ describe("project lab lifecycle", () => { last_error: { name: "ProvisioningLeaseExpired" }, }, }); - await expect(service.start("project-1")).resolves.toMatchObject({ status: "running" }); + await expect(service.start("project-1")).resolves.toMatchObject({ + status: "running", + }); }); it("starts an existing named container when docker run reports a name conflict", async () => { @@ -304,7 +430,9 @@ describe("project lab lifecycle", () => { const running = await service.start("project-1"); const restarted = await service.restart("project-1"); - const commands = restarted.lab?.runtime_metadata.commands as Array<{ args: string[] }>; + const commands = restarted.lab?.runtime_metadata.commands as Array<{ + args: string[]; + }>; expect(repository.createCount).toBe(1); expect(restarted.status).toBe("running"); @@ -334,7 +462,9 @@ describe("project lab lifecycle", () => { const service = buildTestLabService(repository); const first = await service.start("project-1", { threadId: "thread-one" }); - const switched = await service.start("project-1", { threadId: "thread-two" }); + const switched = await service.start("project-1", { + threadId: "thread-two", + }); expect(switched.lab?.id).toBe(first.lab?.id); expect(switched.lab?.metadata.activeThreadId).toBe("thread-two"); @@ -351,9 +481,13 @@ describe("project lab lifecycle", () => { const repository = new InMemoryProjectLabRepository(); const service = buildTestLabService(repository); - const stopped = await service.create("project-1", { networkProfile: "full" }); + const stopped = await service.create("project-1", { + networkProfile: "full", + }); const running = await service.start("project-1"); - const restarted = await service.restart("project-1", { networkProfile: "package-egress" }); + const restarted = await service.restart("project-1", { + networkProfile: "package-egress", + }); expect(stopped.lab?.metadata.networkProfile).toBe("full"); expect(running.lab?.metadata.networkProfile).toBe("full"); @@ -375,7 +509,9 @@ describe("project lab lifecycle", () => { }); const service = new ProjectLabService(repository, runtime); - const started = await service.start("project-1", { networkProfile: "package-egress" }); + const started = await service.start("project-1", { + networkProfile: "package-egress", + }); const reloaded = await service.status("project-1"); expect(started.status).toBe("running"); @@ -403,7 +539,10 @@ describe("project lab lifecycle", () => { const reloaded = await service.status("project-1"); expect(reloaded.status).toBe("failed"); - expect(reloaded.lab).toMatchObject({ runtime_id: null, container_id: null }); + expect(reloaded.lab).toMatchObject({ + runtime_id: null, + container_id: null, + }); expect(reloaded.lab?.runtime_metadata.egressEnforcement).toMatchObject({ mode: "managed", disposition: "enforcement-failed-safe", @@ -435,7 +574,13 @@ describe("project lab lifecycle", () => { ); expect(updated.lab?.metadata.firewall).toMatchObject({ defaultPolicy: "deny", - rules: [expect.objectContaining({ action: "allow", host: "example.com", port: 443 })], + rules: [ + expect.objectContaining({ + action: "allow", + host: "example.com", + port: 443, + }), + ], }); expect(updated.lab?.metadata.firewallUfwPlan).toEqual( expect.arrayContaining([ @@ -477,6 +622,50 @@ describe("project lab lifecycle", () => { }); }); +const microvmAttestationRunner = ( + observedRuntimeClass: string, + options: { failWorkloadCleanup?: boolean; verificationError?: string } = {}, +): DockerCommandRunner => ({ + async isAvailable() { + return true; + }, + async run(command) { + if (command.args[0] === "info") { + return { + stdout: JSON.stringify({ "kata-fc": {}, runc: {} }), + stderr: "", + exitCode: 0, + }; + } + if (command.reason.startsWith("Attest the running lab isolation boundary")) { + return { + stdout: [ + JSON.stringify("container-id"), + JSON.stringify(true), + JSON.stringify(observedRuntimeClass), + JSON.stringify({ + "exploit-hunter.isolation": "microvm", + "exploit-hunter.workload-class": "malware-analysis", + }), + ].join("\n"), + stderr: "", + exitCode: 0, + }; + } + if (options.failWorkloadCleanup && command.reason.includes("Remove the project lab container")) { + return { stdout: "", stderr: "workload cleanup failed", exitCode: 1 }; + } + if (command.reason.startsWith("Verify from the Docker daemon")) { + if (options.verificationError) throw new Error(options.verificationError); + if (options.failWorkloadCleanup) { + return { stdout: "container still exists", stderr: "", exitCode: 0 }; + } + throw new Error("No such container"); + } + return { stdout: "", stderr: "", exitCode: 0 }; + }, +}); + const buildTestLabService = (repository: ProjectLabRepository) => new ProjectLabService(repository, new ProjectLabRuntime({ mode: "dry-run" })); @@ -572,7 +761,11 @@ class InMemoryProjectLabRepository implements ProjectLabRepository { throw new Error(`Lab ${labId} was not found.`); } - const updated = { ...this.rows[index], ...input, updated_at: new Date().toISOString() }; + const updated = { + ...this.rows[index], + ...input, + updated_at: new Date().toISOString(), + }; this.rows[index] = updated; return updated; } @@ -602,7 +795,13 @@ class BlockingDockerRunner implements DockerCommandRunner { if (command.args[0] === "container" && command.args[1] === "inspect") { throw new Error("No such container"); } - return { stdout: "", stderr: "", exitCode: 0, timedOut: false, killed: false }; + return { + stdout: "", + stderr: "", + exitCode: 0, + timedOut: false, + killed: false, + }; } waitUntilStarted() { @@ -626,7 +825,13 @@ class FailingDockerRunner implements DockerCommandRunner { if (command.args[0] === "build") { throw new Error("managed image build failed"); } - return { stdout: "", stderr: "", exitCode: 0, timedOut: false, killed: false }; + return { + stdout: "", + stderr: "", + exitCode: 0, + timedOut: false, + killed: false, + }; } } @@ -648,7 +853,13 @@ class ConflictRecoveringDockerRunner implements DockerCommandRunner { 'docker: Error response from daemon: Conflict. The container name "/exploit-hunter-human-project-1-lab" is already in use by container "abc123".', ); } - return { stdout: "", stderr: "", exitCode: 0, timedOut: false, killed: false }; + return { + stdout: "", + stderr: "", + exitCode: 0, + timedOut: false, + killed: false, + }; } } @@ -684,7 +895,13 @@ class RecreateConflictRecoveringDockerRunner implements DockerCommandRunner { 'docker: Error response from daemon: Conflict. The container name "/exploit-hunter-human-project-1-lab" is already in use by container "abc123".', ); } - return { stdout: "", stderr: "", exitCode: 0, timedOut: false, killed: false }; + return { + stdout: "", + stderr: "", + exitCode: 0, + timedOut: false, + killed: false, + }; } } From dad17f0392a20dc23c0e11fdadbe75bfbf935987 Mon Sep 17 00:00:00 2001 From: Dan Levy Date: Thu, 27 Aug 2026 02:18:52 -0400 Subject: [PATCH 4/4] Persist MicroVM containment failure artifacts --- docs/lab-runtime-hardening.md | 2 + src/server/evidence/ingestion.ts | 1 + src/server/labs/service.ts | 107 ++++++++- .../lab-attestation-persistence.test.ts | 224 +++++++++++++++++- 4 files changed, 327 insertions(+), 7 deletions(-) diff --git a/docs/lab-runtime-hardening.md b/docs/lab-runtime-hardening.md index 9868dc00b..23e07cdbc 100644 --- a/docs/lab-runtime-hardening.md +++ b/docs/lab-runtime-hardening.md @@ -41,6 +41,8 @@ Because isolation strength is a security property, the runtime never silently do Registration is only a preflight. After every MicroVM start path, the runtime also asks the external Docker daemon for the running container's ID, state, selected OCI runtime, isolation label, and workload-class label. The lab is trusted as running only when those observed facts match the request. Missing, malformed, stopped, or mismatched observations trigger fail-safe removal of the recorder, egress controller, and workload. The failed attestation and cleanup outcome remain in the lab's durable runtime metadata. +The project lab service also records each failed MicroVM start attestation as a redacted canonical Artifact using the `exploit-hunter.lab-containment-event.v1` schema. The event distinguishes an isolation-attestation failure from an unresolved cleanup failure, preserves requested and daemon-observed isolation facts, records the cleanup target and independently verified removal outcome, and carries the project, active thread, and trusted research-run correlation available at the lifecycle boundary. A verified start does not emit a violation Artifact. Artifact storage is forensic best effort: an Artifact outage is surfaced in diagnostics but never replaces, suppresses, or weakens the original fail-closed containment result. + `dry-run` mode records an attestation as `planned`, never `verified`. The project lab service refuses to persist a MicroVM lab as running until it receives a verified external attestation. This check proves what the Docker daemon selected; it does not independently prove guest-kernel health or cover ongoing namespace, mount, privilege, device, socket, firewall, or listener integrity. ### Host setup diff --git a/src/server/evidence/ingestion.ts b/src/server/evidence/ingestion.ts index a5554206a..a8de7bb8b 100644 --- a/src/server/evidence/ingestion.ts +++ b/src/server/evidence/ingestion.ts @@ -23,6 +23,7 @@ export const EVIDENCE_SOURCES = [ "patch-verification", "patch-remediation", "stage-handoff", + "containment-observation", ] as const; export type EvidenceSource = (typeof EVIDENCE_SOURCES)[number]; diff --git a/src/server/labs/service.ts b/src/server/labs/service.ts index c45a0d008..2e08e60aa 100644 --- a/src/server/labs/service.ts +++ b/src/server/labs/service.ts @@ -1,4 +1,6 @@ import type { JsonObject, ProjectLabRow } from "../db/types"; +import { getArtifactService, redactEvidenceSecrets } from "../evidence"; +import { readResearchRunContext } from "../research/run-context"; import { DEFAULT_KALI_LAB_IMAGE, PREVIOUS_KALI_LAB_IMAGE, @@ -244,9 +246,7 @@ export class ProjectLabService { error instanceof LabRuntimeAttestationError ? mergeMetadata(claimed.runtime_metadata, { provider: "docker", - state: error.attestation.workloadRunning - ? "isolation-cleanup-failed" - : "isolation-attestation-failed", + state: containmentFailureEventClass(error.attestation), commands: error.commands.map((command) => ({ command: command.command, args: command.args, @@ -279,6 +279,9 @@ export class ProjectLabService { }) : claimed.metadata, }); + if (error instanceof LabRuntimeAttestationError) { + await persistContainmentEventArtifact(projectId, input, error); + } throw error; } } @@ -378,9 +381,7 @@ export class ProjectLabService { error instanceof LabRuntimeAttestationError ? mergeMetadata(lab.runtime_metadata, { provider: "docker", - state: error.attestation.workloadRunning - ? "isolation-cleanup-failed" - : "isolation-attestation-failed", + state: containmentFailureEventClass(error.attestation), commands: error.commands.map((command) => ({ command: command.command, args: command.args, @@ -413,6 +414,9 @@ export class ProjectLabService { }) : lab.metadata, }); + if (error instanceof LabRuntimeAttestationError) { + await persistContainmentEventArtifact(projectId, input, error); + } throw error; } const runtimeResult = mergeRuntimeResults(destroyResult, startResult); @@ -780,6 +784,97 @@ function serializeLabFailure(error: unknown): JsonObject & { message: string } { }; } +const LAB_CONTAINMENT_EVENT_SCHEMA = "exploit-hunter.lab-containment-event.v1"; + +function containmentFailureEventClass( + attestation: LabRuntimeAttestationError["attestation"], +): "isolation-attestation-failed" | "isolation-cleanup-failed" { + return attestation.workloadRunning || + (attestation.cleanupAttempted && attestation.cleanupSucceeded !== true) + ? "isolation-cleanup-failed" + : "isolation-attestation-failed"; +} + +async function persistContainmentEventArtifact( + projectId: string, + input: LabCreateInput, + error: LabRuntimeAttestationError, +): Promise { + const attestation = error.attestation; + const eventClass = containmentFailureEventClass(attestation); + const threadId = cleanText(input.threadId) ?? undefined; + const researchRunContext = readResearchRunContext(); + const cleanupTarget = attestation.containerId ?? attestation.containerName; + const content = { + schema: LAB_CONTAINMENT_EVENT_SCHEMA, + eventClass, + observedAt: attestation.observedAt ?? timestamp(), + correlation: { + projectId, + ...(threadId ? { threadId } : {}), + ...(researchRunContext ?? {}), + }, + requested: { + isolation: attestation.expectedIsolation, + runtimeClass: attestation.expectedRuntimeClass, + workloadClass: attestation.expectedWorkloadClass, + }, + observed: { + containerName: attestation.containerName, + ...(attestation.containerId ? { containerId: attestation.containerId } : {}), + ...(attestation.observedRuntimeClass + ? { runtimeClass: attestation.observedRuntimeClass } + : {}), + ...(attestation.observedIsolation ? { isolation: attestation.observedIsolation } : {}), + ...(attestation.observedWorkloadClass + ? { workloadClass: attestation.observedWorkloadClass } + : {}), + ...(typeof attestation.running === "boolean" ? { running: attestation.running } : {}), + }, + cleanup: { + attempted: attestation.cleanupAttempted === true, + succeeded: attestation.cleanupSucceeded === true, + workloadRemovalVerified: attestation.workloadCleanupSucceeded === true, + workloadRunning: attestation.workloadRunning === true, + target: cleanupTarget, + outcomes: attestation.cleanupOutcomes ?? [], + }, + failure: attestation.failure ?? error.message, + }; + + try { + await getArtifactService().createArtifact({ + projectId, + ...(threadId ? { threadId } : {}), + projectScoped: true, + name: `containment-${eventClass}.json`, + kind: "log", + contentType: "application/json", + content: JSON.stringify(content, null, 2), + agentGenerated: true, + source: "containment-observation", + metadata: { + schema: LAB_CONTAINMENT_EVENT_SCHEMA, + eventClass, + requestedIsolation: attestation.expectedIsolation, + requestedRuntimeClass: attestation.expectedRuntimeClass, + requestedWorkloadClass: attestation.expectedWorkloadClass, + cleanupAttempted: attestation.cleanupAttempted === true, + cleanupSucceeded: attestation.cleanupSucceeded === true, + workloadRemovalVerified: attestation.workloadCleanupSucceeded === true, + workloadRunning: attestation.workloadRunning === true, + cleanupTarget, + }, + }); + } catch (artifactError) { + const artifactFailure = + artifactError instanceof Error ? artifactError.message : String(artifactError); + console.warn( + `[project-lab] containment failure persisted without an Artifact: ${redactEvidenceSecrets(artifactFailure)}`, + ); + } +} + function requireVerifiedMicrovmStart(isolation: LabIsolationMode, result: LabRuntimeResult): void { if (isolation !== "microvm" || result.startAttestation?.disposition === "verified") return; const attestation = result.startAttestation ?? { diff --git a/tests/integration/lab-attestation-persistence.test.ts b/tests/integration/lab-attestation-persistence.test.ts index 3c478cc92..37c5aba93 100644 --- a/tests/integration/lab-attestation-persistence.test.ts +++ b/tests/integration/lab-attestation-persistence.test.ts @@ -2,16 +2,21 @@ import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { getProjectStore } from "../../src/server/chat/projectAdapter"; import { withDatabase } from "../../src/server/db/client"; +import { + createArtifactService, + setArtifactService, +} from "../../src/server/evidence"; import { type DockerCommandRunner, ProjectLabRuntime, ProjectLabService, SqliteProjectLabRepository, } from "../../src/server/labs"; +import { withResearchRunContext } from "../../src/server/research/run-context"; describe("lab start attestation persistence", () => { let databaseRoot: string; @@ -21,9 +26,12 @@ describe("lab start attestation persistence", () => { previousDatabaseUrl = process.env.EH_APP_DB_URL; databaseRoot = await mkdtemp(join(tmpdir(), "exploit-hunter-lab-attestation-")); process.env.EH_APP_DB_URL = `sqlite://${join(databaseRoot, "app.sqlite")}`; + setArtifactService(createArtifactService({ storage: null })); }); afterEach(async () => { + setArtifactService(undefined); + vi.restoreAllMocks(); if (previousDatabaseUrl === undefined) delete process.env.EH_APP_DB_URL; else process.env.EH_APP_DB_URL = previousDatabaseUrl; await rm(databaseRoot, { recursive: true, force: true }); @@ -65,9 +73,185 @@ describe("lab start attestation persistence", () => { }, }, }); + const artifacts = await withDatabase((db) => + db.query("SELECT id FROM artifacts WHERE project_id = $1", [project.id]), + ); + expect(artifacts.rows).toHaveLength(0); + }); + + it("persists a correlated redacted Artifact for a failed external attestation", async () => { + const store = await getProjectStore(); + const project = await store.createProject({ name: "Failed attestation lab" }); + const thread = await store.createThread(project.id, { title: "Isolation review" }); + + await expect( + withResearchRunContext( + { researchRunId: "research-run-1", researchTurnLedgerId: "turn-ledger-1" }, + () => + withDatabase((db) => + new ProjectLabService( + new SqliteProjectLabRepository(db), + new ProjectLabRuntime({ + mode: "docker", + runner: mismatchedDockerRunner("runc API_KEY=attestation-secret"), + }), + ).start(project.id, { + threadId: thread.id, + workloadClass: "malware-analysis", + networkProfile: "full", + }), + ), + ), + ).rejects.toThrow(/Expected runtime kata-fc/); + + const artifact = await readOnlyContainmentArtifact(project.id); + expect(artifact).toMatchObject({ + thread_id: thread.id, + metadata: { + source: "containment-observation", + schema: "exploit-hunter.lab-containment-event.v1", + eventClass: "isolation-attestation-failed", + researchRunId: "research-run-1", + }, + content: { + schema: "exploit-hunter.lab-containment-event.v1", + eventClass: "isolation-attestation-failed", + correlation: { + projectId: project.id, + threadId: thread.id, + researchRunId: "research-run-1", + researchTurnLedgerId: "turn-ledger-1", + }, + requested: { + isolation: "microvm", + runtimeClass: "kata-fc", + workloadClass: "malware-analysis", + }, + observed: { + containerId: "durable-container-id", + isolation: "microvm", + workloadClass: "malware-analysis", + }, + cleanup: { + attempted: true, + succeeded: true, + workloadRemovalVerified: true, + workloadRunning: false, + target: "durable-container-id", + }, + }, + }); + expect(JSON.stringify(artifact)).not.toContain("attestation-secret"); + }); + + it("records unresolved cleanup as a distinct containment event", async () => { + const project = await (await getProjectStore()).createProject({ name: "Cleanup failure lab" }); + + await expect( + withDatabase((db) => + new ProjectLabService( + new SqliteProjectLabRepository(db), + new ProjectLabRuntime({ + mode: "docker", + runner: mismatchedDockerRunner("runc", { removalUnverified: true }), + }), + ).start(project.id, { + workloadClass: "malware-analysis", + networkProfile: "full", + }), + ), + ).rejects.toThrow(/Expected runtime kata-fc/); + + const artifact = await readOnlyContainmentArtifact(project.id); + expect(artifact.content).toMatchObject({ + eventClass: "isolation-cleanup-failed", + cleanup: { + attempted: true, + succeeded: false, + workloadRemovalVerified: false, + workloadRunning: true, + target: "durable-container-id", + }, + }); + const reloaded = await withDatabase((db) => + new ProjectLabService( + new SqliteProjectLabRepository(db), + new ProjectLabRuntime({ mode: "dry-run" }), + ).status(project.id), + ); + expect(reloaded).toMatchObject({ + status: "failed", + lab: { + container_id: "durable-container-id", + runtime_metadata: { state: "isolation-cleanup-failed" }, + }, + }); + }); + + it("preserves the containment failure when Artifact persistence also fails", async () => { + const project = await (await getProjectStore()).createProject({ name: "Artifact failure lab" }); + setArtifactService({ + createArtifact: vi.fn(async () => { + throw new Error("Artifact storage unavailable"); + }), + createFinding: vi.fn(), + }); + vi.spyOn(console, "warn").mockImplementation(() => undefined); + + await expect( + withDatabase((db) => + new ProjectLabService( + new SqliteProjectLabRepository(db), + new ProjectLabRuntime({ mode: "docker", runner: mismatchedDockerRunner("runc") }), + ).start(project.id, { + workloadClass: "malware-analysis", + networkProfile: "full", + }), + ), + ).rejects.toThrow(/Expected runtime kata-fc/); + + const reloaded = await withDatabase((db) => + new ProjectLabService( + new SqliteProjectLabRepository(db), + new ProjectLabRuntime({ mode: "dry-run" }), + ).status(project.id), + ); + expect(reloaded).toMatchObject({ + status: "failed", + lab: { + runtime_metadata: { state: "isolation-attestation-failed" }, + }, + }); }); }); +async function readOnlyContainmentArtifact(projectId: string) { + return withDatabase(async (db) => { + const result = await db.query<{ + thread_id: string | null; + metadata: unknown; + inline_content: string; + }>( + `SELECT thread_id, metadata, inline_content + FROM artifacts + WHERE project_id = $1 + ORDER BY created_at DESC + LIMIT 1`, + [projectId], + ); + const row = result.rows[0]; + if (!row) throw new Error("Expected a containment Artifact."); + return { + thread_id: row.thread_id, + metadata: + typeof row.metadata === "string" + ? (JSON.parse(row.metadata) as Record) + : row.metadata, + content: JSON.parse(row.inline_content) as Record, + }; + }); +} + function attestedDockerRunner(): DockerCommandRunner { return { async isAvailable() { @@ -96,3 +280,41 @@ function attestedDockerRunner(): DockerCommandRunner { }, }; } + +function mismatchedDockerRunner( + observedRuntimeClass: string, + options: { removalUnverified?: boolean } = {}, +): DockerCommandRunner { + return { + async isAvailable() { + return true; + }, + async run(command) { + if (command.args[0] === "info") { + return { stdout: JSON.stringify({ "kata-fc": {}, runc: {} }), stderr: "", exitCode: 0 }; + } + if (command.reason.startsWith("Attest the running lab isolation boundary")) { + return { + stdout: [ + JSON.stringify("durable-container-id"), + JSON.stringify(true), + JSON.stringify(observedRuntimeClass), + JSON.stringify({ + "exploit-hunter.isolation": "microvm", + "exploit-hunter.workload-class": "malware-analysis", + }), + ].join("\n"), + stderr: "", + exitCode: 0, + }; + } + if (command.reason.startsWith("Verify from the Docker daemon")) { + if (options.removalUnverified) { + return { stdout: "container still exists", stderr: "", exitCode: 0 }; + } + throw new Error("No such container"); + } + return { stdout: "", stderr: "", exitCode: 0 }; + }, + }; +}