diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index d394835db..917e3c97d 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -33,6 +33,23 @@ permissions: packages: read jobs: + nestedvirt-tools: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-python@v7 + with: + python-version: "3.12" + - name: Check nested KVM smoke syntax + run: | + python3 -m py_compile scripts/tests/nestedvirt-smoke.py + sh -n scripts/tests/nestedvirt-guest.sh + - name: Verify real fixture rebuild + run: | + python3 scripts/tests/nestedvirt-smoke.py check-fixture + python3 -O scripts/tests/nestedvirt-smoke.py check-fixture + agentenv-tests: runs-on: ubuntu-latest steps: diff --git a/config/deps_manifest.toml b/config/deps_manifest.toml index 0a32db491..951b0bb52 100644 --- a/config/deps_manifest.toml +++ b/config/deps_manifest.toml @@ -1,14 +1,14 @@ [firecracker.kvm] -version = "1.15.1-patch-v1" -url = "https://github.com/kvcache-ai/firecracker/releases/download/aenv-deps/firecracker-{version}-{arch}.tgz" +version = "1.15.1-patch-v2" +url = "https://github.com/kvcache-ai/firecracker/releases/download/v{version}/firecracker-{version}-{arch}.tgz" [firecracker.pvm] version = "v1.17.0-next.1" url = "https://github.com/kvcache-ai/firecracker-next/releases/download/{version}/firecracker-next-{version}-{arch}.tgz" [kernel.kvm] -version = "vmlinux-6.1.175" -url = "https://github.com/kvcache-ai/firecracker/releases/download/aenv-deps/{version}-{arch}" +version = "vmlinux-6.18" +url = "https://github.com/kvcache-ai/firecracker/releases/download/v1.15.1-patch-v2/{version}-{arch}" [kernel.pvm] version = "6.12.33-pvm" diff --git a/docs/src/internals/sandbox-testing.md b/docs/src/internals/sandbox-testing.md index 5cab2b4b9..ec962ac5f 100644 --- a/docs/src/internals/sandbox-testing.md +++ b/docs/src/internals/sandbox-testing.md @@ -87,6 +87,9 @@ This document describes the public sandbox API, the global `ConfigManager` under - Each sandbox gets its own temporary work directory. - `FirecrackerSandbox::resume_from_snapshot_config(&FirecrackerSnapshotConfig)` - Creates a new sandbox handle and immediately starts it from a snapshot config. + - Uses the node's currently configured Firecracker, whether taking a prewarmed + process or starting a new one. Cross-version snapshot compatibility is + provided by that Firecracker runtime. - Uses `vm_state_path`, the memory overlay image config, and `base_rootfs_path` from the config. - Returns a running sandbox after Firecracker and envd are ready. @@ -563,6 +566,78 @@ client API and the binary that setup downloads: cargo test -p agentenv --lib ``` +## 4.2) Nested KVM Validation + +`scripts/tests/nestedvirt-smoke.py` builds a test image and exercises template +restoration, fork, and pause/resume with QEMU running inside an x86_64 sandbox. +Run it when changing the nested-KVM runtime, snapshot handling, or guest kernel. + +1. Use an x86_64 host with nested KVM and an outer guest kernel with built-in + KVM. Install Python 3.11+, QEMU x86 system binaries, static BusyBox at + `/bin/busybox`, cpio, and Bash. Provide a bootable bzImage for the inner guest + and a fresh work directory with at least 5 GiB free. +2. Prepare a dedicated AgentENV node using the host setup above. Use separate + `AENV_HOME_PATH`, `AENV_RUNTIME_PATH`, and network pools; select test assets + with `AENV_DEPS_PATH`. Bind the API to loopback and set + `[sandbox_proxy].domains`, for example to `["nested.local"]`. +3. Keep the complete `[firecracker].boot_args` string and append `no-kvmapf`; + the runner checks this argument. Set `[pool.block].enabled = true`. + Configure regctl for the runner's loopback registry with + `--host reg=127.0.0.1:18091,tls=disabled`; match the port to `--port`. +4. Start the node, then use `python3 scripts/tests/nestedvirt-smoke.py run --help` + for SDK installation, parameters, and a complete invocation. Pass the node's + TOML file with `--config`, including its effective boot arguments. +5. Require a successful exit and one result per selected phase and round in + `/evidence/api-results.json` (nine with defaults). Guest commands and + output are in `api-commands.jsonl`. Stop the test node after the run. + +The runner builds one template, then creates a source sandbox and two fork +children per case. The guest payload `nestedvirt-guest.sh` starts and stops QEMU with `accel=kvm` and supplies +an inner-guest counter. + +| Phase | Recovery check | +|-------|----------------| +| `unused` | A fresh inner VM starts after fork and pause/resume. | +| `initialized` | QEMU created with `-S` retains KVM-enabled `prelaunch` state. | +| `running` | The inner counter continues advancing after fork and pause/resume. | + +Each case also starts and stops a fresh inner VM in the source and both children; +the initialized case replaces its stopped inner VM. Existing snapshots retain +their captured kernel and boot arguments, so use fresh templates to test changes +to either. Hosted CI runs syntax and `check-fixture` checks; the lifecycle matrix +runs on nested-KVM hardware. + +## 4.3) Legacy v9 Snapshot Recovery + +`orchestrator_v9_snapshot_upgrade` in `tests/integration/orchestrator.rs` runs the +current AgentENV test binary with old and current Firecracker processes. It +saves v9 state, invalidates the recorded executable path, restores with the +configured runtime, checks envd, and saves again as v1000. + +1. Prepare `v9-node.toml` and `current-node.toml` with their respective runtimes + and matching helpers. Use absolute paths and the same state, storage, + network, guest kernel, and boot arguments in both configurations. +2. Complete the integration host setup above and build the daemon with + `make build-ublk PROFILE=debug`. +3. Run the test explicitly; it is ignored by default because it needs both + runtime versions: + + ```bash + AENV_HOME_PATH=/absolute/path/isolated-test/home \ + AENV_RUNTIME_PATH=/absolute/path/isolated-test/run \ + AENV_DEPS_PATH=/absolute/path/test-deps \ + AENV_UBLK_DAEMON_BINARY_PATH="$PWD/target/debug/uvm-ublk-daemon" \ + AENV_CONFIG_PATH=/absolute/path/current-node.toml \ + AENV_TEST_V9_CONFIG=/absolute/path/v9-node.toml \ + scripts/run-with-capabilities.sh cargo test -p agentenv \ + --test orchestrator_integration orchestrator::orchestrator_v9_snapshot_upgrade \ + -- --exact --ignored --nocapture + ``` + +For service-upgrade validation, create templates and paused records with the old +AgentENV binary, stop it, start the new binary against the same state, and verify +restore, fork, and another pause/resume cycle. + ## 5) Running Custom Scripts ### Run commands via envd diff --git a/scripts/tests/nestedvirt-guest.sh b/scripts/tests/nestedvirt-guest.sh new file mode 100755 index 000000000..7d72ee7c8 --- /dev/null +++ b/scripts/tests/nestedvirt-guest.sh @@ -0,0 +1,45 @@ +#!/bin/sh +# Copyright 2026 AgentENV contributors. +# SPDX-License-Identifier: Apache-2.0 +set -eu +mode=${1:-fixture-init} +if [ "$#" -gt 0 ]; then shift; fi +case "$mode" in + fixture-init) + while true; do /bin/busybox sleep 3600; done + ;; + inner-init) + /bin/busybox mount -t proc proc /proc + /bin/busybox mount -t sysfs sysfs /sys + n=0 + while true; do + echo "INNER_TICK=$n" + n=$((n+1)) + /bin/busybox sleep 1 + done + ;; + start-inner) + /bin/busybox ip link set lo up + exec /usr/bin/qemu-system-x86_64 -machine microvm,accel=kvm -cpu host \ + -m 128 -smp 1 -nodefaults -no-reboot -nographic -serial stdio \ + -qmp tcp:127.0.0.1:4444,server=on,wait=off \ + -kernel /inner-bzImage -initrd /inner.cpio \ + -append 'console=ttyS0 panic=1 rdinit=/init' "$@" + ;; + stop-inner) + pid=$(/bin/busybox cat "${1:?PID_FILE}") + test "$(/bin/busybox readlink /proc/"$pid"/exe)" = /usr/bin/qemu-system-x86_64 + kill -TERM "$pid" + attempt=0 + while kill -0 "$pid" 2>/dev/null; do + processes=$(/bin/busybox ps -o pid,stat) + state=$(printf '%s\n' "$processes" | /bin/busybox awk -v target="$pid" '$1 == target {print $2}') + case "$state" in Z*|"") break ;; esac + attempt=$((attempt + 1)) + test "$attempt" -lt 100 + /bin/busybox sleep 0.1 + done + echo "QEMU_STOPPED=$pid" + ;; + *) echo "Unknown guest operation: $mode" >&2; exit 2 ;; +esac diff --git a/scripts/tests/nestedvirt-smoke.py b/scripts/tests/nestedvirt-smoke.py new file mode 100644 index 000000000..a731185eb --- /dev/null +++ b/scripts/tests/nestedvirt-smoke.py @@ -0,0 +1,502 @@ +# Copyright 2026 AgentENV contributors. +# SPDX-License-Identifier: Apache-2.0 +"""Prepare an OCI fixture and test nested KVM through a running AgentENV node.""" +import argparse +import contextlib +import hashlib +import http.client +import http.server +import itertools +import json +import os +import re +import shutil +import subprocess +import sys +import tarfile +import tempfile +import threading +import time +import tomllib +from pathlib import Path +from urllib.parse import urlsplit + +GUEST_SCRIPT = Path(__file__).with_name("nestedvirt-guest.sh") + + +def copy_binary(path, root): + """Copy a host executable and its shared libraries into the fixture.""" + result = subprocess.run(["ldd", path], text=True, capture_output=True, check=True) + for source in [path] + [word for word in result.stdout.split() if word.startswith("/")]: + target = root / source.lstrip("/") + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source, target) + + +def prepare_fixture(work, inner_kernel): + """Build fresh outer and inner roots inside this run's temporary directory.""" + root, inner = work / "root", work / "inner" + for base in (root, inner): + for name in ("bin", "dev", "proc", "sys", "run", "root"): + (base / name).mkdir(parents=True) + shutil.copy2("/bin/busybox", base / "bin/busybox") + for name in subprocess.check_output(["/bin/busybox", "--list"], text=True).splitlines(): + if name != "busybox": + (base / "bin" / name).symlink_to("busybox") + shutil.copy2(GUEST_SCRIPT, base / "nestedvirt-guest") + (base / "nestedvirt-guest").chmod(0o755) + (root / "init").symlink_to("nestedvirt-guest") + (inner / "init").write_text("#!/bin/sh\nexec /nestedvirt-guest inner-init\n") + (inner / "init").chmod(0o755) + with (root / "inner.cpio").open("wb") as output: + subprocess.run( + ["bash", "-o", "pipefail", "-c", "find . -print0 | cpio --null -o --format=newc"], + cwd=inner, stdout=output, check=True, + ) + shutil.copy2(inner_kernel, root / "inner-bzImage") + copy_binary("/usr/bin/qemu-system-x86_64", root) + copy_binary("/bin/bash", root) + for source in (Path("/usr/share/qemu"), Path("/usr/share/seabios")): + if source.exists(): + shutil.copytree(source, root / source.relative_to("/")) + return root + + +@contextlib.contextmanager +def serve_fixture(root, port): + """Serve the current root as an OCI image for the duration of the API test.""" + archive = root.parent / "fixture.tar" + with tarfile.open(archive, "w") as output: + output.add(root, arcname=".") + layer = archive.read_bytes() + digest = lambda data: "sha256:" + hashlib.sha256(data).hexdigest() + ld = digest(layer) + config = json.dumps({ + "architecture": "amd64", "os": "linux", "config": {}, + "rootfs": {"type": "layers", "diff_ids": [ld]}, + }).encode() + cd = digest(config) + manifest_type = "application/vnd.oci.image.manifest.v1+json" + manifest = json.dumps({ + "schemaVersion": 2, "mediaType": manifest_type, + "config": {"mediaType": "application/vnd.oci.image.config.v1+json", "digest": cd, "size": len(config)}, + "layers": [{"mediaType": "application/vnd.oci.image.layer.v1.tar", "digest": ld, "size": len(layer)}], + }).encode() + blobs = {ld: layer, cd: config} + + class Handler(http.server.BaseHTTPRequestHandler): + def do_HEAD(self): + self.respond(False) + + def do_GET(self): + self.respond(True) + + def respond(self, include_body): + if self.path == "/v2/": + body, kind = b"{}", "application/json" + elif self.path.startswith("/v2/nested/manifests/"): + body, kind = manifest, manifest_type + elif self.path.startswith("/v2/nested/blobs/") and self.path.rsplit("/", 1)[-1] in blobs: + body, kind = blobs[self.path.rsplit("/", 1)[-1]], "application/octet-stream" + else: + self.send_error(404) + return + self.send_response(200) + self.send_header("Content-Type", kind) + self.send_header("Content-Length", str(len(body))) + self.send_header("Docker-Content-Digest", digest(body)) + self.end_headers() + if include_body: + self.wfile.write(body) + + with http.server.ThreadingHTTPServer(("127.0.0.1", port), Handler) as server: + thread = threading.Thread(target=server.serve_forever) + thread.start() + try: + yield f"127.0.0.1:{server.server_port}/nested:latest" + finally: + server.shutdown() + thread.join() + + +def check_fixture(): + """Check real OCI rebuilding and registry cleanup without KVM or the SDK.""" + with tempfile.TemporaryDirectory(prefix="nestedvirt-fixture-check-") as directory: + root = Path(directory) / "root" + root.mkdir() + (root / "retained").write_text("current\n") + (root / "removed").write_text("old\n") + shutil.copy2(GUEST_SCRIPT, root / "nestedvirt-guest") + (root / "init").symlink_to("nestedvirt-guest") + for iteration in range(2): + if iteration: + (root / "removed").unlink() + (root / "retained").write_text("updated\n") + with serve_fixture(root, 0) as image: + connection = http.client.HTTPConnection(image.split("/")[0], timeout=5) + try: + connection.request("GET", "/v2/nested/manifests/latest") + response = connection.getresponse() + if response.status != 200: + raise RuntimeError(f"Manifest returned HTTP {response.status}") + manifest = json.loads(response.read()) + finally: + connection.close() + archive_path = root.parent / "fixture.tar" + if manifest["layers"][0]["size"] != archive_path.stat().st_size: + raise RuntimeError("Fixture layer size mismatch") + with tarfile.open(archive_path) as archive: + if ("./removed" in archive.getnames()) != (iteration == 0): + raise RuntimeError("Fixture retained a removed file") + expected = b"updated\n" if iteration else b"current\n" + if archive.extractfile("./retained").read() != expected: + raise RuntimeError("Fixture content mismatch") + if archive.extractfile("./init").read() != GUEST_SCRIPT.read_bytes(): + raise RuntimeError("Fixture init mismatch") + print("PASS fixture rebuild and registry cleanup") + + +def run_api(args): + endpoint = urlsplit(args.endpoint) + if endpoint.scheme != "http": + raise ValueError("This local hardware smoke expects an HTTP endpoint") + expected_boot_args = tomllib.loads(args.config.read_text())["firecracker"][ + "boot_args" + ].split() + if "no-kvmapf" not in expected_boot_args: + raise ValueError( + "The selected node config must append no-kvmapf to its complete firecracker.boot_args" + ) + + from e2b import Sandbox + + key = args.api_key_file.read_text().strip() + args.evidence.mkdir(parents=True, exist_ok=True) + results = [] + clients = {} + (args.evidence / "api-results.json").write_text("[]\n") + + + def api(method, path, body=None): + connection = http.client.HTTPConnection( + endpoint.hostname, endpoint.port, timeout=180 + ) + connection.request( + method, + path, + None if body is None else json.dumps(body), + {"Content-Type": "application/json", "X-API-Key": key}, + ) + response = connection.getresponse() + data = response.read() + connection.close() + if not 200 <= response.status < 300: + raise RuntimeError(f"{method} {path}: {response.status} {data.decode()}") + return json.loads(data) if data else None + + + def command(sandbox, shell): + sandbox_id = sandbox["sandboxID"] + if sandbox_id not in clients: + clients[sandbox_id] = Sandbox.connect( + sandbox_id, + timeout=600, + api_key=key, + api_url=args.endpoint, + sandbox_url=args.endpoint, + http_version="1.1", + retries=0, + ) + with (args.evidence / "api-commands.jsonl").open("a") as log: + log.write( + json.dumps( + { + "event": "start", + "sandbox": sandbox_id, + "command": shell, + "phase": phase, + "iteration": iteration, + "started_at": time.time(), + } + ) + + "\n" + ) + result = clients[sandbox_id].commands.run( + shell, user="root", timeout=90, request_timeout=100 + ) + with (args.evidence / "api-commands.jsonl").open("a") as log: + log.write( + json.dumps( + { + "event": "complete", + "sandbox": sandbox_id, + "command": shell, + "phase": phase, + "iteration": iteration, + "exit_code": result.exit_code, + "stdout": result.stdout, + "stderr": result.stderr, + } + ) + + "\n" + ) + if result.exit_code != 0: + raise RuntimeError(f"Guest command failed with exit code {result.exit_code}") + return result.stdout + + + def ticks(sandbox): + text = command(sandbox, "sleep 3; cat /run/nestedvirt-probe/inner.log") + values = re.findall(r"INNER_TICK=(\d+)", text) + if not values: + raise RuntimeError(f"Inner guest did not run: {text[-4000:]}") + return int(values[-1]) + + + def advancing_ticks(sandbox): + first, second = ticks(sandbox), ticks(sandbox) + if second <= first: + raise RuntimeError(f"Inner guest stopped advancing: {first} -> {second}") + return second + + + def check(sandbox): + text = command( + sandbox, + "test -c /dev/kvm && test $(grep -c '^processor' /proc/cpuinfo) -eq 2 && uname -r && cat /proc/cmdline", + ) + if args.kernel_release not in text.splitlines(): + raise RuntimeError(f"Wrong guest kernel: {text}") + missing = set(expected_boot_args) - set(text.split()) + if missing: + raise RuntimeError(f"Guest boot arguments omit {sorted(missing)}") + + + def start_inner(sandbox, stopped=False): + command( + sandbox, + f"mkdir -p /run/nestedvirt-probe; /nestedvirt-guest start-inner{' -S' if stopped else ''} >/run/nestedvirt-probe/inner.log 2>&1 & echo $! >/run/nestedvirt-probe/inner.pid", + ) + if stopped: + initialized_inner(sandbox) + if not stopped: + deadline = time.monotonic() + 90 + while time.monotonic() < deadline: + text = command( + sandbox, "sleep 1; tail -n 30 /run/nestedvirt-probe/inner.log" + ) + values = re.findall(r"INNER_TICK=(\d+)", text) + if values: + return advancing_ticks(sandbox) + raise TimeoutError(f"Inner guest did not execute: {text}") + + + def initialized_inner(sandbox): + deadline = time.monotonic() + 30 + while time.monotonic() < deadline: + text = command( + sandbox, + "printf '%s\\n' '{\"execute\":\"qmp_capabilities\"}' " + "'{\"execute\":\"query-kvm\",\"id\":\"kvm\"}' " + "'{\"execute\":\"query-status\",\"id\":\"status\"}' " + "| /bin/busybox nc -w 1 127.0.0.1 4444; true", + ) + replies = {} + for line in text.splitlines(): + if line.startswith('{"'): + reply = json.loads(line) + if "id" in reply: + replies[reply["id"]] = reply.get("return", {}) + if ( + replies.get("kvm", {}).get("enabled") is True + and replies.get("kvm", {}).get("present") is True + and replies.get("status", {}).get("status") == "prelaunch" + and replies.get("status", {}).get("running") is False + ): + return + raise TimeoutError(f"Inner KVM did not initialize in stopped state: {text}") + + + @contextlib.contextmanager + def cleanup_resources(): + paths = [] + try: + yield paths + finally: + original = sys.exc_info()[1] + errors = [] + for path in reversed(paths): + try: + api("DELETE", path) + except Exception as error: + errors.append(error) + finally: + if path.startswith("/sandboxes/"): + clients.pop(path.rsplit("/", 1)[-1], None) + if errors: + # Preserve the original failure and all cleanup errors; fail a successful + # run if cleanup fails. + failure = ExceptionGroup("Resource cleanup failed", errors) + if original is not None: + original.add_note(str(failure) + ": " + repr(errors)) + else: + raise failure + + + def replace_inner(sandbox, phase): + if phase != "unused": + command(sandbox, "/nestedvirt-guest stop-inner /run/nestedvirt-probe/inner.pid") + start_inner(sandbox) + command(sandbox, "/nestedvirt-guest stop-inner /run/nestedvirt-probe/inner.pid") + + + with cleanup_resources() as templates: + template = api( + "POST", + "/v3/templates", + {"name": f"nestedvirt-smoke-{time.time_ns()}", "cpuCount": 2, "memoryMB": 512}, + ) + tid, bid = template["templateID"], template["buildID"] + templates.append(f"/templates/{tid}") + api("POST", f"/v2/templates/{tid}/builds/{bid}", {"fromImage": args.image}) + deadline = time.monotonic() + 300 + while time.monotonic() < deadline: + status = api("GET", f"/templates/{tid}/builds/{bid}/status") + if status["status"] == "ready": + break + if status["status"] == "error": + raise RuntimeError(f"Template build failed: {status}") + time.sleep(1) + else: + raise TimeoutError("Template build did not complete") + print(json.dumps({"template": tid, "build": "PASS"}), flush=True) + + for phase, iteration in itertools.product( + [args.phase] if args.phase else ("unused", "initialized", "running"), + range(args.rounds), + ): + sandboxes = [] + with cleanup_resources() as sandbox_paths: + source = api("POST", "/v2/sandboxes", {"templateID": tid, "timeout": 600}) + sandboxes.append(source) + sandbox_paths.append(f"/sandboxes/{source['sandboxID']}") + check(source) + before = None + if phase != "unused": + before = start_inner(source, phase == "initialized") + forks = api( + "POST", + f"/sandboxes/{source['sandboxID']}/fork", + {"count": 2, "timeout": 600}, + ) + sandboxes.extend( + outcome["sandbox"] for outcome in forks if outcome.get("sandbox") + ) + sandbox_paths.extend( + f"/sandboxes/{sandbox['sandboxID']}" for sandbox in sandboxes[1:] + ) + if len(sandboxes) != 3: + raise RuntimeError( + f"Expected two fork children: {[o.get('error') for o in forks]}" + ) + for sandbox in sandboxes: + check(sandbox) + if phase == "initialized": + initialized_inner(sandbox) + if phase == "running" and advancing_ticks(sandbox) <= before: + raise RuntimeError("Inner guest failed to continue across fork") + before_pause = ticks(source) if phase == "running" else None + api("POST", f"/sandboxes/{source['sandboxID']}/pause") + source = api( + "POST", f"/v2/sandboxes/{source['sandboxID']}/connect", {"timeout": 600} + ) + clients.pop(source["sandboxID"], None) + sandboxes[0] = source + check(source) + if phase == "initialized": + initialized_inner(source) + if phase == "running" and advancing_ticks(source) <= before_pause: + raise RuntimeError("Inner guest failed to continue after pause/resume") + for sandbox in sandboxes: + replace_inner(sandbox, phase) + results.append( + { + "phase": phase, + "iteration": iteration, + "template": tid, + "forks": 2, + "pause_resume": True, + "new_inner_after_restore": True, + "result": "PASS", + } + ) + print(json.dumps(results[-1]), flush=True) + + # Publish the complete results only after the template and every sandbox are cleaned up. + (args.evidence / "api-results.json").write_text(json.dumps(results, indent=2) + "\n") + + +def main(): + parser = argparse.ArgumentParser( + description=__doc__, + epilog="Use 'run --help' for installation and invocation examples.", + ) + commands = parser.add_subparsers(dest="command", required=True) + commands.add_parser("check-fixture", help="Check OCI rebuilding without KVM") + run = commands.add_parser( + "run", + help="Prepare the fixture and run the API matrix", + description="Test nested KVM through a running AgentENV node on this host.", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=r"""Setup and example (run from the repository root): + Prepare the node as described in docs/src/internals/sandbox-testing.md, + section 4.2. In a Python virtual environment, install the SDK: + python3 -m pip install 'e2b==2.55.1' + + python3 scripts/tests/nestedvirt-smoke.py run \ + http://127.0.0.1:18090 /absolute/path/test-home/secrets/api-key \ + /absolute/path/nestedvirt-work --inner-kernel /absolute/path/bzImage \ + --config /absolute/path/node.toml --kernel-release EXPECTED_UNAME_R --rounds 3 + + Replace EXPECTED_UNAME_R with the outer guest's exact uname -r. + Results and guest-command logs are written to WORK/evidence/. + The runner removes its image fixture, registry, sandboxes and template. + Stop the dedicated AgentENV node after the test. +""", + ) + run.add_argument("endpoint", help="HTTP API URL of the running local test node") + run.add_argument("api_key_file", type=Path, help="File containing the node's API key") + run.add_argument("work", type=Path, help="Test work directory; results go in evidence/") + run.add_argument("--inner-kernel", type=Path, required=True, + help="Path to the inner QEMU guest's bootable bzImage") + run.add_argument("--kernel-release", required=True, help="Expected outer guest uname -r") + run.add_argument("--port", type=int, default=18091, + help="Loopback registry port; match the node's regctl config (default: 18091)") + run.add_argument("--rounds", type=int, default=3, help="Rounds per phase (default: 3)") + run.add_argument("--phase", choices=("unused", "initialized", "running"), + help="Select one inner-VM phase (default: all three)") + run.add_argument( + "--config", type=Path, + default=Path(os.environ.get("AENV_CONFIG_PATH") + or Path(__file__).resolve().parents[2] / "config/default.toml"), + help="Node TOML with effective boot_args (default: AENV_CONFIG_PATH or config/default.toml)", + ) + args = parser.parse_args() + if args.command == "check-fixture": + check_fixture() + return + if args.rounds < 1: + parser.error("--rounds must be positive") + args.work = args.work.resolve() + args.inner_kernel = args.inner_kernel.resolve(strict=True) + args.evidence = args.work / "evidence" + args.evidence.mkdir(parents=True, exist_ok=True) + (args.evidence / "api-results.json").write_text("[]\n") + with tempfile.TemporaryDirectory(prefix="fixture-", dir=args.work) as directory: + root = prepare_fixture(Path(directory), args.inner_kernel) + with serve_fixture(root, args.port) as args.image: + run_api(args) + + +if __name__ == "__main__": + main() diff --git a/src/sandbox/firecracker/sandbox.rs b/src/sandbox/firecracker/sandbox.rs index e36bc0283..b52a686c0 100644 --- a/src/sandbox/firecracker/sandbox.rs +++ b/src/sandbox/firecracker/sandbox.rs @@ -723,6 +723,10 @@ impl FirecrackerSandbox { snapshot.common.extra_drives.len() - snapshot.common.physical_extra_drive_count, snapshot.common.volume_drive_slots ); + // Restore with the node's current runtime, matching the prewarmed pool. + // Captured guest assets remain unchanged. + snapshot.common.firecracker_binary = + ConfigManager::global_config().resolved_firecracker_binary_path(); // Runtime identity and auth override their values in the source snapshot. snapshot.common.envd_access_token = envd_access_token; let FirecrackerCommonConfig { @@ -2978,6 +2982,43 @@ mod tests { ); } + #[test] + fn snapshot_restore_selects_current_runtime_and_preserves_guest_assets() -> Result<()> { + let mut common = fresh_config().common; + common.firecracker_binary = "/missing/old/firecracker".into(); + common.tools_drive_version = "captured-tools".into(); + common.envd_version = "captured-envd".into(); + let rootfs = common.rootfs_image_config.clone(); + let drives = serde_json::to_value(&common.extra_drives)?; + let snapshot = FirecrackerSnapshotConfig { + common, + vm_state_path: "snapshot/vm_state.bin".into(), + mem_overlaybd_config: OverlaybdConfig { + image_config_path: "snapshot/mem_image.json".into(), + read_only: true, + runtime_upper_mode: overlaybd::config::UpperMode::LogStructured, + }, + mem_virtual_size: 4096, + managed_snapshot_root: None, + pack_recording: false, + memory_startup_pack: None, + }; + let sandbox = FirecrackerSandbox::from_snapshot_config(&snapshot)?; + let restored = sandbox.launch.common(); + assert_eq!( + restored.firecracker_binary, + ConfigManager::global_config().resolved_firecracker_binary_path() + ); + assert_eq!(restored.tools_drive_version, "captured-tools"); + assert_eq!(restored.envd_version, "captured-envd"); + assert_eq!( + serde_json::to_value(&restored.rootfs_image_config)?, + serde_json::to_value(rootfs)? + ); + assert_eq!(serde_json::to_value(&restored.extra_drives)?, drives); + Ok(()) + } + #[test] fn snapshot_config_runtime_identity_replaces_source_auth() -> Result<()> { let source_id = SandboxId::new(); diff --git a/tests/integration/orchestrator.rs b/tests/integration/orchestrator.rs index e9faf74a3..3fc796453 100644 --- a/tests/integration/orchestrator.rs +++ b/tests/integration/orchestrator.rs @@ -13,13 +13,16 @@ use agentenv::snapshot::{ use anyhow::Result; use envd::process::{ListRequest, ProcessClient}; -use std::path::PathBuf; +use sha2::{Digest, Sha256}; +use std::path::{Path, PathBuf}; use tempfile::tempdir; use tokio::time::{timeout, Duration}; use tonic::Request; use uuid::Uuid; const TEST_TIMEOUT: Duration = Duration::from_secs(120); +// These processes share the configured host network rules. +static ORCHESTRATOR_TEST_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); async fn envd_process_list_status( target: &agentenv::orchestrator::ProxyTarget, @@ -49,121 +52,266 @@ fn host_file_persister(root: PathBuf) -> FileBackedSandboxPersister { FileBackedSandboxPersister::new(root, ConfigManager::global_config().virtualization_mode) } +// Only this opt-in test needs two installed Firecracker versions. It reuses +// the real pause/restart/resume lifecycle rather than constructing VM state. +#[cfg(target_arch = "x86_64")] +#[tokio::test] +#[ignore = "requires isolated KVM host, AENV_TEST_V9_CONFIG and current v1000 runtime"] +async fn orchestrator_v9_snapshot_upgrade() -> Result<()> { + let _guard = ORCHESTRATOR_TEST_LOCK.lock().await; + let old_config = std::fs::canonicalize(std::env::var("AENV_TEST_V9_CONFIG")?)?; + let current = ConfigManager::new()?; + let current_config = std::fs::canonicalize(current.config_path().expect("config path"))?; + let old = ConfigManager::new_from_path(&old_config)?; + assert_eq!( + old.config().virtualization_mode, + agentenv::virtualization::VirtualizationMode::Kvm + ); + assert_eq!( + current.config().virtualization_mode, + agentenv::virtualization::VirtualizationMode::Kvm + ); + let current_binary = current.config().resolved_firecracker_binary_path(); + assert_ne!( + old.config().resolved_firecracker_binary_path(), + current_binary + ); + let root = tempdir()?; + for (phase, config) in [("pause", old_config), ("resume", current_config)] { + let mut child = tokio::process::Command::new(std::env::current_exe()?); + child + .args([ + "--exact", + "orchestrator::orchestrator_lifecycle", + "--nocapture", + ]) + .env("AENV_CONFIG_PATH", config) + .env("AENV_ORCHESTRATOR_RESTART_TEST_ROOT", root.path()) + .env("AENV_ORCHESTRATOR_RESTART_TEST_PHASE", phase) + .env("AENV_ORCHESTRATOR_UPGRADE_TEST", "1") + .kill_on_drop(true); + let status = timeout(TEST_TIMEOUT, child.status()).await??; + anyhow::ensure!(status.success(), "{phase} upgrade process failed: {status}"); + if phase == "pause" { + // Reading the witness also proves the child executed the pause body. + let (id, _): (agentenv::types::SandboxId, String) = + serde_json::from_slice(&std::fs::read(root.path().join("restart-witness.json"))?)?; + let db = rocksdb::DB::open_default(root.path().join("paused-sandboxes/records.db"))?; + let key = id.to_string(); + let mut record: serde_json::Value = serde_json::from_slice( + &db.get(&key)? + .ok_or_else(|| anyhow::anyhow!("missing paused record"))?, + )?; + assert_snapshot_format(¤t_binary, &record["state"], "v9.0.0").await?; + let missing = root.path().join("removed-old-runtime/firecracker"); + assert!(!missing.exists()); + let binary = record + .pointer_mut("/state/common/firecracker_binary") + .ok_or_else(|| anyhow::anyhow!("missing captured binary path"))?; + *binary = serde_json::to_value(missing)?; + db.put(&key, serde_json::to_vec(&record)?)?; + db.flush_wal(true)?; + // Drop the DB before the resume process opens the paused store. + } else { + assert_eq!(std::fs::read(root.path().join("resume-complete"))?, b"ok"); + } + } + Ok(()) +} + +async fn assert_snapshot_format( + binary: &Path, + state: &serde_json::Value, + expected: &str, +) -> Result<()> { + let path = state["vm_state_path"] + .as_str() + .ok_or_else(|| anyhow::anyhow!("missing VM state path"))?; + let mut command = tokio::process::Command::new(binary); + command + .args(["--describe-snapshot", path]) + .kill_on_drop(true); + let output = timeout(TEST_TIMEOUT, command.output()).await??; + anyhow::ensure!( + output.status.success(), + "snapshot inspection failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!( + String::from_utf8(output.stdout)?.lines().next(), + Some(expected) + ); + Ok(()) +} + #[tokio::test] async fn orchestrator_lifecycle() -> Result<()> { + let _guard = ORCHESTRATOR_TEST_LOCK.lock().await; + const ROOT_ENV: &str = "AENV_ORCHESTRATOR_RESTART_TEST_ROOT"; + const PHASE_ENV: &str = "AENV_ORCHESTRATOR_RESTART_TEST_PHASE"; + + let Some(root) = std::env::var_os(ROOT_ENV) else { + // Shutdown is terminal for process-global managers. Restart the test + // process, retaining the same paused records and every lifecycle check. + let root = tempdir()?; + for phase in ["pause", "resume"] { + let mut child = tokio::process::Command::new(std::env::current_exe()?); + child + .args([ + "--exact", + "orchestrator::orchestrator_lifecycle", + "--nocapture", + ]) + .env(ROOT_ENV, root.path()) + .env(PHASE_ENV, phase) + .kill_on_drop(true); + let status = timeout(TEST_TIMEOUT, child.status()) + .await + .map_err(|_| anyhow::anyhow!("{phase} test process timed out"))??; + anyhow::ensure!(status.success(), "{phase} test process failed: {status}"); + let completed = match phase { + "pause" => "restart-witness.json", + _ => "resume-complete", + }; + anyhow::ensure!( + root.path().join(completed).is_file(), + "{phase} child exited without completing the lifecycle test" + ); + } + return Ok(()); + }; + common::setup().await; timeout(TEST_TIMEOUT, async { - let root = tempdir()?; - let (builder, snapshot_manager, _) = common::snapshot_test_parts(root.path()); - let alias = format!("orchestrator-test-shared-{}", Uuid::now_v7()); + let root = PathBuf::from(root); + let phase = std::env::var(PHASE_ENV)?; + let upgrade = std::env::var_os("AENV_ORCHESTRATOR_UPGRADE_TEST").is_some(); + if phase == "pause" { + let (builder, snapshot_manager, _) = common::snapshot_test_parts(&root); + let alias = format!("orchestrator-test-shared-{}", Uuid::now_v7()); - let stored = builder - .build_and_publish( - &snapshot_manager, - common::default_rootfs_template_build_spec() - .alias(alias) - .run("mkdir -p /workspace"), - ) - .await?; - let runnable = snapshot_manager.resolve_runnable(stored).await?; + let stored = builder + .build_and_publish( + &snapshot_manager, + common::default_rootfs_template_build_spec() + .alias(alias) + .run("mkdir -p /workspace"), + ) + .await?; + let runnable = snapshot_manager.resolve_runnable(stored).await?; - let store = InMemoryMetadataStore::new(); - let factory = FirecrackerSandboxFactory::new(); - let paused_store = root.path().join("paused-sandboxes"); - let persister = host_file_persister(paused_store.clone()); - let orchestrator = Orchestrator::new(store, factory, persister).await?; - let case_id = Uuid::now_v7().to_string(); + let store = InMemoryMetadataStore::new(); + let factory = FirecrackerSandboxFactory::new(); + let paused_store = &root.join("paused-sandboxes"); + let persister = host_file_persister(paused_store.clone()); + let orchestrator = Orchestrator::new(store, factory, persister).await?; + let case_id = Uuid::now_v7().to_string(); - let request = CreateSandboxRequest { - source: SandboxLaunchSource::Snapshot(Box::new(runnable)), - extra_drives: Vec::new(), - extra_drives_in_snapshot: false, - timeout: Some(Duration::from_secs(30)), - timeout_action: SandboxTimeoutAction::Pause, - user_metadata: Some( - [ - ("team".to_string(), "alpha".to_string()), - ("case_id".to_string(), case_id.clone()), - ] - .iter() - .cloned() - .collect(), - ), - env_vars: None, - network_policy: SandboxNetworkPolicy::default(), - auto_resume: false, - custom_extension_params: None, - secure: true, - volume_mounts: std::collections::HashMap::new(), - }; + let request = CreateSandboxRequest { + source: SandboxLaunchSource::Snapshot(Box::new(runnable)), + extra_drives: Vec::new(), + extra_drives_in_snapshot: false, + timeout: Some(Duration::from_secs(30)), + timeout_action: SandboxTimeoutAction::Pause, + user_metadata: Some( + [ + ("team".to_string(), "alpha".to_string()), + ("case_id".to_string(), case_id.clone()), + ] + .iter() + .cloned() + .collect(), + ), + env_vars: None, + network_policy: SandboxNetworkPolicy::default(), + auto_resume: false, + custom_extension_params: None, + secure: true, + volume_mounts: std::collections::HashMap::new(), + }; - let created = orchestrator.create_sandbox(request).await?; - assert_eq!(created.state, SandboxState::Running); - let access_token = orchestrator - .get_envd_access_token(&created) - .expect("secure sandbox access token"); + let created = orchestrator.create_sandbox(request).await?; + assert_eq!(created.state, SandboxState::Running); + let access_token = orchestrator + .get_envd_access_token(&created) + .expect("secure sandbox access token"); - let sandbox_id = created.id; - let lookup = orchestrator.proxy_lookup_for(&sandbox_id).await?; - assert!( - matches!(lookup, ProxyLookupResult::Ready(_)), - "expected proxy lookup Ready for sandbox {sandbox_id}, got {lookup:?}" - ); - let ProxyLookupResult::Ready(target) = lookup else { - unreachable!(); - }; - assert_eq!( - envd_process_list_status(&target, None).await?, - envd::reqwest::StatusCode::UNAUTHORIZED - ); - assert_eq!( - envd_process_list_status(&target, Some("wrong-token")).await?, - envd::reqwest::StatusCode::UNAUTHORIZED - ); - assert_envd_process_list_succeeds(&target, access_token.expose()).await?; + let sandbox_id = created.id; + let lookup = orchestrator.proxy_lookup_for(&sandbox_id).await?; + assert!( + matches!(lookup, ProxyLookupResult::Ready(_)), + "expected proxy lookup Ready for sandbox {sandbox_id}, got {lookup:?}" + ); + let ProxyLookupResult::Ready(target) = lookup else { + unreachable!(); + }; + assert_envd_process_list_succeeds(&target, access_token.expose()).await?; + if !upgrade { + assert_eq!( + envd_process_list_status(&target, None).await?, + envd::reqwest::StatusCode::UNAUTHORIZED + ); + assert_eq!( + envd_process_list_status(&target, Some("wrong-token")).await?, + envd::reqwest::StatusCode::UNAUTHORIZED + ); - let child = orchestrator - .fork_sandbox(sandbox_id, 1, NewTimeout::UseExisting) - .await? - .pop() - .expect("one fork result")?; - let child_token = orchestrator - .get_envd_access_token(&child) - .expect("secure fork access token"); - assert_ne!(child_token, access_token); - let ProxyLookupResult::Ready(child_target) = - orchestrator.proxy_lookup_for(&child.id).await? - else { - panic!("secure fork should have a proxy route"); - }; - assert_eq!( - envd_process_list_status(&child_target, Some(access_token.expose())).await?, - envd::reqwest::StatusCode::UNAUTHORIZED - ); - assert_envd_process_list_succeeds(&child_target, child_token.expose()).await?; - orchestrator.delete_sandbox(child.id).await?; + let child = orchestrator + .fork_sandbox(sandbox_id, 1, NewTimeout::UseExisting) + .await? + .pop() + .expect("one fork result")?; + let child_token = orchestrator + .get_envd_access_token(&child) + .expect("secure fork access token"); + assert_ne!(child_token, access_token); + let ProxyLookupResult::Ready(child_target) = + orchestrator.proxy_lookup_for(&child.id).await? + else { + panic!("secure fork should have a proxy route"); + }; + assert_eq!( + envd_process_list_status(&child_target, Some(access_token.expose())).await?, + envd::reqwest::StatusCode::UNAUTHORIZED + ); + assert_envd_process_list_succeeds(&child_target, child_token.expose()).await?; + orchestrator.delete_sandbox(child.id).await?; + } - let fetched = orchestrator - .get_sandbox(&sandbox_id) - .await? - .expect("sandbox metadata should exist after create"); - assert_eq!(fetched.id, sandbox_id); - assert_eq!(fetched.state, SandboxState::Running); + let fetched = orchestrator + .get_sandbox(&sandbox_id) + .await? + .expect("sandbox metadata should exist after create"); + assert_eq!(fetched.id, sandbox_id); + assert_eq!(fetched.state, SandboxState::Running); - orchestrator.pause_sandbox(sandbox_id).await?; - let paused = orchestrator - .get_sandbox(&sandbox_id) - .await? - .expect("sandbox metadata should exist after pause"); - assert_eq!(paused.state, SandboxState::Paused); - assert_eq!( - orchestrator.proxy_lookup_for(&sandbox_id).await?, - ProxyLookupResult::Paused { auto_resume: false } - ); + orchestrator.pause_sandbox(sandbox_id).await?; + let paused = orchestrator + .get_sandbox(&sandbox_id) + .await? + .expect("sandbox metadata should exist after pause"); + assert_eq!(paused.state, SandboxState::Paused); + assert_eq!( + orchestrator.proxy_lookup_for(&sandbox_id).await?, + ProxyLookupResult::Paused { auto_resume: false } + ); - orchestrator.shutdown().await?; - drop(orchestrator); + orchestrator.shutdown().await?; + drop(orchestrator); + std::fs::write( + root.join("restart-witness.json"), + serde_json::to_vec(&( + sandbox_id, + hex::encode(Sha256::digest(access_token.expose().as_bytes())), + ))?, + )?; + return Ok(()); + } + anyhow::ensure!(phase == "resume", "unknown lifecycle test phase: {phase}"); + let (sandbox_id, access_token_digest): (agentenv::types::SandboxId, String) = + serde_json::from_slice(&std::fs::read(root.join("restart-witness.json"))?)?; + let paused_store = root.join("paused-sandboxes"); let restarted = Orchestrator::new( InMemoryMetadataStore::new(), FirecrackerSandboxFactory::new(), @@ -193,25 +341,50 @@ async fn orchestrator_lifecycle() -> Result<()> { let resumed_token = restarted .get_envd_access_token(&resumed) .expect("resumed secure sandbox access token"); - assert_eq!(resumed_token, access_token); + assert_eq!( + hex::encode(Sha256::digest(resumed_token.expose().as_bytes())), + access_token_digest, + "sandbox access token changed after restart" + ); let ProxyLookupResult::Ready(target) = lookup else { unreachable!(); }; - assert_eq!( - envd_process_list_status(&target, None).await?, - envd::reqwest::StatusCode::UNAUTHORIZED - ); - assert_eq!( - envd_process_list_status(&target, Some("wrong-token")).await?, - envd::reqwest::StatusCode::UNAUTHORIZED - ); + if !upgrade { + assert_eq!( + envd_process_list_status(&target, None).await?, + envd::reqwest::StatusCode::UNAUTHORIZED + ); + assert_eq!( + envd_process_list_status(&target, Some("wrong-token")).await?, + envd::reqwest::StatusCode::UNAUTHORIZED + ); + } assert_envd_process_list_succeeds(&target, resumed_token.expose()).await?; + if upgrade { + restarted.pause_sandbox(sandbox_id).await?; + let paused = restarted + .get_sandbox(&sandbox_id) + .await? + .expect("paused sandbox"); + let state = paused + .paused_state + .as_ref() + .expect("paused state") + .encode()?; + let binary = ConfigManager::global_config().resolved_firecracker_binary_path(); + assert_eq!( + state["common"]["firecracker_binary"], + serde_json::to_value(&binary)? + ); + assert_snapshot_format(&binary, &state, "v1000.0.0").await?; + } restarted.delete_sandbox(sandbox_id).await?; let deleted = restarted.get_sandbox(&sandbox_id).await?; assert!(deleted.is_none(), "sandbox should be removed after delete"); let lookup = restarted.proxy_lookup_for(&sandbox_id).await?; assert_eq!(lookup, ProxyLookupResult::NotFound); + std::fs::write(root.join("resume-complete"), b"ok")?; Ok(()) }) @@ -221,6 +394,32 @@ async fn orchestrator_lifecycle() -> Result<()> { #[tokio::test] async fn orchestrator_capture_snapshot_can_be_published_and_relaunched() -> Result<()> { + let _guard = ORCHESTRATOR_TEST_LOCK.lock().await; + const CHILD_ENV: &str = "AENV_ORCHESTRATOR_CAPTURE_TEST_CHILD"; + if std::env::var_os(CHILD_ENV).is_none() { + // Keep the parent free of global daemon clients and cache database locks + // while the lifecycle test restarts its runtime processes. + let completion = tempdir()?; + let completed = completion.path().join("capture-complete"); + let mut child = tokio::process::Command::new(std::env::current_exe()?); + child + .args([ + "--exact", + "orchestrator::orchestrator_capture_snapshot_can_be_published_and_relaunched", + "--nocapture", + ]) + .env(CHILD_ENV, &completed) + .kill_on_drop(true); + let status = timeout(TEST_TIMEOUT, child.status()) + .await + .map_err(|_| anyhow::anyhow!("capture test process timed out"))??; + anyhow::ensure!(status.success(), "capture test process failed: {status}"); + anyhow::ensure!( + completed.is_file(), + "capture child exited without completing the test" + ); + return Ok(()); + } common::setup().await; timeout(TEST_TIMEOUT, async { let root = tempdir()?; @@ -375,8 +574,13 @@ async fn orchestrator_capture_snapshot_can_be_published_and_relaunched() -> Resu orchestrator.delete_sandbox(relaunched.id).await?; orchestrator.delete_sandbox(sandbox_id).await?; - Ok(()) + Ok::<(), anyhow::Error>(()) }) .await - .map_err(|_| anyhow::anyhow!("test timed out"))? + .map_err(|_| anyhow::anyhow!("test timed out"))??; + std::fs::write( + std::env::var_os(CHILD_ENV).expect("capture completion path"), + b"ok", + )?; + Ok(()) }