From 9eecbbca426fec9c903929109bd6c799f96fb989 Mon Sep 17 00:00:00 2001 From: Luke Craig Date: Sun, 17 May 2026 18:55:11 -0400 Subject: [PATCH 1/6] hyperfile: back native mmap with qemu aperture --- pyplugins/compat/qemu_compat.py | 26 ++ pyplugins/hyperfile/anonfs.py | 18 +- pyplugins/hyperfile/devfs.py | 24 +- pyplugins/hyperfile/procfs.py | 24 +- pyplugins/hyperfile/qemu_mem.py | 294 ++++++++++++++++++ pyplugins/testing/native_mmap.py | 122 ++++++++ .../patches/tests/native_mmap.yaml | 88 ++++++ tests/unit_tests/test_target/test.py | 10 +- 8 files changed, 602 insertions(+), 4 deletions(-) create mode 100644 pyplugins/hyperfile/qemu_mem.py create mode 100644 pyplugins/testing/native_mmap.py create mode 100644 tests/unit_tests/test_target/patches/tests/native_mmap.yaml diff --git a/pyplugins/compat/qemu_compat.py b/pyplugins/compat/qemu_compat.py index 3acb36941..abf3b323b 100644 --- a/pyplugins/compat/qemu_compat.py +++ b/pyplugins/compat/qemu_compat.py @@ -34,6 +34,10 @@ uint64_t *ret); typedef int (*kvm_penguin_after_guest_init_cb_t)(MachineState *machine, void *opaque); +typedef uint64_t (*penguin_mmio_read_cb_t)(uint64_t addr, unsigned size, + void *opaque); +typedef void (*penguin_mmio_write_cb_t)(uint64_t addr, uint64_t data, + unsigned size, void *opaque); extern MachineState *current_machine; extern int (*qemu_main)(void); @@ -61,6 +65,11 @@ uint64_t a2, uint64_t a3, uint64_t a4, uint64_t a5, uint64_t *ret); +int penguin_qemu_add_mmio_region(uint64_t base, uint64_t size, + const char *name, + penguin_mmio_read_cb_t read_cb, + penguin_mmio_write_cb_t write_cb, + void *opaque); """ @@ -359,6 +368,23 @@ def __init__( ): if declaration not in cdef_source: cdef_source += f"\n{declaration}\n" + if "penguin_mmio_read_cb_t" not in cdef_source: + cdef_source += ( + "\ntypedef uint64_t (*penguin_mmio_read_cb_t)" + "(uint64_t addr, unsigned size, void *opaque);\n" + ) + if "penguin_mmio_write_cb_t" not in cdef_source: + cdef_source += ( + "\ntypedef void (*penguin_mmio_write_cb_t)" + "(uint64_t addr, uint64_t data, unsigned size, void *opaque);\n" + ) + if "penguin_qemu_add_mmio_region" not in cdef_source: + cdef_source += ( + "\nint penguin_qemu_add_mmio_region(" + "uint64_t base, uint64_t size, const char *name, " + "penguin_mmio_read_cb_t read_cb, " + "penguin_mmio_write_cb_t write_cb, void *opaque);\n" + ) self.ffi.cdef(cdef_source) flags = getattr(os, "RTLD_GLOBAL", 0) | getattr(os, "RTLD_NOW", 0) diff --git a/pyplugins/hyperfile/anonfs.py b/pyplugins/hyperfile/anonfs.py index 70e84c4e1..db674cee4 100644 --- a/pyplugins/hyperfile/anonfs.py +++ b/pyplugins/hyperfile/anonfs.py @@ -90,11 +90,13 @@ def register_anon_file(self, vfs_file: VFSFile, name: str = "[igloo_anon]") -> G fops = yield from self._make_fops_struct(vfs_file) kffi = plugins.kffi + mmap_phys_addr = self._mmap_phys_addr(vfs_file) init_data = { "name": name.encode("latin-1", errors="ignore"), "hf_id": hf_id, - "ops": fops + "ops": fops, + "mmap_phys_addr": mmap_phys_addr } req = kffi.new("struct portal_anonfs_create_req", init_data) @@ -111,6 +113,20 @@ def register_anon_file(self, vfs_file: VFSFile, name: str = "[igloo_anon]") -> G self.logger.debug(f"Injected anon file '{name}' at FD {fd}") return fd + def _mmap_phys_addr(self, vfs_file: VFSFile) -> int: + supports_default_mmap = any( + ( + getattr(vfs_file, "SUPPORT_MMAP", False), + getattr(vfs_file, "SIZE", 0), + ) + ) and not vfs_file._is_overridden("mmap") + if not supports_default_mmap: + return 0 + qemu_mem = getattr(plugins, "qemu_mem", None) + if qemu_mem is None: + return 0 + return qemu_mem.allocate_file(vfs_file) + def register_socket(self, sock_file: SocketFile) -> Generator[int, None, int]: """ Injects a true kernel socket object into the guest process table. diff --git a/pyplugins/hyperfile/devfs.py b/pyplugins/hyperfile/devfs.py index 1bb5c198b..46e52afc2 100644 --- a/pyplugins/hyperfile/devfs.py +++ b/pyplugins/hyperfile/devfs.py @@ -182,6 +182,13 @@ def _register_devfs(self, devfs_list: List[Tuple[str, DevFile, int, int]]) -> Ge ops = yield from self._make_ops_struct(devfs_file) kffi = plugins.kffi + mmap_phys_addr = self._mmap_phys_addr(devfs_file) + support_mmap = any( + ( + getattr(devfs_file, "SUPPORT_MMAP", False), + mmap_phys_addr, + ) + ) init_data = { "name": file_name.encode("latin-1", errors="ignore"), @@ -194,7 +201,8 @@ def _register_devfs(self, devfs_list: List[Tuple[str, DevFile, int, int]]) -> Ge "parent_id": parent_id, "size": getattr(devfs_file, "SIZE", 0), "mode": getattr(devfs_file, "MODE", 0o666), - "support_mmap": 1 if getattr(devfs_file, "SUPPORT_MMAP", False) else 0, + "support_mmap": 1 if support_mmap else 0, + "mmap_phys_addr": mmap_phys_addr, "is_block": 1 if getattr(devfs_file, "IS_BLOCK", False) else 0, "logical_block_size": getattr(devfs_file, "LOGICAL_BLOCK_SIZE", 512) } @@ -216,6 +224,20 @@ def _register_devfs(self, devfs_list: List[Tuple[str, DevFile, int, int]]) -> Ge self.logger.debug(f"Registered devfs device '{fname}' with kernel") + def _mmap_phys_addr(self, devfs_file: DevFile) -> int: + supports_default_mmap = any( + ( + getattr(devfs_file, "SUPPORT_MMAP", False), + getattr(devfs_file, "SIZE", 0), + ) + ) and not devfs_file._is_overridden("mmap") + if not supports_default_mmap: + return 0 + qemu_mem = getattr(plugins, "qemu_mem", None) + if qemu_mem is None: + return 0 + return qemu_mem.allocate_file(devfs_file) + def _hyperdevfs_interrupt_handler(self) -> Generator[bool, None, bool]: if not self._pending_devfs: return False diff --git a/pyplugins/hyperfile/procfs.py b/pyplugins/hyperfile/procfs.py index 25cef9d57..3d6102428 100644 --- a/pyplugins/hyperfile/procfs.py +++ b/pyplugins/hyperfile/procfs.py @@ -180,6 +180,13 @@ def _register_procs(self, procs: List[Tuple[str, ProcFile]]) -> Generator[int, N fops = yield from self._make_fops_struct(proc) kffi = plugins.kffi + mmap_phys_addr = self._mmap_phys_addr(proc) + support_mmap = any( + ( + getattr(proc, "SUPPORT_MMAP", False), + mmap_phys_addr, + ) + ) init_data = { "path": file_name.encode("latin-1", errors="ignore"), @@ -188,7 +195,8 @@ def _register_procs(self, procs: List[Tuple[str, ProcFile]]) -> Generator[int, N "mode": getattr(proc, "MODE", 0o444), "parent_id": parent_id, "replace": 1, - "support_mmap": 1 if getattr(proc, "SUPPORT_MMAP", False) else 0 + "support_mmap": 1 if support_mmap else 0, + "mmap_phys_addr": mmap_phys_addr } req = kffi.new("struct portal_procfs_create_req", init_data) @@ -206,6 +214,20 @@ def _register_procs(self, procs: List[Tuple[str, ProcFile]]) -> Generator[int, N continue self.logger.debug(f"Registered proc '{fname}' with kernel") + def _mmap_phys_addr(self, proc: ProcFile) -> int: + supports_default_mmap = any( + ( + getattr(proc, "SUPPORT_MMAP", False), + getattr(proc, "SIZE", 0), + ) + ) and not proc._is_overridden("mmap") + if not supports_default_mmap: + return 0 + qemu_mem = getattr(plugins, "qemu_mem", None) + if qemu_mem is None: + return 0 + return qemu_mem.allocate_file(proc) + def _proc_interrupt_handler(self) -> Generator[bool, None, bool]: """ Process pending proc registrations. diff --git a/pyplugins/hyperfile/qemu_mem.py b/pyplugins/hyperfile/qemu_mem.py new file mode 100644 index 000000000..a972fda4d --- /dev/null +++ b/pyplugins/hyperfile/qemu_mem.py @@ -0,0 +1,294 @@ +from dataclasses import dataclass +from typing import Callable, Optional + +from penguin import Plugin + + +PAGE_SIZE = 4096 +DEFAULT_MMAP_BASE = 0xfe000000 +DEFAULT_MMAP_SIZE = 16 * 1024 * 1024 + + +@dataclass +class Allocation: + name: str + offset: int + size: int + storage: bytearray + read_cb: Optional[Callable] + write_cb: Optional[Callable] + + +def _parse_int(value, default: int) -> int: + if value is None: + return default + if isinstance(value, int): + return value + if isinstance(value, str): + text = value.strip() + suffixes = { + "k": 1024, + "kb": 1024, + "m": 1024 * 1024, + "mb": 1024 * 1024, + "g": 1024 * 1024 * 1024, + "gb": 1024 * 1024 * 1024, + } + lowered = text.lower() + for suffix, scale in suffixes.items(): + if lowered.endswith(suffix): + return int(text[:-len(suffix)], 0) * scale + return int(text, 0) + raise ValueError(f"Unsupported integer value {value!r}") + + +def _align_up(value: int, alignment: int = PAGE_SIZE) -> int: + return (value + alignment - 1) & ~(alignment - 1) + + +class QemuMem(Plugin): + """ + Native QEMU MMIO aperture used as a backing store for guest mmap() users. + """ + + def __init__(self): + if not hasattr(self.panda, "set_after_guest_init_callback"): + raise RuntimeError( + "qemu_mem requires a QEMU compatibility backend with " + "set_after_guest_init_callback; update the Penguin QEMU " + "compat layer or disable native mmap users." + ) + if not hasattr(self.panda, "ffi") or not hasattr(self.panda, "lib"): + raise RuntimeError( + "qemu_mem requires Penguin's QEMU CFFI backend; it cannot run " + "on a PANDA-only backend." + ) + if not hasattr(self.panda.lib, "penguin_qemu_add_mmio_region"): + raise RuntimeError( + "qemu_mem requires QEMU to export " + "penguin_qemu_add_mmio_region; rebuild/install Penguin's " + "QEMU package before enabling native mmap." + ) + + self.base = _parse_int(self.get_arg("mmap_base"), DEFAULT_MMAP_BASE) + self.size = _parse_int(self.get_arg("mmap_size"), DEFAULT_MMAP_SIZE) + if self.base % PAGE_SIZE: + raise ValueError( + f"qemu_mem mmap_base must be page aligned: 0x{self.base:x}" + ) + if self.size <= 0 or self.size % PAGE_SIZE: + raise ValueError( + "qemu_mem mmap_size must be a positive page-aligned size: " + f"{self.size}" + ) + + self._used = 0 + self._allocations: list[Allocation] = [] + self._installed = False + ffi = self.panda.ffi + self._read_cb = ffi.callback( + "uint64_t(uint64_t, unsigned, void *)" + )(self._read) + self._write_cb = ffi.callback( + "void(uint64_t, uint64_t, unsigned, void *)" + )(self._write) + self.panda.set_after_guest_init_callback(self._after_guest_init) + + def allocate_region( + self, + name: str, + size: int, + read_cb=None, + write_cb=None, + initial=None, + ) -> int: + size = _align_up(max(int(size), 1)) + offset = _align_up(self._used) + end = offset + size + if end > self.size: + raise RuntimeError( + f"qemu_mem mmap aperture exhausted allocating {name!r}: " + f"requested={size} used={self._used} limit={self.size}. " + "Increase the aperture, for example:\n" + "plugins:\n" + " qemu_mem:\n" + " mmap_size: 64M" + ) + + storage = bytearray(size) + if initial: + initial_bytes = bytes(initial) + storage[:min(len(initial_bytes), size)] = initial_bytes[:size] + + self._used = end + alloc = Allocation(name, offset, size, storage, read_cb, write_cb) + self._allocations.append(alloc) + self.logger.debug( + "Allocated mmap region %s at 0x%x size=0x%x", + name, + self.base + offset, + size, + ) + return self.base + offset + + def allocate_file(self, file_model) -> int: + if getattr(file_model, "_qemu_mem_phys_addr", 0): + return file_model._qemu_mem_phys_addr + + size = int(getattr(file_model, "SIZE", 0) or PAGE_SIZE) + initial = self._initial_data(file_model, size) + name = getattr( + file_model, + "full_path", + getattr(file_model, "PATH", "mmap"), + ) + addr = self.allocate_region( + name, + size, + read_cb=lambda offset, length: self._file_read( + file_model, + offset, + length, + ), + write_cb=lambda offset, data: self._file_write( + file_model, + offset, + data, + ), + initial=initial, + ) + file_model._qemu_mem_phys_addr = addr + file_model._qemu_mem_storage = self._allocation_for_addr(addr).storage + return addr + + def _after_guest_init(self, _machine, _opaque): + name = self.panda.ffi.new("char[]", b"penguin-mmap-aperture") + ret = self.panda.lib.penguin_qemu_add_mmio_region( + self.base, + self.size, + name, + self._read_cb, + self._write_cb, + self.panda.ffi.NULL, + ) + if ret != 0: + self.logger.error( + "Failed to install qemu_mem aperture at 0x%x size=0x%x", + self.base, + self.size, + ) + return 1 + self._installed = True + self.logger.info( + "Installed qemu_mem mmap aperture at 0x%x size=0x%x", + self.base, + self.size, + ) + return 0 + + def _allocation_for_addr(self, addr: int) -> Allocation: + relative = addr - self.base + for alloc in self._allocations: + if alloc.offset <= relative < alloc.offset + alloc.size: + return alloc + raise KeyError(f"No qemu_mem allocation owns address 0x{addr:x}") + + def _allocation_for_offset(self, offset: int) -> tuple[Allocation, int]: + for alloc in self._allocations: + if alloc.offset <= offset < alloc.offset + alloc.size: + return alloc, offset - alloc.offset + raise KeyError( + f"No qemu_mem allocation owns aperture offset 0x{offset:x}" + ) + + def _read(self, addr, size, _opaque): + try: + alloc, file_offset = self._allocation_for_offset(int(addr)) + read_len = min(int(size), alloc.size - file_offset) + if alloc.read_cb: + data = alloc.read_cb(file_offset, read_len) + else: + data = bytes( + alloc.storage[file_offset:file_offset + read_len] + ) + data = bytes(data or b"")[:read_len] + if len(data) < read_len: + data += b"\x00" * (read_len - len(data)) + self.logger.debug( + "mmap read %s offset=0x%x size=%d", + alloc.name, + file_offset, + read_len, + ) + return int.from_bytes( + data.ljust(int(size), b"\x00")[:int(size)], + "little", + ) + except Exception as exc: + self.logger.error( + "qemu_mem mmap read failed at offset 0x%x: %s", + int(addr), + exc, + ) + return 0 + + def _write(self, addr, data, size, _opaque): + try: + alloc, file_offset = self._allocation_for_offset(int(addr)) + write_len = min(int(size), alloc.size - file_offset) + payload = int(data).to_bytes(int(size), "little")[:write_len] + alloc.storage[file_offset:file_offset + write_len] = payload + if alloc.write_cb: + alloc.write_cb(file_offset, payload) + self.logger.debug( + "mmap write %s offset=0x%x size=%d", + alloc.name, + file_offset, + write_len, + ) + except Exception as exc: + self.logger.error( + "qemu_mem mmap write failed at offset 0x%x: %s", + int(addr), + exc, + ) + + def _initial_data(self, file_model, size: int) -> bytes: + if hasattr(file_model, "_data"): + data = file_model._data + if isinstance(data, str): + data = data.encode("utf-8") + return bytes(data)[:size] + if hasattr(file_model, "written_data"): + return bytes(file_model.written_data)[:size] + if hasattr(file_model, "data"): + return bytes(file_model.data)[:size] + return b"" + + def _file_read(self, file_model, offset: int, length: int) -> bytes: + storage = getattr(file_model, "_qemu_mem_storage") + return bytes(storage[offset:offset + length]) + + def _file_write(self, file_model, offset: int, data: bytes) -> None: + storage = getattr(file_model, "_qemu_mem_storage") + end = offset + len(data) + storage[offset:end] = data + if hasattr(file_model, "_data"): + current_len = len(getattr(file_model, "_data", b"")) + file_model._data = bytes(storage[:max(current_len, end)]) + file_model.SIZE = max( + getattr(file_model, "SIZE", 0), + len(file_model._data), + ) + if hasattr(file_model, "written_data"): + previous = file_model.written_data[:offset] + if len(previous) < offset: + previous += b"\x00" * (offset - len(previous)) + file_model.written_data = ( + previous + data + file_model.written_data[end:] + ) + if ( + hasattr(file_model, "data") + and isinstance(file_model.data, bytearray) + ): + file_model.data[offset:end] = data diff --git a/pyplugins/testing/native_mmap.py b/pyplugins/testing/native_mmap.py new file mode 100644 index 000000000..94c1172b7 --- /dev/null +++ b/pyplugins/testing/native_mmap.py @@ -0,0 +1,122 @@ +from penguin import Plugin, plugins +from apis.syscalls import ValueFilter +from hyperfile.models.base import ( + AnonFile, + CharPtr, + DevFile, + LoffT, + MtdDevice, + ProcFile, + SizeT, +) +from hyperfile.models.read import ReadConstBuf +from hyperfile.models.write import WriteRecord + + +class NativeMmapDevFile(ReadConstBuf, WriteRecord, DevFile): + PATH = "mmap_native" + SUPPORT_MMAP = True + + def __init__(self): + super().__init__(buffer=b"dev native mmap\n", size=4096) + + +class NativeMmapProcFile(ReadConstBuf, WriteRecord, ProcFile): + PATH = "mmap_native" + SUPPORT_MMAP = True + + def __init__(self): + super().__init__(buffer=b"proc native mmap\n", size=4096) + + +class NativeMmapAnonFile(ReadConstBuf, WriteRecord, AnonFile): + SUPPORT_MMAP = True + + def __init__(self): + super().__init__( + path="/tmp/mmap_native_anon", + buffer=b"anon native mmap\n", + size=4096, + ) + + +class NativeMmapMtdDevice(MtdDevice): + NAME = "mmap_native_mtd" + SIZE = 64 * 1024 + ERASE_SIZE = 4096 + WRITE_SIZE = 1 + OOB_SIZE = 0 + TYPE = "nor" + + def __init__(self): + self.data = bytearray(b"\xff" * self.SIZE) + initial = b"mtd native mmap\n" + self.data[:len(initial)] = initial + super().__init__() + + def read(self, ptregs, offset: LoffT, length: SizeT, buf_ptr: CharPtr): + off = int(offset) + size = int(length) + if off >= self.SIZE: + ptregs.retval = 0 + return 0 + chunk = min(size, self.SIZE - off) + yield from plugins.mem.write( + buf_ptr, + bytes(self.data[off:off + chunk]), + ) + ptregs.retval = 0 + return 0 + + def write(self, ptregs, offset: LoffT, length: SizeT, buf_ptr: CharPtr): + off = int(offset) + size = int(length) + if off >= self.SIZE: + ptregs.retval = -28 + return -28 + chunk = min(size, self.SIZE - off) + raw = yield from plugins.mem.read(buf_ptr, chunk, fmt="bytes") + self.data[off:off + chunk] = raw + ptregs.retval = 0 + return 0 + + def erase(self, ptregs, offset: LoffT, length: SizeT): + off = int(offset) + size = int(length) + if off >= self.SIZE: + ptregs.retval = -22 + return -22 + chunk = min(size, self.SIZE - off) + self.data[off:off + chunk] = b"\xff" * chunk + ptregs.retval = 0 + return 0 + + +class NativeMmap(Plugin): + def __init__(self): + self.anon_file = NativeMmapAnonFile() + + plugins.devfs.register_devfs(NativeMmapDevFile()) + plugins.procfs.register_proc(NativeMmapProcFile()) + plugins.mtd.register_mtd(NativeMmapMtdDevice()) + + syscalls = plugins.syscalls + syscalls.syscall( + "on_sys_open_enter", + arg_filters=[ValueFilter.string_exact("/tmp/mmap_native_anon")] + )(self.on_open_anon) + syscalls.syscall( + "on_sys_openat_enter", + arg_filters=[ + None, + ValueFilter.string_exact("/tmp/mmap_native_anon"), + ], + )(self.on_open_anon) + + def on_open_anon(self, regs, proto, syscall, *args): + syscall.skip_syscall = True + fd = yield from plugins.anonfs.register_anon_file( + self.anon_file, + name="[mmap_native_anon]", + ) + syscall.retval = fd diff --git a/tests/unit_tests/test_target/patches/tests/native_mmap.yaml b/tests/unit_tests/test_target/patches/tests/native_mmap.yaml new file mode 100644 index 000000000..9cb581946 --- /dev/null +++ b/tests/unit_tests/test_target/patches/tests/native_mmap.yaml @@ -0,0 +1,88 @@ +plugins: + qemu_mem: + mmap_base: 0x90000000 + mmap_size: 16M + native_mmap: {} + verifier: + conditions: + native_mmap_dev: + type: file_contains + file: shared/tests/mmap_test/stdout + string: "native mmap dev PASS" + native_mmap_proc: + type: file_contains + file: shared/tests/mmap_test/stdout + string: "native mmap proc PASS" + native_mmap_anon: + type: file_contains + file: shared/tests/mmap_test/stdout + string: "native mmap anon PASS" + native_mmap_mtd: + type: file_contains + file: shared/tests/mmap_test/stdout + string: "native mmap mtd PASS" + native_mmap_done: + type: file_contains + file: shared/tests/mmap_test/stdout + string: "/tests/mmap_test PASS" + +static_files: + /tests/mmap_test: + type: inline_file + mode: 73 + contents: | + #!/igloo/utils/sh + set -eux + /igloo/utils/busybox cat > /tmp/mmap_native.py <<'PY' + import uos, uctypes, ffilib + + libc = ffilib.libc() + mmap = libc.func("p", "mmap", "pLiiiq") + munmap = libc.func("i", "munmap", "pL") + + PROT_READ = 1 + PROT_WRITE = 2 + MAP_SHARED = 1 + + def check_mmap(path, expected, replacement, marker): + with open(path, "r+b") as f: + addr = mmap(0, 4096, PROT_READ | PROT_WRITE, MAP_SHARED, f.fileno(), 0) + if addr == -1: + raise OSError(uos.errno()) + assert uctypes.bytes_at(addr, len(expected)) == expected + buf = uctypes.bytearray_at(addr, len(replacement)) + buf[:] = replacement + munmap(addr, 4096) + with open(path, "rb") as f: + assert f.read(len(replacement)) == replacement + print(marker) + + check_mmap( + "/dev/mmap_native", + b"dev native mmap\n", + b"dev mmap changed\n", + "native mmap dev PASS", + ) + check_mmap( + "/proc/mmap_native", + b"proc native mmap\n", + b"proc mmap change\n", + "native mmap proc PASS", + ) + check_mmap( + "/tmp/mmap_native_anon", + b"anon native mmap\n", + b"anon mmap change\n", + "native mmap anon PASS", + ) + PY + + /igloo/utils/busybox mdev -s + /igloo/utils/micropython /tmp/mmap_native.py + /igloo/utils/busybox mdev -s || true + MTD=$(/igloo/utils/busybox grep '"mmap_native_mtd"' /proc/mtd | /igloo/utils/busybox cut -d: -f1) + /igloo/utils/busybox cat /dev/${MTD} | /igloo/utils/busybox strings | /igloo/utils/busybox grep "mtd native mmap" + /igloo/utils/busybox echo "mtd changed" > /dev/${MTD} + /igloo/utils/busybox cat /dev/${MTD} | /igloo/utils/busybox strings | /igloo/utils/busybox grep "mtd changed" + /igloo/utils/busybox echo "native mmap mtd PASS" + /igloo/utils/busybox echo "/tests/mmap_test PASS" diff --git a/tests/unit_tests/test_target/test.py b/tests/unit_tests/test_target/test.py index 0f25cb1ec..c29ca084b 100755 --- a/tests/unit_tests/test_target/test.py +++ b/tests/unit_tests/test_target/test.py @@ -163,7 +163,15 @@ def test(kernel, arch, image, name, test_file, docs_only, mode): continue logger.info(f"Running tests for kernel {k} on arch {a} (mode={mode})") - run_test(k, a, image, None, docs_only, execution_mode=mode, name=name or "test_target") + run_test( + k, + a, + image, + test_file, + docs_only, + execution_mode=mode, + name=name or "test_target", + ) if __name__ == "__main__": From 71e05f1a2b38cfbd2bb8e5439b1909a370133988 Mon Sep 17 00:00:00 2001 From: Luke Craig Date: Sun, 17 May 2026 22:48:00 -0400 Subject: [PATCH 2/6] hyperfile: preserve legacy mmap request shape --- pyplugins/hyperfile/anonfs.py | 3 ++- pyplugins/hyperfile/devfs.py | 19 +++++++++++-------- pyplugins/hyperfile/procfs.py | 21 ++++++++++++--------- 3 files changed, 25 insertions(+), 18 deletions(-) diff --git a/pyplugins/hyperfile/anonfs.py b/pyplugins/hyperfile/anonfs.py index db674cee4..51256f4a4 100644 --- a/pyplugins/hyperfile/anonfs.py +++ b/pyplugins/hyperfile/anonfs.py @@ -96,8 +96,9 @@ def register_anon_file(self, vfs_file: VFSFile, name: str = "[igloo_anon]") -> G "name": name.encode("latin-1", errors="ignore"), "hf_id": hf_id, "ops": fops, - "mmap_phys_addr": mmap_phys_addr } + if mmap_phys_addr: + init_data["mmap_phys_addr"] = mmap_phys_addr req = kffi.new("struct portal_anonfs_create_req", init_data) req_bytes = bytes(req) diff --git a/pyplugins/hyperfile/devfs.py b/pyplugins/hyperfile/devfs.py index 46e52afc2..6c5f01662 100644 --- a/pyplugins/hyperfile/devfs.py +++ b/pyplugins/hyperfile/devfs.py @@ -202,10 +202,11 @@ def _register_devfs(self, devfs_list: List[Tuple[str, DevFile, int, int]]) -> Ge "size": getattr(devfs_file, "SIZE", 0), "mode": getattr(devfs_file, "MODE", 0o666), "support_mmap": 1 if support_mmap else 0, - "mmap_phys_addr": mmap_phys_addr, "is_block": 1 if getattr(devfs_file, "IS_BLOCK", False) else 0, "logical_block_size": getattr(devfs_file, "LOGICAL_BLOCK_SIZE", 512) } + if mmap_phys_addr: + init_data["mmap_phys_addr"] = mmap_phys_addr req = kffi.new("struct portal_devfs_create_req", init_data) req_bytes = bytes(req) @@ -225,19 +226,21 @@ def _register_devfs(self, devfs_list: List[Tuple[str, DevFile, int, int]]) -> Ge self.logger.debug(f"Registered devfs device '{fname}' with kernel") def _mmap_phys_addr(self, devfs_file: DevFile) -> int: - supports_default_mmap = any( - ( - getattr(devfs_file, "SUPPORT_MMAP", False), - getattr(devfs_file, "SIZE", 0), - ) - ) and not devfs_file._is_overridden("mmap") - if not supports_default_mmap: + if not self._supports_default_mmap(devfs_file): return 0 qemu_mem = getattr(plugins, "qemu_mem", None) if qemu_mem is None: return 0 return qemu_mem.allocate_file(devfs_file) + def _supports_default_mmap(self, devfs_file: DevFile) -> bool: + return any( + ( + getattr(devfs_file, "SUPPORT_MMAP", False), + getattr(devfs_file, "SIZE", 0), + ) + ) and not devfs_file._is_overridden("mmap") + def _hyperdevfs_interrupt_handler(self) -> Generator[bool, None, bool]: if not self._pending_devfs: return False diff --git a/pyplugins/hyperfile/procfs.py b/pyplugins/hyperfile/procfs.py index 3d6102428..f0e315bba 100644 --- a/pyplugins/hyperfile/procfs.py +++ b/pyplugins/hyperfile/procfs.py @@ -195,9 +195,10 @@ def _register_procs(self, procs: List[Tuple[str, ProcFile]]) -> Generator[int, N "mode": getattr(proc, "MODE", 0o444), "parent_id": parent_id, "replace": 1, - "support_mmap": 1 if support_mmap else 0, - "mmap_phys_addr": mmap_phys_addr + "support_mmap": 1 if support_mmap else 0 } + if mmap_phys_addr: + init_data["mmap_phys_addr"] = mmap_phys_addr req = kffi.new("struct portal_procfs_create_req", init_data) req_bytes = bytes(req) @@ -215,19 +216,21 @@ def _register_procs(self, procs: List[Tuple[str, ProcFile]]) -> Generator[int, N self.logger.debug(f"Registered proc '{fname}' with kernel") def _mmap_phys_addr(self, proc: ProcFile) -> int: - supports_default_mmap = any( - ( - getattr(proc, "SUPPORT_MMAP", False), - getattr(proc, "SIZE", 0), - ) - ) and not proc._is_overridden("mmap") - if not supports_default_mmap: + if not self._supports_default_mmap(proc): return 0 qemu_mem = getattr(plugins, "qemu_mem", None) if qemu_mem is None: return 0 return qemu_mem.allocate_file(proc) + def _supports_default_mmap(self, proc: ProcFile) -> bool: + return any( + ( + getattr(proc, "SUPPORT_MMAP", False), + getattr(proc, "SIZE", 0), + ) + ) and not proc._is_overridden("mmap") + def _proc_interrupt_handler(self) -> Generator[bool, None, bool]: """ Process pending proc registrations. From 3d2324eafbfe2ddeed39e8eac83bb99b67fff8e1 Mon Sep 17 00:00:00 2001 From: Luke Craig Date: Sun, 17 May 2026 23:13:17 -0400 Subject: [PATCH 3/6] hyperfile: keep qemu_mem mmap opt-in --- pyplugins/hyperfile/anonfs.py | 4 ++-- pyplugins/hyperfile/devfs.py | 4 ++-- pyplugins/hyperfile/procfs.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pyplugins/hyperfile/anonfs.py b/pyplugins/hyperfile/anonfs.py index 51256f4a4..d3fb9042c 100644 --- a/pyplugins/hyperfile/anonfs.py +++ b/pyplugins/hyperfile/anonfs.py @@ -123,9 +123,9 @@ def _mmap_phys_addr(self, vfs_file: VFSFile) -> int: ) and not vfs_file._is_overridden("mmap") if not supports_default_mmap: return 0 - qemu_mem = getattr(plugins, "qemu_mem", None) - if qemu_mem is None: + if "qemu_mem" not in plugins: return 0 + qemu_mem = plugins.get_plugin_by_name("qemu_mem") return qemu_mem.allocate_file(vfs_file) def register_socket(self, sock_file: SocketFile) -> Generator[int, None, int]: diff --git a/pyplugins/hyperfile/devfs.py b/pyplugins/hyperfile/devfs.py index 6c5f01662..76f7e6848 100644 --- a/pyplugins/hyperfile/devfs.py +++ b/pyplugins/hyperfile/devfs.py @@ -228,9 +228,9 @@ def _register_devfs(self, devfs_list: List[Tuple[str, DevFile, int, int]]) -> Ge def _mmap_phys_addr(self, devfs_file: DevFile) -> int: if not self._supports_default_mmap(devfs_file): return 0 - qemu_mem = getattr(plugins, "qemu_mem", None) - if qemu_mem is None: + if "qemu_mem" not in plugins: return 0 + qemu_mem = plugins.get_plugin_by_name("qemu_mem") return qemu_mem.allocate_file(devfs_file) def _supports_default_mmap(self, devfs_file: DevFile) -> bool: diff --git a/pyplugins/hyperfile/procfs.py b/pyplugins/hyperfile/procfs.py index f0e315bba..4925699df 100644 --- a/pyplugins/hyperfile/procfs.py +++ b/pyplugins/hyperfile/procfs.py @@ -218,9 +218,9 @@ def _register_procs(self, procs: List[Tuple[str, ProcFile]]) -> Generator[int, N def _mmap_phys_addr(self, proc: ProcFile) -> int: if not self._supports_default_mmap(proc): return 0 - qemu_mem = getattr(plugins, "qemu_mem", None) - if qemu_mem is None: + if "qemu_mem" not in plugins: return 0 + qemu_mem = plugins.get_plugin_by_name("qemu_mem") return qemu_mem.allocate_file(proc) def _supports_default_mmap(self, proc: ProcFile) -> bool: From 2bfad70fa016986f97fd7f728a2d6c618a01c47a Mon Sep 17 00:00:00 2001 From: Luke Craig Date: Sun, 17 May 2026 23:40:15 -0400 Subject: [PATCH 4/6] portalcall: ignore non-portal sendto events --- pyplugins/apis/portalcall.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pyplugins/apis/portalcall.py b/pyplugins/apis/portalcall.py index d26789424..cd3c94006 100644 --- a/pyplugins/apis/portalcall.py +++ b/pyplugins/apis/portalcall.py @@ -37,6 +37,7 @@ def my_portalcall_handler(arg1, arg2): PORTAL_MAGIC = 0xc1d1e1f1 PORTAL_MAGIC_64 = 0xffffffffc1d1e1f1 +PORTAL_MAGIC_MASK = 0xffffffff class PortalCall(Plugin): @@ -53,6 +54,8 @@ def __init__(self) -> None: plugins.syscalls.syscall("on_sys_sendto_enter", arg_filters=[PORTAL_MAGIC, None, None, None, None])(self._portalcall_syscall_handler) def _portalcall_syscall_handler(self, regs, proto, syscall, magic, user_magic, argc, args, dest_addr, addrlen): + if not self._is_portal_magic(magic): + return result = yield from self._dispatch_portalcall(user_magic, argc, args) syscall.skip_syscall = True if isinstance(result, int): @@ -60,6 +63,9 @@ def _portalcall_syscall_handler(self, regs, proto, syscall, magic, user_magic, a else: syscall.retval = 0 # Default to 0 if result is not an int + def _is_portal_magic(self, magic: int) -> bool: + return int(magic) & PORTAL_MAGIC_MASK == PORTAL_MAGIC + def _dispatch_portalcall(self, user_magic, argc, args): handler = self._portalcall_registry.get(user_magic & 0xffffffff) if handler is None: From 00d552bbadb1bb6779aaaeb6685e71df12b581e3 Mon Sep 17 00:00:00 2001 From: Luke Craig Date: Sun, 17 May 2026 23:44:19 -0400 Subject: [PATCH 5/6] events: accept portalcall transport --- pyplugins/apis/events.py | 42 +++++++++++++++++++++++++++++++ pyplugins/interventions/nvram2.py | 26 ++++++++++++------- 2 files changed, 59 insertions(+), 9 deletions(-) diff --git a/pyplugins/apis/events.py b/pyplugins/apis/events.py index 2530a8923..dd6534187 100644 --- a/pyplugins/apis/events.py +++ b/pyplugins/apis/events.py @@ -51,7 +51,9 @@ def on_open(cpu, filename, flags): """ from penguin import plugins, Plugin +from penguin.plugin_manager import resolve_bound_method_from_class from hyper.consts import igloo_hypercall_constants as iconsts +from typing import Iterator EVENTS = { @@ -103,6 +105,7 @@ def __init__(self): """ # MAGIC -> [fn1, fn2, fn3,...] self.callbacks = {} + self._portalcall_handlers = set() for event_num, (name, args) in EVENTS.items(): plugins.register(self, name, register_notify=self.register_notify) @@ -148,6 +151,44 @@ def generic_hypercall(cpu): raise ValueError(f"Unknown argument type {arg}") plugins.publish(self, self.callbacks[magic], *args) + def _setup_portalcall_handler(self, magic, arg_types): + if magic in self._portalcall_handlers: + return + self._portalcall_handlers.add(magic) + + @plugins.portalcall.portalcall(magic) + def generic_portalcall(*raw_args): + args = [None] + for i, arg in enumerate(arg_types): + argval = raw_args[i] if i < len(raw_args) else 0 + if arg is int: + args.append(argval) + elif arg is str: + try: + args.append((yield from plugins.mem.read_str(argval))) + except ValueError: + self.logger.debug( + f"arg read fail: {magic} {argval:x} {i} {arg}" + ) + return 1 + elif arg is bool: + args.append(argval != 0) + elif arg is None: + pass + else: + raise ValueError(f"Unknown argument type {arg}") + + result = 0 + for cb in plugins.plugin_cbs[self][self.callbacks[magic]]: + if not hasattr(cb, '__self__') and hasattr(cb, '__qualname__') and '.' in cb.__qualname__: + cb = resolve_bound_method_from_class(cb) + cb_result = cb(*args) + if isinstance(cb_result, Iterator): + cb_result = yield from cb_result + if isinstance(cb_result, int): + result = cb_result + return result + def register_notify(self, name, callback): """ Register a callback for an event. @@ -166,6 +207,7 @@ def register_notify(self, name, callback): if ename == name: if self.callbacks.get(magic, None) is None: self._setup_hypercall_handler(magic, arg_types) + self._setup_portalcall_handler(magic, arg_types) self.callbacks[magic] = [] self.callbacks[magic] = name return diff --git a/pyplugins/interventions/nvram2.py b/pyplugins/interventions/nvram2.py index 3ba10fd81..1f3b6670b 100644 --- a/pyplugins/interventions/nvram2.py +++ b/pyplugins/interventions/nvram2.py @@ -317,7 +317,7 @@ def on_nvram_get_hit(self, regs, key: str) -> None: ------- None """ - self.on_nvram_get(regs, key, True) + return self.on_nvram_get(regs, key, True) @plugins.subscribe(plugins.Events, "igloo_nvram_get_miss") def on_nvram_get_miss(self, regs, key: str) -> None: @@ -335,7 +335,7 @@ def on_nvram_get_miss(self, regs, key: str) -> None: ------- None """ - self.on_nvram_get(regs, key, False) + return self.on_nvram_get(regs, key, False) def on_nvram_get(self, regs, key: str, hit: bool) -> None: """ @@ -355,12 +355,14 @@ def on_nvram_get(self, regs, key: str, hit: bool) -> None: None """ if "/" not in key: - return + return 0 key = key.split("/")[-1] # It's the full /igloo/libnvram_tmpfs/keyname path status = "hit" if hit else "miss" self.log_write(f"{key},{status},\n") - self.panda.arch.set_arg(regs, 1, 0) + if regs is not None: + self.panda.arch.set_arg(regs, 1, 0) + return 0 # self.logger.debug(f"nvram get {key} {status}") @plugins.subscribe(plugins.Events, "igloo_nvram_set") @@ -382,11 +384,13 @@ def on_nvram_set(self, regs, key: str, newval: str) -> None: None """ if "/" not in key: - return + return 0 key = key.split("/")[-1] # It's the full /igloo/libnvram_tmpfs/keyname path self.log_write(f"{key},set,{newval}\n") - self.panda.arch.set_arg(regs, 1, 0) + if regs is not None: + self.panda.arch.set_arg(regs, 1, 0) self.logger.debug(f"nvram set {key} {newval}") + return 0 @plugins.subscribe(plugins.Events, "igloo_nvram_clear") def on_nvram_clear(self, regs, key: str) -> None: @@ -405,10 +409,12 @@ def on_nvram_clear(self, regs, key: str) -> None: None """ if "/" not in key: - return + return 0 key = key.split("/")[-1] # It's the full /igloo/libnvram_tmpfs/keyname path self.log_write(f"{key},clear,\n") - self.panda.arch.set_arg(regs, 1, 0) + if regs is not None: + self.panda.arch.set_arg(regs, 1, 0) + return 0 # self.logger.debug(f"nvram clear {key}") # self.logger.debug(f"nvram clear {key}") @@ -428,4 +434,6 @@ def on_nvram_logging_enabled(self, regs,) -> None: """ rval = (1 if self.logging_enabled else 0) self.logger.debug(f"nvram logging enabled query, returning {rval}") - self.panda.arch.set_retval(regs, rval) + if regs is not None: + self.panda.arch.set_retval(regs, rval) + return rval From bd71b494700360812ab662553e4d199e3ef098b7 Mon Sep 17 00:00:00 2001 From: Luke Craig Date: Mon, 18 May 2026 00:58:23 -0400 Subject: [PATCH 6/6] tests: avoid powerpc64 busybox pipelines --- .../test_target/patches/tests/pseudofiles_comprehensive.yaml | 3 ++- tests/unit_tests/test_target/patches/tests/sysctl_test.yaml | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/unit_tests/test_target/patches/tests/pseudofiles_comprehensive.yaml b/tests/unit_tests/test_target/patches/tests/pseudofiles_comprehensive.yaml index a5ac750a9..f0a7f9166 100644 --- a/tests/unit_tests/test_target/patches/tests/pseudofiles_comprehensive.yaml +++ b/tests/unit_tests/test_target/patches/tests/pseudofiles_comprehensive.yaml @@ -151,7 +151,8 @@ static_files: echo "test" > /dev/yaml_discard - zero_bytes=$($bb dd if=/dev/yaml_zero bs=10 count=1 2>/dev/null | $bb wc -c) + $bb dd if=/dev/yaml_zero of=/tmp/yaml_zero.out bs=10 count=1 2>/dev/null + zero_bytes=$($bb wc -c < /tmp/yaml_zero.out) if [ "$zero_bytes" != "1" ]; then echo "Error: zero fail (expected 1, got $zero_bytes)" exit 1 diff --git a/tests/unit_tests/test_target/patches/tests/sysctl_test.yaml b/tests/unit_tests/test_target/patches/tests/sysctl_test.yaml index dd4b6cbcf..706bb8172 100644 --- a/tests/unit_tests/test_target/patches/tests/sysctl_test.yaml +++ b/tests/unit_tests/test_target/patches/tests/sysctl_test.yaml @@ -46,7 +46,8 @@ static_files: # Trigger the read callback and verify the python layer returns EXACTLY the data # We use 'dd' to read exactly the bytes we expect and nothing more, # or just check 'wc -c' of the whole output. - actual_count=$(/igloo/utils/busybox cat /proc/sys/vm/drop_caches | /igloo/utils/busybox wc -c) + /igloo/utils/busybox cat /proc/sys/vm/drop_caches > /tmp/drop_caches.out + actual_count=$(/igloo/utils/busybox wc -c < /tmp/drop_caches.out) if [ "$actual_count" -ne 24 ]; then echo "Error: vm/drop_caches returned $actual_count bytes, expected 24" exit 1 @@ -61,4 +62,4 @@ static_files: echo "/tests/sysctl_test.sh PASS" exit 0 - mode: 73 \ No newline at end of file + mode: 73