From c16a86c3ba1082727e61d88096ff005c3084efba Mon Sep 17 00:00:00 2001 From: Matt Brown Date: Sat, 17 Oct 2026 12:20:17 -0400 Subject: [PATCH] nilfs2: extract the identified filesystem moria identified NILFS2 (issue #24) but had no extractor, so -e wrote nothing. Add src/extract/nilfs2.{cpp,hpp} and register it in extractor_for(). NILFS2 is log-structured: the live tree is reached through the latest checkpoint, not a fixed inode table. The extractor: - picks the valid superblock (primary at 1024, secondary near EOF) with the greater write time, since NILFS2 updates them lazily and the primary can lag the newest log; - follows its last partial segment to the super root (the last block of that segment), which embeds the DAT, cpfile and sufile inodes; - reads the DAT (disk address translation), a palloc file whose own block map uses physical pointers, to translate the virtual block numbers that every other inode's pointers carry; - reads the cpfile checkpoint for the last checkpoint number to get the ifile inode, reads the root inode from the ifile (also a palloc file), and walks the directory tree, rebuilding regular files, directories and symlinks. Block maps are NILFS2 bmaps: an inline "direct" map (up to 6 pointers) or a B-tree whose non-root nodes carry an 8-byte pad between header and keys. All reads go through the bounds-checked Reader, so a truncated or corrupt image yields a partial result rather than a crash, and a declared file size larger than the image is clamped to bound the allocation. Tests (tests/test_nilfs2_extract.py): a real empty mkfs.nilfs2 image is identified and extracted cleanly (always-on, no privilege), exercising the whole metadata chain against genuine bytes; a populated round-trip builds a known tree, mounts and copies it in, extracts and compares byte-for-byte (self-skips without the kernel module and mount privilege, since mkfs.nilfs2 has no populate mode); and mangled superblock/metadata inputs must not crash. Adds fuzz/fuzz_nilfs2.cpp over the parse-only read chain. --- CMakeLists.txt | 2 + README.md | 2 +- fuzz/build.sh | 2 +- fuzz/fuzz_nilfs2.cpp | 32 +++ moria.1 | 4 +- src/extract/manifest.cpp | 2 + src/extract/nilfs2.cpp | 481 +++++++++++++++++++++++++++++++++++ src/extract/nilfs2.hpp | 34 +++ tests/run.sh | 2 + tests/test_nilfs2_extract.py | 216 ++++++++++++++++ 10 files changed, 773 insertions(+), 4 deletions(-) create mode 100644 fuzz/fuzz_nilfs2.cpp create mode 100644 src/extract/nilfs2.cpp create mode 100644 src/extract/nilfs2.hpp create mode 100644 tests/test_nilfs2_extract.py diff --git a/CMakeLists.txt b/CMakeLists.txt index 3380b26..49a7db9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -92,6 +92,7 @@ add_executable(moria src/extract/android_sparse.cpp src/extract/ext.cpp src/extract/reiserfs.cpp + src/extract/nilfs2.cpp src/extract/jffs2.cpp src/extract/ubifs.cpp src/extract/tar.cpp @@ -256,6 +257,7 @@ if(Python3_Interpreter_FOUND) test_esp32_nvs test_legacy_fs test_reiserfs_extract + test_nilfs2_extract test_luks test_lzma test_jzlzma diff --git a/README.md b/README.md index e14734c..26c92fd 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ moria --help Unpacked in-process, no external tools and no sudo: -- **Filesystems:** SquashFS, ext2/3/4, F2FS, FAT12/16/32, exFAT, NTFS, HFS+/HFSX, XFS, btrfs, JFFS2, UBI/UBIFS, romfs, YAFFS2, cramfs, EROFS, ReiserFS +- **Filesystems:** SquashFS, ext2/3/4, F2FS, FAT12/16/32, exFAT, NTFS, HFS+/HFSX, XFS, btrfs, JFFS2, UBI/UBIFS, romfs, YAFFS2, cramfs, EROFS, ReiserFS, NILFS2 - **Archives and images:** ZIP, tar, cpio, MS-CAB (stored/MSZIP/LZX), CFBF compound files / Windows Installer `.msi`, ISO 9660, Android sparse, Android boot - **Kernels and wrappers:** U-Boot uImage, U-Boot FIT, Windows CE XIP ROM (nk.bin / .cos), standalone gzip / xz / zstd / lz4 streams, Ingenic jzlzma (hardware LZ77) - **Firmware packages:** RAE Systems / Honeywell RFP (section table; LZARI-decompresses each section); Artosyn OTRA (AR8030 / VT4 / VR04; SHA-256-verified, LZO1X-decompresses each partition, or exposes the raw flash body on flat images) diff --git a/fuzz/build.sh b/fuzz/build.sh index 0e7e39f..1017e16 100755 --- a/fuzz/build.sh +++ b/fuzz/build.sh @@ -11,7 +11,7 @@ mapfile -t SRCS < <(find src -name '*.cpp' ! -name 'main.cpp' | sort) # Xcode clang lacks the libFuzzer runtime; override with a clang that has it # (e.g. CXX=/opt/homebrew/opt/llvm/bin/clang++ on macOS). CXX="${CXX:-clang++}" -for target in fuzz_scan fuzz_esp32_part fuzz_esp32_nvs fuzz_lzx fuzz_reiserfs; do +for target in fuzz_scan fuzz_esp32_part fuzz_esp32_nvs fuzz_lzx fuzz_reiserfs fuzz_nilfs2; do "$CXX" -std=c++20 -O1 -g -fsanitize=fuzzer,address -I src -I third_party \ -DFT_FUZZ_SIGDIR="\"$PWD/signatures\"" \ "${SRCS[@]}" "fuzz/${target}.cpp" -o "$target" diff --git a/fuzz/fuzz_nilfs2.cpp b/fuzz/fuzz_nilfs2.cpp new file mode 100644 index 0000000..f8f3e22 --- /dev/null +++ b/fuzz/fuzz_nilfs2.cpp @@ -0,0 +1,32 @@ +// fuzz_nilfs2.cpp — libFuzzer entry point for the NILFS2 extractor. +// Feeds fuzzer bytes to the parse-only read chain (superblock selection, super +// root, DAT translation, cpfile, ifile, directory walk, block maps) with a +// partition at offset 0, and to the full scan pipeline so the identify/validator +// path also sees adversarial input (issue #64). +// +// Build via fuzz/build.sh. Run: ./fuzz_nilfs2 -max_total_time=60 +#include +#include +#include + +#include "extract/nilfs2.hpp" +#include "reader.hpp" +#include "scan.hpp" +#include "sigload.hpp" + +#ifndef FT_FUZZ_SIGDIR +#define FT_FUZZ_SIGDIR "signatures" +#endif + +extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { + static const ft::LoadResult sigs = ft::load_signatures(FT_FUZZ_SIGDIR); + ft::Reader reader(std::span(data, size)); + + // Direct: the whole read chain with the filesystem at offset 0. + ft::nilfs2_probe(reader, 0); + + // Via the pipeline: magic-driven identification on the raw bytes. + auto findings = ft::scan(reader, sigs.signatures); + (void)findings; + return 0; +} diff --git a/moria.1 b/moria.1 index 6d363b9..b38ca39 100644 --- a/moria.1 +++ b/moria.1 @@ -13,7 +13,7 @@ embedded formats (filesystems, containers, bootloaders, executables, compression, media) with byte offsets and confidence tiers. It is identification-first; with .B \-\-extract -it also unpacks SquashFS, ext2/3/4, F2FS, JFFS2, UBIFS, romfs, YAFFS2, cramfs, FAT, exFAT, NTFS, HFS+, XFS, btrfs, EROFS, ReiserFS, and ISO9660 filesystems, ZIP/tar/cpio/MS-CAB archives, CFBF compound files (Windows Installer .msi), U-Boot uImage/FIT containers, Windows CE XIP ROM images and registry hives, RAE Systems/Honeywell RFP firmware packages (LZARI-decompressing their sections), and Ingenic jzlzma-compressed kernel/rootfs images (hardware LZ77) +it also unpacks SquashFS, ext2/3/4, F2FS, JFFS2, UBIFS, romfs, YAFFS2, cramfs, FAT, exFAT, NTFS, HFS+, XFS, btrfs, EROFS, ReiserFS, NILFS2, and ISO9660 filesystems, ZIP/tar/cpio/MS-CAB archives, CFBF compound files (Windows Installer .msi), U-Boot uImage/FIT containers, Windows CE XIP ROM images and registry hives, RAE Systems/Honeywell RFP firmware packages (LZARI-decompressing their sections), and Ingenic jzlzma-compressed kernel/rootfs images (hardware LZ77) internally (no sudo). Every other format is still identified and located by offset, ready for another tool such as binwalk. .PP Output is human-readable by default. A single file shows a findings tree @@ -55,7 +55,7 @@ List tar/cpio/zip/MS-CAB member names and sizes, or a Windows CE ROM's modules a .SS Extract .TP .BR \-e ", " \-\-extract -Unpack identified SquashFS, ext2/3/4, F2FS, JFFS2, UBIFS (incl. UBI), romfs, YAFFS2, cramfs, FAT, exFAT, NTFS, EROFS, ReiserFS, and ISO9660 filesystems, ZIP/tar/cpio/MS-CAB archives, CFBF compound files such as Windows Installer .msi packages, U-Boot uImage/FIT containers, Android boot/sparse images, Windows CE XIP ROM images, Windows CE registry hives, and RAE Systems/Honeywell RFP firmware packages into +Unpack identified SquashFS, ext2/3/4, F2FS, JFFS2, UBIFS (incl. UBI), romfs, YAFFS2, cramfs, FAT, exFAT, NTFS, EROFS, ReiserFS, NILFS2, and ISO9660 filesystems, ZIP/tar/cpio/MS-CAB archives, CFBF compound files such as Windows Installer .msi packages, U-Boot uImage/FIT containers, Android boot/sparse images, Windows CE XIP ROM images, Windows CE registry hives, and RAE Systems/Honeywell RFP firmware packages into .IR file .extracted/ (one subdir per region, named .IR 0x\- ). diff --git a/src/extract/manifest.cpp b/src/extract/manifest.cpp index c9972c8..92fee0e 100644 --- a/src/extract/manifest.cpp +++ b/src/extract/manifest.cpp @@ -24,6 +24,7 @@ #include "extract/fat.hpp" #include "extract/iso9660.hpp" #include "extract/jffs2.hpp" +#include "extract/nilfs2.hpp" #include "extract/ntfs.hpp" #include "extract/otra.hpp" #include "extract/rae_rfp.hpp" @@ -58,6 +59,7 @@ Extractor find_extractor(const std::string& type) { if (type == "romfs") return extract_romfs; if (type == "cramfs") return extract_cramfs; if (type == "reiserfs") return extract_reiserfs; + if (type == "nilfs2") return extract_nilfs2; if (type == "erofs") return extract_erofs; if (type == "f2fs") return extract_f2fs; if (type == "android_sparse") return extract_android_sparse; diff --git a/src/extract/nilfs2.cpp b/src/extract/nilfs2.cpp new file mode 100644 index 0000000..672eeca --- /dev/null +++ b/src/extract/nilfs2.cpp @@ -0,0 +1,481 @@ +// nilfs2.cpp — NILFS2 read-only extraction. See nilfs2.hpp. +// +// Format reference: the kernel on-disk header (uapi/linux/nilfs2_ondisk.h) and +// fs/nilfs2 (bmap/btree/alloc/cpfile). Everything is little-endian. +// +// Read chain (validated against a real mkfs.nilfs2 image): +// 1. Two superblocks (primary at 1024, secondary near EOF). Pick the valid one +// with the greater s_wtime; it names the last partial segment and the last +// checkpoint number. +// 2. The super root is the last block of that partial segment; it embeds the +// DAT, cpfile and sufile inodes directly. +// 3. DAT (disk address translation) is a palloc file whose own block map uses +// physical pointers. dat_translate(vblocknr) reads DAT entry vblocknr and +// returns its physical block. +// 4. cpfile is an mdt file (header + checkpoint array); checkpoint s_last_cno +// embeds the ifile inode. +// 5. ifile is a palloc file of inodes; read inode NILFS_ROOT_INO (2) and walk. +// +// Block mapping (nilfs_bmap): the inode's 56-byte i_bmap holds either a "direct" +// map (header + up to 6 pointers for file blocks 0..5) or a B-tree root (flag +// NILFS_BMAP_LARGE). B-tree non-root nodes carry an 8-byte pad between header and +// keys. Pointers are physical for the DAT inode and virtual (DAT-translated) for +// every other inode. +#include "extract/nilfs2.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "extract/safepath.hpp" + +namespace ft { + +namespace { + +constexpr uint16_t NILFS_MAGIC = 0x3434; +constexpr uint64_t SB_OFFSET = 1024; +constexpr uint32_t ROOT_INO = 2; +constexpr uint32_t SEGSUM_MAGIC = 0x1eaffa11; + +// inode layout +constexpr uint64_t I_SIZE_OFF = 8; +constexpr uint64_t I_MODE_OFF = 48; +constexpr uint64_t I_BMAP_OFF = 56; +constexpr uint64_t I_BMAP_BYTES = 56; // NILFS_BMAP_SIZE (7 * 8) + +// bmap +constexpr uint8_t BMAP_LARGE = 0x1; // i_bmap[0] low byte: B-tree if set +constexpr unsigned DIRECT_NBLOCKS = 6; // direct keys 0..5 at i_bmap[1..6] +constexpr uint64_t BT_NODE_HDR = 8; // struct nilfs_btree_node +constexpr uint64_t BT_EXTRA_PAD = 8; // non-root key pad +constexpr int BT_ROOT_NCMAX = 3; // (56 - 8) / 16 + +// file mode bits +constexpr uint16_t S_IFMT = 0xF000; +constexpr uint16_t S_IFREG = 0x8000; +constexpr uint16_t S_IFDIR = 0x4000; +constexpr uint16_t S_IFLNK = 0xA000; + +// cpfile header is 32 bytes; checkpoint.cp_ifile_inode sits at offset 64. +constexpr uint64_t CPFILE_HDR = 32; +constexpr uint64_t CP_IFILE_INODE_OFF = 64; + +// Guardrails. +constexpr size_t MAX_FILES = 2000000; +constexpr size_t MAX_DEPTH = 128; +constexpr size_t MAX_BTREE_NODES = 1u << 20; +constexpr size_t MAX_BLOCKS_PER_FILE = 1u << 22; +constexpr uint64_t MAX_FILE_BYTES = uint64_t(8) << 30; + +std::optional rd8(const Reader& r, uint64_t o) { + return r.at(static_cast(o), Endian::Little); +} +std::optional rd16(const Reader& r, uint64_t o) { + return r.at(static_cast(o), Endian::Little); +} +std::optional rd32(const Reader& r, uint64_t o) { + return r.at(static_cast(o), Endian::Little); +} +std::optional rd64(const Reader& r, uint64_t o) { + return r.at(static_cast(o), Endian::Little); +} + +struct Ctx { + const Reader& r; + uint64_t base = 0; // absolute file offset of block 0 + uint64_t bs = 0; + uint16_t inode_size = 0; + uint16_t dat_entry_size = 0; + uint16_t checkpoint_size = 0; + uint64_t block_count = 0; // device size in blocks (bound for ptrs) + int bt_node_ncmax = 0; // B-tree children per full block node + SafeRoot* root = nullptr; + std::string subdir; + Extracted& out; + bool truncated = false; + size_t files_seen = 0; + // DAT file-block -> physical block, and ifile file-block -> virtual block. + std::unordered_map dat_map{}; + std::unordered_map ifile_map{}; + std::unordered_set dir_stack{}; // inode recursion cycle guard +}; + +uint64_t blk_off(const Ctx& c, uint64_t b) { return c.base + b * c.bs; } + +// Collect a bmap into {file_block -> pointer}. For the DAT inode pointers are +// physical and `translate` is identity; for other inodes pointers are virtual +// and `translate` maps a virtual block to a physical one (to descend B-tree +// index nodes). Leaf pointers are stored raw (the caller translates them). +template +void collect_bmap(Ctx& c, uint64_t inode_off, Translate translate, + std::unordered_map& out) { + const uint64_t bmap = inode_off + I_BMAP_OFF; + auto flags = rd8(c.r, bmap); + if (!flags) { c.truncated = true; return; } + + if ((*flags & BMAP_LARGE) == 0) { + // Direct: pointer for key k at i_bmap[1 + k]. + for (unsigned k = 0; k < DIRECT_NBLOCKS; ++k) { + auto p = rd64(c.r, bmap + 8 + uint64_t(k) * 8); + if (p && *p != 0) out[k] = *p; + } + return; + } + + // B-tree. Root node is inline in i_bmap (no pad); block nodes carry an + // 8-byte pad between header and keys. + size_t nodes = 0; + struct Frame { uint64_t off; bool is_root; int level; int ncmax; }; + std::vector stack; + auto level = rd8(c.r, bmap + 1); + if (!level || *level == 0) { c.truncated = true; return; } + stack.push_back({bmap, true, *level, BT_ROOT_NCMAX}); + + while (!stack.empty()) { + Frame f = stack.back(); + stack.pop_back(); + if (++nodes > MAX_BTREE_NODES || out.size() > MAX_BLOCKS_PER_FILE) { + c.truncated = true; + return; + } + auto nch = rd16(c.r, f.off + 2); + if (!nch) { c.truncated = true; continue; } + const uint64_t dkeys = f.off + BT_NODE_HDR + (f.is_root ? 0 : BT_EXTRA_PAD); + const uint64_t dptrs = dkeys + uint64_t(f.ncmax) * 8; + int n = *nch; + if (f.is_root && n > BT_ROOT_NCMAX) { n = BT_ROOT_NCMAX; c.truncated = true; } + if (!f.is_root && n > f.ncmax) { n = f.ncmax; c.truncated = true; } + for (int i = 0; i < n; ++i) { + auto key = rd64(c.r, dkeys + uint64_t(i) * 8); + auto ptr = rd64(c.r, dptrs + uint64_t(i) * 8); + if (!key || !ptr) { c.truncated = true; break; } + if (f.level == 1) { + out[*key] = *ptr; // leaf: file block -> data block pointer + } else { + uint64_t child = translate(*ptr); + if (child == 0 || child >= c.block_count) { c.truncated = true; continue; } + stack.push_back({blk_off(c, child), false, f.level - 1, c.bt_node_ncmax}); + } + } + } +} + +// palloc: byte position of entry `nr` (entry size `es`) within its file -> the +// file block offset and the byte offset inside that block. +void palloc_pos(const Ctx& c, uint64_t nr, uint64_t es, uint64_t& fblk, uint64_t& boff) { + const uint64_t G = c.bs * 8; // entries per group (one bitmap block) + const uint64_t P = c.bs / es; // entries per block + const uint64_t D = c.bs / 4; // groups per descriptor block + const uint64_t bpg = (G + P - 1) / P + 1; // entry blocks + 1 bitmap + const uint64_t bpd = D * bpg + 1; // + 1 descriptor block + const uint64_t group = nr / G; + const uint64_t off = nr % G; + const uint64_t region = (group / D) * bpd; + const uint64_t bitmap = region + 1 + (group % D) * bpg; + fblk = bitmap + 1 + off / P; + boff = (off % P) * es; +} + +// Translate a virtual block number to a physical block via the DAT. +uint64_t dat_translate(Ctx& c, uint64_t vblk) { + uint64_t fblk, boff; + palloc_pos(c, vblk, c.dat_entry_size, fblk, boff); + auto it = c.dat_map.find(fblk); + if (it == c.dat_map.end()) { c.truncated = true; return 0; } + auto de_blocknr = rd64(c.r, blk_off(c, it->second) + boff); // nilfs_dat_entry.de_blocknr @0 + return de_blocknr ? *de_blocknr : 0; +} + +// Physical block byte offset for file block `fblk` of a virtual-pointer file. +std::optional meta_block(Ctx& c, const std::unordered_map& map, + uint64_t fblk) { + auto it = map.find(fblk); + if (it == map.end()) return std::nullopt; + uint64_t phys = dat_translate(c, it->second); + if (phys == 0 || phys >= c.block_count) { c.truncated = true; return std::nullopt; } + return blk_off(c, phys); +} + +// Byte offset of inode `ino` within the ifile. +std::optional read_inode(Ctx& c, uint64_t ino) { + uint64_t fblk, boff; + palloc_pos(c, ino, c.inode_size, fblk, boff); + auto b = meta_block(c, c.ifile_map, fblk); + if (!b) return std::nullopt; + return *b + boff; +} + +// Read the full content of a regular file / symlink target into `out`. +void read_content(Ctx& c, uint64_t inode_off, uint64_t size, std::vector& out) { + if (size > MAX_FILE_BYTES) { size = MAX_FILE_BYTES; c.truncated = true; } + if (size > c.r.size()) { size = c.r.size(); c.truncated = true; } + out.assign(static_cast(size), 0); + std::unordered_map map; + collect_bmap(c, inode_off, [&](uint64_t v) { return dat_translate(c, v); }, map); + for (const auto& kv : map) { + const uint64_t fblk = kv.first; + const uint64_t start = fblk * c.bs; + if (start >= size) continue; + uint64_t phys = dat_translate(c, kv.second); + if (phys == 0 || phys >= c.block_count) { c.truncated = true; continue; } + const uint64_t want = std::min(c.bs, size - start); + auto span = c.r.bytes(static_cast(blk_off(c, phys)), static_cast(want)); + if (!span) { c.truncated = true; continue; } + std::memcpy(out.data() + static_cast(start), span->data(), + static_cast(want)); + } +} + +struct DirEnt { + uint64_t ino; + std::string name; +}; + +// Parse the ext2-like directory of an inode into entries. +std::vector read_dir(Ctx& c, uint64_t inode_off, uint64_t size) { + std::vector ents; + std::unordered_map map; + collect_bmap(c, inode_off, [&](uint64_t v) { return dat_translate(c, v); }, map); + std::vector fblocks; + for (const auto& kv : map) fblocks.push_back(kv.first); + std::sort(fblocks.begin(), fblocks.end()); + for (uint64_t fblk : fblocks) { + if (fblk * c.bs >= size) break; + uint64_t phys = dat_translate(c, map[fblk]); + if (phys == 0 || phys >= c.block_count) { c.truncated = true; continue; } + const uint64_t base = blk_off(c, phys); + uint64_t pos = 0; + while (pos + 12 <= c.bs) { + auto ino = rd64(c.r, base + pos); + auto rec_len = rd16(c.r, base + pos + 8); + auto name_len = rd8(c.r, base + pos + 10); + if (!ino || !rec_len || !name_len) break; + if (*rec_len < 12) break; + if (*ino != 0 && *name_len > 0 && pos + 12 + *name_len <= c.bs) { + auto nm = c.r.bytes(static_cast(base + pos + 12), *name_len); + if (nm) ents.push_back({*ino, std::string(reinterpret_cast(nm->data()), + nm->size())}); + } + pos += *rec_len; + } + } + return ents; +} + +void walk(Ctx& c, uint64_t ino, const std::string& rel, size_t depth) { + if (depth > MAX_DEPTH) { c.truncated = true; return; } + if (c.files_seen++ > MAX_FILES) { c.truncated = true; return; } + if (!c.dir_stack.insert(ino).second) return; // directory cycle + + auto io = read_inode(c, ino); + if (!io) { c.truncated = true; c.dir_stack.erase(ino); return; } + auto dsize = rd64(c.r, *io + I_SIZE_OFF); + if (!dsize) { c.truncated = true; c.dir_stack.erase(ino); return; } + + for (const DirEnt& e : read_dir(c, *io, *dsize)) { + if (e.name.empty() || e.name == "." || e.name == "..") continue; + if (e.name.find('/') != std::string::npos || e.name.find('\0') != std::string::npos) + continue; + auto cio = read_inode(c, e.ino); + if (!cio) { c.truncated = true; continue; } + auto mode = rd16(c.r, *cio + I_MODE_OFF); + auto size = rd64(c.r, *cio + I_SIZE_OFF); + if (!mode || !size) { c.truncated = true; continue; } + const std::string child_rel = rel.empty() ? e.name : rel + "/" + e.name; + const std::string full = c.subdir + "/" + child_rel; + const uint16_t type = *mode & S_IFMT; + + if (type == S_IFDIR) { + if (c.root && c.root->make_dir(full)) c.out.dirs++; + walk(c, e.ino, child_rel, depth + 1); + } else if (type == S_IFLNK) { + std::vector buf; + read_content(c, *cio, std::min(*size, 4096), buf); + std::string target(buf.begin(), buf.end()); + if (auto z = target.find('\0'); z != std::string::npos) target.resize(z); + if (!target.empty() && c.root && c.root->make_symlink(full, target)) c.out.symlinks++; + } else if (type == S_IFREG) { + std::vector data; + read_content(c, *cio, *size, data); + if (c.root && c.root->write_file(full, data, *mode & 0777)) { + c.out.files++; + c.out.bytes += data.size(); + } else if (c.root) { + c.out.warnings.push_back("write failed: " + child_rel); + } + } + } + c.dir_stack.erase(ino); +} + +// Fill geometry from a superblock at absolute offset `sbo`; false if implausible. +bool parse_super(const Reader& r, uint64_t sbo, Ctx& c, uint64_t& last_cno, uint64_t& last_pseg) { + auto magic = rd16(r, sbo + 6); + auto logbs = rd32(r, sbo + 0x14); + auto devsize = rd64(r, sbo + 0x20); + auto lcno = rd64(r, sbo + 0x38); + auto lpseg = rd64(r, sbo + 0x40); + auto isize = rd16(r, sbo + 0x90); + auto dsz = rd16(r, sbo + 0x92); + auto cpsz = rd16(r, sbo + 0x94); + if (!magic || *magic != NILFS_MAGIC || !logbs || !devsize || !lcno || !lpseg || !isize || + !dsz || !cpsz) + return false; + if (*logbs > 6) return false; // 1 KiB .. 64 KiB + c.bs = uint64_t(1) << (*logbs + 10); + if (*isize < 128 || *isize > c.bs) return false; + if (*dsz < 32 || *cpsz < 64) return false; + c.inode_size = *isize; + c.dat_entry_size = *dsz; + c.checkpoint_size = *cpsz; + c.block_count = *devsize / c.bs; + c.bt_node_ncmax = static_cast((c.bs - BT_NODE_HDR - BT_EXTRA_PAD) / 16); + last_cno = *lcno; + last_pseg = *lpseg; + return c.block_count > 0; +} + +} // namespace + +bool extract_nilfs2(const Reader& r, const Finding& f, SafeRoot& root, const std::string& subdir, + Extracted& out) { + out.offset = f.offset; + out.type = "nilfs2"; + out.root = subdir; + + Ctx c{r, f.offset, 0, 0, 0, 0, 0, 0, &root, subdir, out}; + + // Pick the valid superblock (primary at 1024, secondary near EOF) with the + // greater write time; NILFS2 updates them lazily, so the primary can lag. + uint64_t last_cno = 0, last_pseg = 0; + uint64_t chosen = SIZE_MAX; + uint64_t best_wtime = 0; + { + Ctx probe{r, f.offset, 0, 0, 0, 0, 0, 0, nullptr, "", out}; + uint64_t cno, pseg; + for (int which = 0; which < 2; ++which) { + uint64_t sbo; + if (which == 0) { + sbo = f.offset + SB_OFFSET; + } else { + auto devsize = rd64(r, f.offset + SB_OFFSET + 0x20); + if (!devsize || *devsize < 8192) continue; + sbo = f.offset + (((*devsize >> 12) - 1) << 12); + } + Ctx t{r, f.offset, 0, 0, 0, 0, 0, 0, nullptr, "", out}; + if (!parse_super(r, sbo, t, cno, pseg)) continue; + auto wt = rd64(r, sbo + 0x68); // s_wtime + uint64_t w = wt ? *wt : 0; + if (chosen == SIZE_MAX || w >= best_wtime) { + best_wtime = w; + chosen = sbo; + } + } + (void)probe; + } + if (chosen == SIZE_MAX || !parse_super(r, chosen, c, last_cno, last_pseg)) { + out.status = "error:bad-superblock"; + return true; + } + + // Super root is the last block of the last partial segment. + const uint64_t ss = blk_off(c, last_pseg); + auto ssmagic = rd32(r, ss + 8); + auto ss_nblocks = rd32(r, ss + 0x28); + if (!ssmagic || *ssmagic != SEGSUM_MAGIC || !ss_nblocks || *ss_nblocks == 0) { + out.status = "error:segment-summary"; + return true; + } + const uint64_t sr = blk_off(c, last_pseg + *ss_nblocks - 1); + // super_root: sr_sum(4) sr_bytes(2) sr_flags(2) sr_nongc_ctime(8) then inodes. + const uint64_t dat_inode = sr + 16; + const uint64_t cpfile_inode = sr + 16 + c.inode_size; + + if (!root.make_dir(subdir)) { + out.status = "error:mkdir"; + return true; + } + + // DAT block map (physical pointers): the bootstrap for all translation. + collect_bmap(c, dat_inode, [](uint64_t p) { return p; }, c.dat_map); + if (c.dat_map.empty()) { + out.status = "error:dat"; + return true; + } + + // cpfile: locate checkpoint `last_cno` and read its embedded ifile inode. + std::unordered_map cpfile_map; + collect_bmap(c, cpfile_inode, [&](uint64_t v) { return dat_translate(c, v); }, cpfile_map); + const uint64_t cps = c.checkpoint_size; + const uint64_t first = (CPFILE_HDR + cps - 1) / cps; // mi_first_entry_offset + const uint64_t per_blk = c.bs / cps; + if (per_blk == 0) { out.status = "error:cpfile"; return true; } + const uint64_t tcno = last_cno + first - 1; + auto cp_block = meta_block(c, cpfile_map, tcno / per_blk); + if (!cp_block) { out.status = "error:cpfile"; return true; } + const uint64_t ifile_inode = *cp_block + (tcno % per_blk) * cps + CP_IFILE_INODE_OFF; + + // ifile block map, then walk from the root inode. + collect_bmap(c, ifile_inode, [&](uint64_t v) { return dat_translate(c, v); }, c.ifile_map); + if (c.ifile_map.empty()) { out.status = "error:ifile"; return true; } + + walk(c, ROOT_INO, "", 0); + out.status = c.truncated ? "partial" : "ok"; + return true; +} + +size_t nilfs2_probe(const Reader& r, size_t base) { + Extracted out; + Ctx c{r, base, 0, 0, 0, 0, 0, 0, nullptr, "", out}; + uint64_t last_cno = 0, last_pseg = 0; + uint64_t chosen = SIZE_MAX, best = 0; + for (int which = 0; which < 2; ++which) { + uint64_t sbo; + if (which == 0) { + sbo = base + SB_OFFSET; + } else { + auto devsize = rd64(r, base + SB_OFFSET + 0x20); + if (!devsize || *devsize < 8192) continue; + sbo = base + (((*devsize >> 12) - 1) << 12); + } + Ctx t{r, base, 0, 0, 0, 0, 0, 0, nullptr, "", out}; + uint64_t cno, pseg; + if (!parse_super(r, sbo, t, cno, pseg)) continue; + auto wt = rd64(r, sbo + 0x68); + if (chosen == SIZE_MAX || (wt && *wt >= best)) { best = wt ? *wt : 0; chosen = sbo; } + } + if (chosen == SIZE_MAX || !parse_super(r, chosen, c, last_cno, last_pseg)) return 0; + + const uint64_t ss = blk_off(c, last_pseg); + auto ssmagic = rd32(r, ss + 8); + auto ss_nblocks = rd32(r, ss + 0x28); + if (!ssmagic || *ssmagic != SEGSUM_MAGIC || !ss_nblocks || *ss_nblocks == 0) return 0; + const uint64_t sr = blk_off(c, last_pseg + *ss_nblocks - 1); + const uint64_t dat_inode = sr + 16; + const uint64_t cpfile_inode = sr + 16 + c.inode_size; + + collect_bmap(c, dat_inode, [](uint64_t p) { return p; }, c.dat_map); + if (c.dat_map.empty()) return 0; + + std::unordered_map cpfile_map; + collect_bmap(c, cpfile_inode, [&](uint64_t v) { return dat_translate(c, v); }, cpfile_map); + const uint64_t cps = c.checkpoint_size; + const uint64_t per_blk = c.bs / cps; + if (per_blk == 0) return 0; + const uint64_t tcno = last_cno + (CPFILE_HDR + cps - 1) / cps - 1; + auto cp_block = meta_block(c, cpfile_map, tcno / per_blk); + if (!cp_block) return 0; + const uint64_t ifile_inode = *cp_block + (tcno % per_blk) * cps + CP_IFILE_INODE_OFF; + collect_bmap(c, ifile_inode, [&](uint64_t v) { return dat_translate(c, v); }, c.ifile_map); + + walk(c, ROOT_INO, "", 0); + return c.dat_map.size() + c.ifile_map.size(); +} + +} // namespace ft diff --git a/src/extract/nilfs2.hpp b/src/extract/nilfs2.hpp new file mode 100644 index 0000000..69c7bee --- /dev/null +++ b/src/extract/nilfs2.hpp @@ -0,0 +1,34 @@ +// nilfs2.hpp — NILFS2 (log-structured) read-only extractor. +// +// NILFS2 is a log-structured filesystem: the live tree is reached through the +// latest checkpoint, not a fixed inode table. This extractor picks the newer of +// the two superblocks, follows its last partial segment to the super root, reads +// the DAT (disk address translation), cpfile (checkpoints) and, via the last +// checkpoint, the ifile (inode file); then it walks the root directory and +// rebuilds regular files, directories and symlinks under a SafeRoot. Block +// mappings are NILFS2 bmaps (inline "direct" or a B-tree); non-DAT pointers are +// virtual block numbers resolved through the DAT. No GC, no roll-forward +// recovery, no write support, no mount (unlike the kernel driver). Every read +// goes through the bounds-checked Reader, so a truncated or corrupt image yields +// a partial result, never an out-of-bounds access. +#pragma once + +#include + +#include "extract/manifest.hpp" +#include "finding.hpp" +#include "reader.hpp" + +namespace ft { + +class SafeRoot; + +bool extract_nilfs2(const Reader& r, const Finding& f, SafeRoot& root, const std::string& subdir, + Extracted& out); + +// Parse-only driver for fuzzing (fuzz/fuzz_nilfs2.cpp): runs the whole read chain +// (superblock, super root, DAT, cpfile, ifile, directory walk, file/content +// block mapping) with no filesystem writes. Every read is bounds-checked. +size_t nilfs2_probe(const Reader& r, size_t base); + +} // namespace ft diff --git a/tests/run.sh b/tests/run.sh index fab8013..6c006c3 100755 --- a/tests/run.sh +++ b/tests/run.sh @@ -36,6 +36,8 @@ python3 tests/test_esp32_nvs.py python3 tests/test_legacy_fs.py # ReiserFS extraction: synthetic 3.5/3.6 round-trip + real empty mkreiserfs image. python3 tests/test_reiserfs_extract.py +# NILFS2 extraction: empty real image + populated round-trip (self-skips without privilege). +python3 tests/test_nilfs2_extract.py # LUKS1/LUKS2 identify with header parameters (synthetic + real cryptsetup). python3 tests/test_luks.py # Legacy standalone .lzma identify (verified via trial-decode) + byte-exact extract + FP guards. diff --git a/tests/test_nilfs2_extract.py b/tests/test_nilfs2_extract.py new file mode 100644 index 0000000..3dbc754 --- /dev/null +++ b/tests/test_nilfs2_extract.py @@ -0,0 +1,216 @@ +#!/usr/bin/env python3 +"""NILFS2 extraction regression (issue #64). + +NILFS2 is log-structured: the live tree is reached through the latest +checkpoint. Layers: + + 1. Real empty image (always-on if mkfs.nilfs2 is present, no privilege): + mkfs.nilfs2 builds an empty filesystem; moria must identify and extract it + cleanly (just the root directory). This exercises the whole metadata chain + against genuine bytes: superblock selection, super root, DAT translation, + cpfile checkpoint lookup, ifile inode read, root-directory walk. + 2. Real populated round-trip (self-skips unless the image can be mounted, i.e. + the nilfs2 module is loaded and the test runs with mount privilege): build a + known tree, mount + copy it in, extract with moria, compare byte-for-byte. + This is the only layer that exercises file content (direct + B-tree block + maps, holes) and symlinks, so it is the primary correctness check where it + can run. mkfs.nilfs2 has no populate-from-directory mode and the kernel + driver is required to write, so an unprivileged populate is not possible. + 3. Corruption robustness: mangled superblock/metadata must never crash and must + report a sane status (the ASan build is the out-of-bounds oracle). + +Run: python3 tests/test_nilfs2_extract.py [path/to/moria] +""" +import hashlib +import json +import os +import shutil +import subprocess +import sys +import tempfile + +HERE = os.path.dirname(os.path.abspath(__file__)) +MORIA = sys.argv[1] if len(sys.argv) > 1 else os.path.join(HERE, "..", "build", "moria") + + +def tree_manifest(root): + out = {} + for dirpath, dirnames, filenames in os.walk(root): + for name in list(dirnames) + list(filenames): + full = os.path.join(dirpath, name) + rel = os.path.relpath(full, root) + if os.path.islink(full): + out[rel] = ("link", os.readlink(full)) + elif os.path.isdir(full): + out[rel] = ("dir",) + else: + with open(full, "rb") as f: + out[rel] = ("file", hashlib.sha256(f.read()).hexdigest()) + return out + + +# mkfs.nilfs2 needs >= 2 MiB. Keep images small: a corrupt superblock fails the +# CRC check, so moria falls back to a full broad scan whose cost scales with the +# image size (and is heavily amplified under ASan), so corruption cases in +# particular must stay small to run quickly. +def mkfs_empty(path, size_mb=4): + with open(path, "wb") as f: + f.truncate(size_mb << 20) + r = subprocess.run(["mkfs.nilfs2", "-q", "-B", "16", path], capture_output=True) + return r.returncode == 0 + + +def extract(outdir, img): + r = subprocess.run([MORIA, "-j", "--extract", "-C", outdir, img], capture_output=True) + if r.returncode != 0: + return None, r + try: + return json.loads(r.stdout.decode())["extraction"]["extracted"][0], r + except Exception: + return None, r + + +def test_real_empty(work): + if not shutil.which("mkfs.nilfs2"): + print("SKIP [empty]: mkfs.nilfs2 (nilfs-utils) not installed") + return 0 + img = os.path.join(work, "empty.img") + if not mkfs_empty(img): + print("SKIP [empty]: mkfs.nilfs2 failed") + return 0 + outdir = os.path.join(work, "empty.out") + entry, r = extract(outdir, img) + if entry is None: + print(f"FAIL [empty]: not identified/extracted\n{r.stdout[:200]}") + return 1 + if entry["type"] == "nilfs2" and entry["status"] == "ok" and entry["files"] == 0: + print("PASS [empty]: empty image identified + extracted clean (status ok)") + return 0 + print(f"FAIL [empty]: type={entry['type']} status={entry['status']} files={entry['files']}") + return 1 + + +def build_tree(src): + os.makedirs(os.path.join(src, "etc")) + os.makedirs(os.path.join(src, "usr", "bin")) + os.makedirs(os.path.join(src, "empty")) + with open(os.path.join(src, "etc", "hostname"), "wb") as f: + f.write(b"nilfs2-roundtrip\n") + with open(os.path.join(src, "etc", "zeros.dat"), "wb") as f: + f.write(b"\x00" * (300 * 1024)) # multi-block, holes-friendly + with open(os.path.join(src, "usr", "bin", "rand.bin"), "wb") as f: + f.write(os.urandom(400 * 1024)) # B-tree block map + with open(os.path.join(src, "small.txt"), "wb") as f: + f.write(b"tiny\n") # direct block map + p = os.path.join(src, "usr", "bin", "run.sh") + with open(p, "wb") as f: + f.write(b"#!/bin/sh\necho hi\n") + os.chmod(p, 0o755) + os.symlink("hostname", os.path.join(src, "etc", "hn.lnk")) + os.symlink("../etc/hostname", os.path.join(src, "usr", "bin", "hn2.lnk")) + + +def test_roundtrip(work): + """Needs mkfs.nilfs2 + the ability to mount (module + privilege).""" + if not shutil.which("mkfs.nilfs2") or not shutil.which("mount"): + print("SKIP [roundtrip]: mkfs.nilfs2 / mount not available") + return 0 + img = os.path.join(work, "rt.img") + src = os.path.join(work, "rt.src") + mnt = os.path.join(work, "rt.mnt") + os.makedirs(mnt) + build_tree(src) + if not mkfs_empty(img, size_mb=32): # room for ~700 KiB of files + nilfs overhead + print("SKIP [roundtrip]: mkfs.nilfs2 failed") + return 0 + m = subprocess.run(["mount", "-t", "nilfs2", "-o", "loop", img, mnt], capture_output=True) + if m.returncode != 0: + print("SKIP [roundtrip]: cannot mount nilfs2 (needs module + privilege)") + return 0 + try: + subprocess.run(["cp", "-a", src + "/.", mnt + "/"], check=False) + subprocess.run(["sync"]) + finally: + subprocess.run(["umount", mnt], capture_output=True) + expected = tree_manifest(src) + outdir = os.path.join(work, "rt.out") + entry, r = extract(outdir, img) + if entry is None: + print(f"FAIL [roundtrip]: no manifest\n{r.stdout[:200]}") + return 1 + if entry["status"] not in ("ok", "partial"): + print(f"FAIL [roundtrip]: status={entry['status']}") + return 1 + got = tree_manifest(os.path.join(outdir, entry["root"])) + if got == expected: + print(f"PASS [roundtrip]: {entry['files']} files, {entry['dirs']} dirs, " + f"{entry['symlinks']} symlinks, round-trips byte-for-byte") + return 0 + only_exp = set(expected) - set(got) + only_got = set(got) - set(expected) + mism = {k for k in expected.keys() & got.keys() if expected[k] != got[k]} + print(f"FAIL [roundtrip]: missing={sorted(only_exp)} extra={sorted(only_got)} " + f"mismatched={sorted(mism)}") + return 1 + + +def test_corruption(work): + if not shutil.which("mkfs.nilfs2"): + print("SKIP [corrupt]: mkfs.nilfs2 not installed") + return 0 + img = os.path.join(work, "corrupt.img") + if not mkfs_empty(img): + print("SKIP [corrupt]: mkfs.nilfs2 failed") + return 0 + base = bytearray(open(img, "rb").read()) + cases = {} + t = bytearray(base); t = t[: len(t) // 3]; cases["truncated"] = t # cut the device + z = bytearray(base) + z[1024 + 0x40: 1024 + 0x48] = (0xFFFFFFFFFFFFFFFF).to_bytes(8, "little") # bad last_pseg + cases["bad-last-pseg"] = z + g = bytearray(base) + for i in range(0x10000, 0x10000 + 0x8000): # garble early blocks + g[i] = (i * 29) & 0xFF + cases["garbled-tree"] = g + failures = 0 + for label, data in cases.items(): + p = os.path.join(work, f"c_{label}.img") + with open(p, "wb") as f: + f.write(bytes(data)) + outdir = os.path.join(work, f"c_{label}.out") + r = subprocess.run([MORIA, "-j", "--extract", "-C", outdir, p], + capture_output=True, timeout=60) + if r.returncode != 0: + print(f"FAIL [corrupt {label}]: moria exit {r.returncode} (crash/sanitizer?)\n" + f"{r.stderr.decode()[-600:]}") + failures += 1 + continue + try: + e = json.loads(r.stdout.decode()).get("extraction", {}).get("extracted", [{}])[0] + status = e.get("status", "") + except Exception: + status = "" + ok = status in ("", "ok", "partial") or status.startswith("error:") + print(f"{'PASS' if ok else 'FAIL'} [corrupt {label}]: no crash, status={status!r}") + failures += 0 if ok else 1 + return failures + + +def main(): + if not os.path.exists(MORIA): + print("moria not built", file=sys.stderr) + return 1 + with tempfile.TemporaryDirectory() as work: + failures = 0 + failures += test_real_empty(work) + failures += test_roundtrip(work) + failures += test_corruption(work) + if failures: + print(f"\n{failures} nilfs2 extraction check(s) failed") + return 1 + print("\nall nilfs2 extraction checks passed") + return 0 + + +if __name__ == "__main__": + sys.exit(main())