win: lay mods and split data over a RomFS image

win/build_app.py

@@ -69,6 +69,30 @@ def host_sources(game_h_dir):
 IMPORT = re.compile(r'^\s*#\s*(?:import|include)\s*<(\w+)/', re.M)
 
 
+def unpack_romfs_image(img, dst):
+    """A DMROMFS1 image as a RomFS folder, for the steps that need one (mods' overlay, split data); reused while the image is unchanged"""
+    img = os.fspath(img); st = os.stat(img); stamp = Path(dst).with_name(Path(dst).name + '.stamp'); want = f'{st.st_size} {st.st_mtime_ns}'
+    if Path(dst).is_dir() and stamp.is_file() and stamp.read_text() == want: return Path(dst)
+    t(f'unpacking {img} into a folder ({st.st_size / 2**30:.1f} GB) for the mods / split data'); shutil.rmtree(dst, ignore_errors=True); stamp.unlink(missing_ok=True)
+    m = rfs.Image(img); n = 0
+    with open(img, 'rb') as f:
+        todo = [(0, Path(dst))]
+        while todo:
+            k, d = todo.pop(); d.mkdir(parents=True, exist_ok=True)
+            for name, j in m.children(k):
+                off, size, _, _, isfile, _ = m.entry(j)
+                if not isfile: todo.append((j, d / name)); continue
+                f.seek(off); left = size
+                with open(d / name, 'wb') as o:
+                    while left:
+                        b = f.read(min(left, 8 << 20))
+                        if not b: die(f'{img}: ends inside {name}')
+                        o.write(b); left -= len(b)
+                n += 1
+    stamp.write_text(want); t(f'unpacked {n:,} files')
+    return Path(dst)
+
+
 def frameworks(sdk, files):
     fw_dir = sdk / 'System/Library/Frameworks'
     out = list(FRAMEWORKS_BASE)
@@ -271,11 +295,11 @@ def main():
     for m in mods:
         if not ((patched / f'{m}.patched.bin').exists() and (patched / f'{m}.fixups.tsv').exists()): die(f'no rewritten module {patched}/{m} (run dm/tools/patch_nso.py first)')
     if not (romfs / marker).exists(): die(f'no RomFS at {romfs} (missing {marker})')
-    if image and env.get('NXW_MODS_ROMFS'): die('mods\' RomFS folders are laid over a RomFS folder, and this extraction is one RomFS image (RomFS.img)')
-    if image and env.get('NXW_SPLIT_DATA', '') not in ('', '0'): die('a split build copies the RomFS as a folder, and this extraction is one RomFS image (RomFS.img): build without --split-data')
+    if image and (env.get('NXW_MODS_ROMFS') or env.get('NXW_SPLIT_DATA', '') not in ('', '0')):   # both lay out the RomFS as a folder, and the NSP script extracts one image (nx_extract --romfs-image)
+        romfs, image = rfs.Path(unpack_romfs_image(romfs, w / 'build/romfs_unpacked')), False
     if env.get('NXW_MODS_ROMFS'):   # mods' RomFS folders (win/mods.py): hard links of the game's files with the mods' laid over them
-        import mods
-        romfs = mods.overlay_romfs(romfs, env['NXW_MODS_ROMFS'].split(os.pathsep), w / 'build/romfs_mods', log=t)
+        import mods as mod_overlay   # not "mods": that name is the module list gen_embed.py takes below
+        romfs = mod_overlay.overlay_romfs(romfs, env['NXW_MODS_ROMFS'].split(os.pathsep), w / 'build/romfs_mods', log=t)
     hdr = (DM / 'gfx/nvn/dmsh_chain.h').read_text(encoding='utf-8')
     v = (re.search(r'^#define DMSH_CHAIN "(.*)"', hdr, re.M) or die('no DMSH_CHAIN in dmsh_chain.h')).group(1)
     gfx = env.get('GFXAPI') or ('nvn' if list((msl_ios / f'b-{v}').glob('*_b.metallib')) else 'vulkan' if (msl_ios / 'vk/vk_shaders.metallib').exists()

win/tests/test_win.py

@@ -178,6 +178,20 @@ def half(h):
                     self.assertLessEqual(max(abs(x - y) for x, y in zip(f8, p)), 1, f'{fmt} mode {mi}')
 
 
+class UnpackRomfsImage(unittest.TestCase):
+    def test_round_trip_and_reuse(self):
+        sys.path.insert(0, str(REPO / 'dm/tools')); import romfs_image
+        with tempfile.TemporaryDirectory() as d:
+            src = Path(d) / 'src'; (src / 'arc/sub').mkdir(parents=True); (src / 'ui').mkdir()
+            files = {'arc/data.trpfd': b'index', 'arc/sub/big.bin': os.urandom(3 << 20), 'ui/empty.bin': b'', 'top.txt': b'top'}
+            for rel, b in files.items(): (src / rel).write_bytes(b)
+            img = Path(d) / 'RomFS.img'; romfs_image.pack(src, img)
+            out = build_app.unpack_romfs_image(img, Path(d) / 'unpacked')
+            self.assertEqual({p.relative_to(out).as_posix(): p.read_bytes() for p in out.rglob('*') if p.is_file()}, files)
+            (out / 'top.txt').write_bytes(b'marker')   # an unchanged image reuses the folder
+            build_app.unpack_romfs_image(img, out); self.assertEqual((out / 'top.txt').read_bytes(), b'marker')
+
+
 class PackIpa(unittest.TestCase):
     def test_exec_bit_on_windows(self):
         with tempfile.TemporaryDirectory() as d: