From a5356952927f9033e3e5dc0de9f87b9842bfa82f Mon Sep 17 00:00:00 2001 From: JS-banana <42017491+JS-banana@users.noreply.github.com> Date: Fri, 13 Nov 2026 15:15:10 +0800 Subject: [PATCH] packing/shader_split: read Phyre shaders from split KIT archives Unravel TWO stores its effects in indexed KIT parts with chained LZ4 blocks. Read those members with length and SHA1 checks, then recognize headerless compute programs within bounded Phyre effects. Keep existing graphics acceptance rules and archive scan budgets. Rechecked on main 38521863 with its GLSLC section reader and shared-DLC shader key merge. A synthetic KIT regression verifies that a repeated DLC effect leaves the game rows unchanged in either scan order. 117 targeted tests pass. Python lint reports no new findings. Game routes and Metal GPU regression remain untested on this revision. --- dm/gfx/tests/test_pg_shader.py | 9 +++ dm/gfx/tests/test_vbf.py | 120 ++++++++++++++++++++++++++++++++- dm/gfx/tools/shader_split.py | 79 ++++++++++++++++++++-- docs/agents/symptoms.md | 1 + docs/reference/shaders.md | 8 +++ 5 files changed, 211 insertions(+), 6 deletions(-) diff --git a/dm/gfx/tests/test_pg_shader.py b/dm/gfx/tests/test_pg_shader.py index c9b66f6df5..93c4d44634 100644 --- a/dm/gfx/tests/test_pg_shader.py +++ b/dm/gfx/tests/test_pg_shader.py @@ -47,6 +47,15 @@ def test_truncated_or_unrelated_data_is_not_a_shader(self): split.scan_blob(bad, '/unrelated', found) self.assertEqual(found, []) + def test_minimal_binding_table_is_not_a_phyre_record(self): + data, ctl, code = object_data('ps') + data = struct.pack(' len(d) or count > (len(d) - base) // 56: red(f'{path}: KIT directory is outside the index') + entries, sizes = [], {} + for i in range(count): + flags, part, off, high, stored, size, sha, _, _ = struct.unpack_from(' sizes[part] or n > sizes[part] - off: red(f'{path}: KIT entry {i} is outside shard {part}') + entries.append((i, part, off, n, size, bool(flags & 8), sha)) + return entries + +def kit_read(path, entry): + import lz4.block + i, part, off, n, size, compressed, sha = entry + label = f'{path}#{i}' + with open(f'{path}.{part}', 'rb') as f: f.seek(off); d = f.read(n) + if len(d) != n: red(f'{label}: truncated KIT member') + if compressed: + out, pos = bytearray(), 0 + while pos + 4 <= len(d) and len(out) < size: + stored = struct.unpack_from('= step: jobs.append((p, r, (KIT, tuple(cur)))); cur, size = [], 0 + if cur: jobs.append((p, r, (KIT, tuple(cur)))) + return jobs, paths + def phyre_shaders(d, label): - """Phyre's NVN code immediately precedes its control section, unaligned in RYHP effects; loose blobs also prefix their two lengths (FFX/X-2).""" + """Phyre's NVN code immediately precedes its control section, unaligned in RYHP effects; loose blobs also prefix their two lengths (FFX/X-2). Unravel TWO also has headerless compute code.""" out = [] for m in re.finditer(re.escape(CTL_MAGIC), d): c = m.start() @@ -2178,9 +2240,10 @@ def phyre_shaders(d, label): cn = struct.unpack_from(' len(d) or d[k:k + 4] != GFX_MAGIC: continue + if k < 0 or kn < 0x80 or cn < 0x760 or c + cn > len(d): continue ctl, code = d[c:c + cn], d[k:c] - st = sph_stage(code) + st = glslc_stage(ctl, code) + if st == 'cs' and code[:4] != CS_MAGIC and (d[:4] != b'RYHP' or CTL_MAGIC in code): continue if st and not check(label, st, ctl, code): out.append((f'{label}#PHYRE@{k:#x}', st, ctl, code)) return out @@ -3038,6 +3101,8 @@ def scan_blob(d, label, out, depth=0, left=None): if d[:4] in PG_STAGES and len(d) >= 0x18 and d[(o := struct.unpack_from(' 2 and args[2][0] == KIT: + for e in args[2][1]: scan_blob(kit_read(path, e), f'/{rel}#{e[0]}', found) + return rel, uniq_keys(found), var_pairs(found), trees if len(args) > 2 and args[2][0] == b'ARH1': arh_shaders(path, rel, args[2][1], found) return rel, uniq_keys(found), var_pairs(found), trees @@ -3471,8 +3539,9 @@ def main(): fb_jobs, fbs = frostbite_jobs(files) # EA SPORTS FC 27's Frostbite bundles, Oodle Leviathan decoded by the game's own decoder in the Mac host pd_jobs, pdats = pdat_jobs(files) # Kirby Air Riders' PDAT archives: the PMAP records in byte ranges rp_jobs, rpfs = rpf_jobs(files, main_v) # Red Dead Redemption's RPF6 archives, their tables decrypted with the key in main (rpf6.py) - big += pak_jobs + tr_jobs + ue_jobs + bd_jobs + arc_jobs + pz_jobs + ck_jobs + d3_jobs + ca_jobs + cr_jobs + zp_jobs + ah_jobs + fb_jobs + pd_jobs + rp_jobs - files = [(p, r) for p, r in files if not linkdata_jobs(p, r) and p not in paks and p not in trp and p not in uep and p not in bdts and p not in arcs and p not in pkzs and p not in cpks and p not in d3s and p not in cascs and p not in cras and p not in zips and p not in arhs and p not in fbs and p not in pdats and p not in rpfs] + kt_jobs, kits = kit_jobs(files) # Unravel TWO's KIT shards: chained LZ4 resources and Phyre effects + big += pak_jobs + tr_jobs + ue_jobs + bd_jobs + arc_jobs + pz_jobs + ck_jobs + d3_jobs + ca_jobs + cr_jobs + zp_jobs + ah_jobs + fb_jobs + pd_jobs + rp_jobs + kt_jobs + files = [(p, r) for p, r in files if not linkdata_jobs(p, r) and p not in paks and p not in trp and p not in uep and p not in bdts and p not in arcs and p not in pkzs and p not in cpks and p not in d3s and p not in cascs and p not in cras and p not in zips and p not in arhs and p not in fbs and p not in pdats and p not in rpfs and p not in kits] with cf.ProcessPoolExecutor(J, initializer=_pool_init, initargs=(common, {}, zdicts)) as ex: n = len(big) + len(files) results += pooled(ex, big, n) + pooled(ex, files, n, len(big), chunk=16) diff --git a/docs/agents/symptoms.md b/docs/agents/symptoms.md index 47709ff1d0..cd8ae8fd59 100644 --- a/docs/agents/symptoms.md +++ b/docs/agents/symptoms.md @@ -933,6 +933,7 @@ One line per symptom in the [subsystem references](../reference/): what you see - the split finds only a small part of a game's shaders, with no `[RED]` line, and its RomFS holds files that start `NARC` (`.narc`, `.nptl`, `.ntex` on Mario Tennis Aces). → [Packed shader banks in loose NARC files](../reference/shaders.md#packed-shader-banks-in-loose-narc-files) - the split finds no shaders in `main` or RomFS, while reporting control sections in `main` with no code area. The game has `.sp4` resources and loose `.nvnbin` files rather than BNSH archives or whole GLSLC outputs. → [Sumo Ninty_NX shaders and MuxFDMR streams](../reference/shaders.md#sumo-ninty_nx-shaders-and-muxfdmr-streams) - the shader split finds no game programs despite loose `.fx.phyre` effects and multi-GB `.vbf` archives. → [Phyre NVN effects in Virtuos VBF archives](../reference/shaders.md#phyre-nvn-effects-in-virtuos-vbf-archives) +- the shader split finds no shaders in `main` or RomFS, which holds a `.kit` index and numbered `.kit.N` parts. Reading the effects alone still misses their compute programs. → [Phyre effects in split KIT archives](../reference/shaders.md#phyre-effects-in-split-kit-archives) - the split finds no shaders (`[RED] the shader split found no shaders in main or the RomFS`), the RomFS holds only `.cpk` files, and `cpk.py` refuses them (`not a CPK`); the run prints `[RED] shader … is not in the shader map` for every program. → [Diablo III's console packages](../reference/shaders.md#diablo-iiis-console-packages) - the split finds no shaders (`[RED] the shader split found no shaders in main or the RomFS`) and names control sections in `main` with `no code area: no data record points at one`; the RomFS is a `data` folder of archives named `00-00000001.data` … `17-00000002.data` beside `.psv` manifests and `build_key.txt`; the run prints `[RED] shader … is not in the shader map` for every program and draws black. → [Blizzard's CASC storage and SHAD containers](../reference/shaders.md#blizzards-casc-storage-and-shad-containers) - the shader split stops on one archive with a Python traceback (an `IndexError` or `struct.error` naming no file), or a reader's `[RED]` line about a format it does not know, and nothing is written; the NSP script loses the build at the split. → [A file no reader reads](../reference/shaders.md#a-file-no-reader-reads) diff --git a/docs/reference/shaders.md b/docs/reference/shaders.md index 2d43d93c32..914b84ec5d 100644 --- a/docs/reference/shaders.md +++ b/docs/reference/shaders.md @@ -35,6 +35,14 @@ Finding, keying and translating shaders (`dm/gfx/tools/shader_split.py`, `dmsh_b - **Found on:** [FINAL FANTASY X/X-2 HD Remaster](../../dm/games/0100BC300CB48000/README.md); base and both supplied DLC packs yield 5,752 unique shaders. - **Check:** `test_vbf.py` covers raw and zlib blocks, shared/missing members, truncation, unaligned effects, loose shader keys and inconsistent lengths. All 5,752 programs translate on the Mac. +### Phyre effects in split KIT archives + +- **Symptom:** the shader split finds no shaders in `main` or RomFS, which holds a `.kit` index and numbered `.kit.N` parts. Reading the effects alone still misses their compute programs. +- **Cause:** the `kit!` index names members by hash and places their data in separate parts. Compressed members use 64 KiB LZ4 blocks whose matches can reach the previous block. Their `RYHP` effects contain both ordinary graphics programs and compute programs without the graphics code header that `phyre_shaders` required. +- **Fix:** `kit_entries` checks the index and part ranges; `kit_read` carries the previous 64 KiB into each block and verifies decoded lengths, complete input consumption and the member's SHA1. Raw members use their decoded length. `kit_jobs` scans members through `scan_blob` and excludes the index's parts from the loose-file scan. `phyre_shaders` uses the existing control/code stage reader; a headerless compute program must be inside a `RYHP` effect, fit its bounds and not overlap a control section. Graphics programs keep their existing acceptance rules. Nonzero high offset words remain unsupported and are rejected. +- **Found on:** Unravel TWO (`0100E5D00CC0C000`) 1.0.1, with a provisional profile: 22,268 archive members yield 2,629 unique shaders (438 vertex, 1,793 pixel, 398 compute), against zero before. The Mac shader build completes with all 2,629. +- **Check:** `test_vbf.py` constructs KIT parts and Phyre effects without game data. It covers a block that cannot decode without the preceding dictionary, raw members, missing or truncated parts, invalid lengths and hashes, compute bounds, the parent archive's scan budget, graphics code containing the control magic as a literal, and stable game rows when a DLC repeats a KIT effect. `test_pg_shader.py` checks that a minimal binding-table record still reaches the existing reader. Format reference: Luigi Auriemma's [Unravel KIT reader](https://aluigi.altervista.org/bms/unravel.bms). + ### Diablo III's console packages - **Symptom:** the split finds no shaders (`[RED] the shader split found no shaders in main or the RomFS`), the RomFS holds only `.cpk` files, and `cpk.py` refuses them (`not a CPK`); the run prints `[RED] shader … is not in the shader map` for every program.