unityfs: Unity 6000.3 Shader and ComputeShader layouts for stripped player files

dm/gfx/tests/test_unity_compute.py

@@ -162,5 +162,63 @@ def test_platform_alignment_and_byte_code_padding(self):
                 self.assertEqual(reader.p, size)
 
 
+def compute_6000_3():
+    # Independent wire fixture from the 6000.3 release dumps (Castlevania: Belmont's Curse): 2022.3's unique
+    # variants and variant indices, each resource with a resource type after its texture dimension.
+    code = shader_split.CS_MAGIC + bytes(60)
+    control = bytearray(0x880)
+    control[:4] = shader_split.CTL_MAGIC
+    struct.pack_into('<I', control, 0x20, len(control))
+    struct.pack_into('<I', control, 0x704, len(code))
+    struct.pack_into('<III', control, 0x748, 8, 8, 1)
+    header = [0] * 6 + [-1] * 6 + [0] * 12
+    header[11], header[17], header[23] = 0, len(control), len(code)
+    program = struct.pack('<24i', *header) + control + code
+    resource = string('_Source') + string('_Source') + struct.pack('<iiii', 0, -1, 2, 5)
+    kernel = (struct.pack('<iI', 1, 0)                         # cbVariantIndices
+              + struct.pack('<i', 0)                         # cbs
+              + struct.pack('<i', 1) + resource              # textures
+              + struct.pack('<3i', 0, 0, 0)                  # samplers, in/out buffers
+              + struct.pack('<i', len(program)) + program
+              + struct.pack('<4Iq', 3, 8, 8, 1, 0))
+    parent = (string('TextureCopy') + struct.pack('<i', 1) + kernel
+              + struct.pack('<i', 1) + string('') + struct.pack('<I', 0)
+              + struct.pack('<3i', 0, 0, 0))
+    variant = struct.pack('<iii', 38, 50, 1) + parent + struct.pack('<i', 0) + b'\1\0\0\0'
+    payload = string('VrsTexture') + struct.pack('<i', 1) + variant
+    type_hash = bytes.fromhex('43a0948a3f403068c0b31bff90ba0c9a')
+    metadata = (b'6000.3.0b4\0' + struct.pack('<i?iiBh', 38, False, 1, 72, 0, -1)
+                + type_hash + struct.pack('<i', 1))
+    metadata += bytes(-(20 + len(metadata)) % 4)
+    metadata += struct.pack('<qIII', 1, 0, len(payload), 0)
+    data = (struct.pack('>IIII', len(metadata), 0x100 + len(payload), 17, 0x100)
+            + bytes(4) + metadata).ljust(0x100, b'\0') + payload
+    return data, type_hash, bytes(control), code
+
+
+class Unity6000_3Tests(unittest.TestCase):
+    def test_stripped_compute_resource_type_reaches_shader_split(self):
+        data, type_hash, control, code = compute_6000_3()
+        trees = {(72, type_hash): unityfs.known_tree(72, type_hash)}
+        obj, = unityfs.serialized_objects(data, {}, {72}, trees)
+        kernel = obj[2]['variants'][0]['kernels'][0]['uniqueVariants'][0]
+        self.assertEqual(kernel['textures'][0]['resType'], 5)
+        self.assertEqual(kernel['threadGroupSize'], [8, 8, 1])
+        self.assertTrue(obj[2]['variants'][0]['resourcesResolved'])
+        le, doff, types, entries = unityfs._tables(data, {}, {72}, trees)
+        _, start, size, ti = entries[0]
+        reader = unityfs.Reader(data[doff + start:doff + start + size], le)
+        unityfs._read(reader, types[ti][2])
+        self.assertEqual(reader.p, size)
+        with contextlib.redirect_stdout(io.StringIO()): shaders = shader_split.unity_compute(data, 'fixture', trees)
+        self.assertEqual(shaders, [('fixture#VrsTexture#TextureCopy.0', 'cs', control, code)])
+
+    def test_shader_appends_local_identifier_to_6000_0(self):
+        old = unityfs.known_tree(48, bytes.fromhex('2ec7223dce3af4232b14ae1441bd7999'))
+        new = unityfs.known_tree(48, bytes.fromhex('57f0dd2d4ef24437b4655529a673a936'))
+        self.assertEqual([k.name for k in new.kids], [k.name for k in old.kids] + ['m_AssetLocalIdentifierInFile'])
+        self.assertEqual((new.kids[-1].type, new.kids[-1].flags), ('SInt64', 0))
+
+
 if __name__ == '__main__':
     unittest.main()

dm/gfx/tests/test_unity_known_trees.py

@@ -11,9 +11,10 @@
 SHADER_HASHES = {'2017.4': '266d53113fa30d2b858f2768f92eaa14', '2018.3': '8db760073e1392061e4a36995f0735af',
                  '2019.4': 'ed2a06ce7fb8203f6efb5ab5e0ce20d5', '2020.3': 'a30e3dcf6c6d25153d6f64e5e41caf4e',
                  '2021.2': 'd8236c9bd87d85edb203acf352348124', '2022.3': 'ea424a5fc1e48db62144df956f9e1005',
-                 '6000.0': '2ec7223dce3af4232b14ae1441bd7999'}
+                 '6000.0': '2ec7223dce3af4232b14ae1441bd7999', '6000.3': '57f0dd2d4ef24437b4655529a673a936'}
 COMPUTE_HASHES = {'2017.4': '20291e020d10074c35d8286f5191593f', '2019.4': '1a95529ac2eb1a97cd9d2a4356bd722f',
-                  '2021.3': 'abd9135b8ce83d043fef4e9ec7f53366', '2022.3': 'bfdf63ab0ce84d5a2c765ee560d90fe7'}
+                  '2021.3': 'abd9135b8ce83d043fef4e9ec7f53366', '2022.3': 'bfdf63ab0ce84d5a2c765ee560d90fe7',
+                  '6000.3': '43a0948a3f403068c0b31bff90ba0c9a'}
 
 
 class KnownTreeTests(unittest.TestCase):

dm/gfx/tools/unityfs.py

@@ -156,6 +156,9 @@ class NoTree(ValueError):
 # and all 214 stripped Shader objects read to their exact ends. Keep the older layout intact: its pass headers still contain those two arrays.
 SHADER_6000_0 = SHADER_2022_3.replace('vector<data:Hash128> m_EditorDataHash; vector<data:UInt8> m_Platforms; ', '').replace(
     'bool! m_ShaderIsBaked\n', 'bool! m_ShaderIsBaked; GUID m_AssetGUID\n') + 'GUID: unsigned int data[0]; unsigned int data[1]; unsigned int data[2]; unsigned int data[3]\n'
+# Unity 6000.3 (Castlevania: Belmont's Curse, its player built by 6000.3.0f1 or later; the built-in resources name 6000.3.0b4) strips every tree and ships bundles without Shaders: Shader version 4
+# adds the object's own local identifier after the asset GUID (AssetRipper's StructsDump/release/6000.3.0f1.dump; 6000.3.0b4's dump still has version 3). Its 2,306 nodes and alignment flags match the 6000.3.0f1 and 6000.3.20f1 dumps; all 97 stripped Shader objects read to their exact ends
+SHADER_6000_3 = SHADER_6000_0.replace('GUID m_AssetGUID\n', 'GUID m_AssetGUID; SInt64 m_AssetLocalIdentifierInFile\n')
 # The Unity 2020.3 Shader (Patrick's Parabox, Unity 2020.3.30f1: only Data/*.assets, its shaders the built-in ones in Data/Resources) matches the type tree in Fire Emblem Engage's bundles
 # (Unity 2020.3.18f1), whose hash is the same: global and local keyword indices per subprogram, every subprogram in m_SubPrograms, no parameter blobs, keyword names or stage counts
 SHADER_2020_3 = _SHADER_COMMON + '''SerializedSubProgram: unsigned int m_BlobIndex; ParserBindChannels m_Channels; vector<data:UInt16> m_GlobalKeywordIndices; vector<data:UInt16> m_LocalKeywordIndices; SInt8 m_ShaderHardwareTier; SInt8! m_GpuProgramType; SerializedProgramParameters m_Parameters; int m_ShaderRequirements
@@ -262,15 +265,20 @@ class NoTree(ValueError):
 # Return of the Obra Dinn's stripped built-ins use this hash; its internalskinning object (12 kernels) reads to its exact end.
 COMPUTE_2017_4 = COMPUTE_2019_4.replace('int samplerBindPoint; int texDimension', 'ComputeBufferCounter counter') + \
     'ComputeBufferCounter: int bindpoint; int offset\n'
+# Unity 6000.3's ComputeShader (Castlevania: Belmont's Curse): each resource gains a resource type after its texture dimension (StructsDump/release/6000.3.0b4.dump to 6000.3.20f1.dump), and the
+# thread group size is a staticvector as the dump names it (read the same as a vector); the rest is 2022.3's. All 159 nodes and alignment flags match; all 11 stripped objects read to their exact ends
+COMPUTE_6000_3 = COMPUTE_2022_3.replace('int texDimension', 'int texDimension; int resType').replace('vector<data:unsigned int>~ threadGroupSize', 'staticvector<data:unsigned int>~ threadGroupSize')
 CLASS_COMPUTE_SHADER = 72
 KNOWN_TREES = {(CLASS_SHADER, bytes.fromhex('ea424a5fc1e48db62144df956f9e1005')): ('Shader', SHADER_2022_3),
                (CLASS_SHADER, bytes.fromhex('d8236c9bd87d85edb203acf352348124')): ('Shader', SHADER_2021_2),
                (CLASS_SHADER, bytes.fromhex('2ec7223dce3af4232b14ae1441bd7999')): ('Shader', SHADER_6000_0),
+               (CLASS_SHADER, bytes.fromhex('57f0dd2d4ef24437b4655529a673a936')): ('Shader', SHADER_6000_3),
                (CLASS_SHADER, bytes.fromhex('a30e3dcf6c6d25153d6f64e5e41caf4e')): ('Shader', SHADER_2020_3),
                (CLASS_SHADER, bytes.fromhex('ed2a06ce7fb8203f6efb5ab5e0ce20d5')): ('Shader', SHADER_2019_4),
                (CLASS_SHADER, bytes.fromhex('8db760073e1392061e4a36995f0735af')): ('Shader', SHADER_2018_3),
                (CLASS_SHADER, bytes.fromhex('266d53113fa30d2b858f2768f92eaa14')): ('Shader', SHADER_2017_4),
                (CLASS_COMPUTE_SHADER, bytes.fromhex('bfdf63ab0ce84d5a2c765ee560d90fe7')): ('ComputeShader', COMPUTE_2022_3),
+               (CLASS_COMPUTE_SHADER, bytes.fromhex('43a0948a3f403068c0b31bff90ba0c9a')): ('ComputeShader', COMPUTE_6000_3),
                (CLASS_COMPUTE_SHADER, bytes.fromhex('abd9135b8ce83d043fef4e9ec7f53366')): ('ComputeShader', COMPUTE_2021_3),
                (CLASS_COMPUTE_SHADER, bytes.fromhex('1a95529ac2eb1a97cd9d2a4356bd722f')): ('ComputeShader', COMPUTE_2019_4),
                (CLASS_COMPUTE_SHADER, bytes.fromhex('20291e020d10074c35d8286f5191593f')): ('ComputeShader', COMPUTE_2017_4)}

docs/reference/shaders.md

@@ -314,6 +314,7 @@ Finding, keying and translating shaders (`dm/gfx/tools/shader_split.py`, `dmsh_b
   - [Donkey Kong Country Returns HD](../../dm/games/01009D901BC56000/README.md) (Unity 6000.0.28f1, no bundles): hash `2ec7223d…` matches BALL x PIT's Unity 6000.0.62f1 bundles. `SHADER_6000_0` derives from the 2022.3 layout by removing each pass's `m_EditorDataHash` and `m_Platforms` arrays and appending the shader's `m_AssetGUID`. The generated tree equals the bundle tree in types, field order and alignment flags; the game note records the object census and build results.
   - LEGO Builder's Journey (Unity 2021.2.13f1, provisional profile: every serialized file is stripped and no bundle carries a `Shader` tree): hash `d8236c9b…`. `SHADER_2021_2` is written from [AssetRipper's 2021.2.13f1 release tree](https://github.com/AssetRipper/TypeTreeDumps/blob/main/StructsDump/release/2021.2.13f1.dump) and matches it node for node (2,174 nodes, the alignment flags included): its subprograms drop the local keyword list (as Unity 2022 does), keep their parameters and use signed 64-bit requirements; each pass appends `m_SerializedKeywordStateMask`, and the parsed form adds keyword names and flags. All 195 `Shader` objects read to their exact ends, and the split finds 9,410 stage shaders (3,567 vertex, 3,571 pixel, 2,271 compute, one geometry) in the player's files and `main`.
   - Dodgeball Academia (Unity 2019.4.21f1, provisional profile: every serialized file is stripped and no bundle carries a `Shader` tree): hash `ed2a06ce…`. `SHADER_2019_4` derives from the 2018.3 layout, checked node for node against [AssetRipper's 2019.4.21f1 release tree](https://github.com/AssetRipper/TypeTreeDumps/blob/main/StructsDump/release/2019.4.21f1.dump) (1,481 nodes, the alignment flags included) and equal to the tree [Subnautica](../../dm/games/0100429011144000/README.md)'s Unity 2019.4.36f1 bundles carry under the same hash: global and local keyword indices in each subprogram, a ray tracing program in each pass, and 2019.3's segments (one `offsets` / `compressedLengths` / `decompressedLengths` list per platform); the parameters stay in each subprogram, without `m_CommonParameters`. All 62 `Shader` objects read to their exact ends, and the split finds 7,996 stage shaders (4,162 vertex, 3,778 pixel, 55 compute, one geometry) where 11 came from `main` alone.
+  - Castlevania: Belmont's Curse (Unity 6000.3; its built-in resources name 6000.3.0b4; no `Shader` in its 368 Addressables bundles, all 97 in the player's files): hash `57f0dd2d…`. `SHADER_6000_3` is the 6000.0 layout with `SInt64 m_AssetLocalIdentifierInFile` after `m_AssetGUID` (`Shader` version 4, from 6000.3.0f1 on; the 6000.3.0b4 dump still has version 3, whose tree hashes like 6000.0's). It equals all 2,306 nodes and alignment flags of AssetRipper's 6000.3.0f1 and 6000.3.20f1 release dumps, and every `Shader` object reads to its exact end. Its `ComputeShader` (hash `43a0948a…`, 11 objects) is `COMPUTE_6000_3` (below). 2,165 stage shaders: 1,034 vertex, 1,038 pixel, 92 compute and one geometry shader; 11 of them are the player's own in `main`.
 - **Note:** a Unity version whose `Shader` hash is not in the table still stops with the same message; read its tree from any bundle of that Unity version, or from AssetRipper's TypeTreeDumps for that exact version (compared node for node, the alignment flags included), and add it with its hash to `KNOWN_TREES` and to `dm/gfx/tests/test_unity_known_trees.py`. To take one file out of an NSP without extracting the RomFS, filter `nx_extract`'s `Copy` by path.
 
 ### Unity 6 Shader objects
@@ -329,6 +330,7 @@ Finding, keying and translating shaders (`dm/gfx/tools/shader_split.py`, `dmsh_b
 - **Fix:** `unity_compute` in `shader_split.py` reads them beside the `Shader` objects of every serialized file. The game has no bundle with a class 72 tree, so `unityfs.py` carries Unity 2022.3.54f1's layout (`COMPUTE_2022_3`, written from AssetRipper's TypeTreeDumps for that version; all 12 objects decode, `resourcesResolved` true at each end). 85 compute shaders came in (86 with the player's own).
 - **Found on:** [Monster Train 2](../../dm/games/010051701FB46000/README.md) (Unity 2022.3.54f1).
 - **Older layouts:** Unity 2019.4 stores each kernel's code directly, without `uniqueVariants` or `variantMap`. `unity_compute` accepts that form too, and `COMPUTE_2019_4` supplies the stripped-file layout. Found on [Pokémon Brilliant Diamond](../../dm/games/0100000011D90000/README.md) (2019.4.27f1); its missing runtime shader is the built-in GPU skinning kernel. The layout matches the release dump's 116 nodes and alignment flags, and both built-in compute objects decode to their exact ends.
+- **Unity 6000.3:** `COMPUTE_6000_3` covers stripped class 72 hash `43a0948a3f403068c0b31bff90ba0c9a`: 2022.3's layout with an `int resType` after each resource's `texDimension`, and the thread group size a `staticvector` as the dumps name it. It matches all 159 nodes and alignment flags of AssetRipper's 6000.3.0b4 to 6000.3.20f1 release dumps; Castlevania: Belmont's Curse's 11 built-in and player objects (VRS texture copy and reduce kernels among them) read to their exact ends.
 - **Unity 2021.3:** `COMPUTE_2021_3` covers stripped class 72 hash `abd9135b8ce83d043fef4e9ec7f53366`, matching all 144 nodes and alignment flags in [AssetRipper's 2021.3.18f1 release tree](https://github.com/AssetRipper/TypeTreeDumps/blob/main/StructsDump/release/2021.3.18f1.dump). Kernel parents use `variantMap`, global and local keywords; each kernel holds byte-vector code, a `staticvector` thread group size and signed 64-bit requirements. Unlike 2022.3 there are no `uniqueVariants`, `variantIndices` or dynamic keywords; both versions store code as bytes. `resourcesResolved` aligns after every platform variant, including when another platform follows. The synthetic tests cover two platforms, multiple keyword variants, byte-code padding, exact object consumption and shader provenance. Found on Shin chan: Shiro and the Coal Town (Unity 2021.3.18f1): [PR #440](https://github.com/seedds/nxnative/pull/440) reports recovered GPU skinning kernels and correctly rendered characters on the iPhone; that gameplay result has not been independently repeated here.
 - **Unity 2017.4:** [Return of the Obra Dinn](../../dm/games/010032E00E6E2000/README.md) needs hash `20291e02…` for stripped `internalskinning` objects. `COMPUTE_2017_4` restores the nested buffer-counter fields; all 120 nodes and alignment flags match the release dump. The game note records the object census and exact-end checks.
 - **Registered tables after a reader update:** a newly supported layout can add kernels to an existing game's shader map. For registered module builds, the NSP script permits a shader-only derived profile only when every original shader's package, stage and source, every original fact and every evidence pair remain unchanged (package indices may move), and every added shader is compute with matching key/source evidence from `unity_compute`. Each added kernel must also have its fact row. Registered settings, build rows and cuts are retained; the repository tables are not edited. A cached comparison pins the registered tables, split inputs and comparison code. Deleted or changed original rows, graphics additions and additions without reader evidence still stop the build. The incremental-build tests cover acceptance, rejection and cache invalidation.

scripts/build_ipa_from_nsp.py

@@ -714,7 +714,7 @@ def listed():   # files in the manifest so far
                      'legacy paks (version 4, DRAGON QUEST XI S) and version 11 paks without shader code archives (OCTOPATH TRAVELER II) keep them inline in the cooked packages, which the shader split reads file by file')
     if extra: notes.append('modules besides main / sdk: {}; the provisional profile loads {} (load order main, subsdk*, sdk)'.format(", ".join(f'{n} ({mods[n]["kind"]})' for n in extra), G["modules"][0])
                            + ('; the build adds {}: the NROs import from {}'.format(', '.join(nro_add), 'it' if len(nro_add) == 1 else 'them') if nro_add else ''))
-    if rom['mags']['UnityFS'] or 'Unity IL2CPP' in F['markers']: notes.append('Unity: the shaders are Shader objects in UnityFS bundles and data.unity3d, which shader_split.py reads through unityfs.py (written for Unity 2020.3\'s Switch format, also read for 2017.4, 2018.3, 2019.4, 2021.2, 2022.3 and 6000.0; check other Unity versions first)')
+    if rom['mags']['UnityFS'] or 'Unity IL2CPP' in F['markers']: notes.append('Unity: the shaders are Shader objects in UnityFS bundles and data.unity3d, which shader_split.py reads through unityfs.py (written for Unity 2020.3\'s Switch format, also read for 2017.4, 2018.3, 2019.4, 2021.2, 2022.3, 6000.0 and 6000.3; check other Unity versions first)')
     if F['glslc']: notes.append(f'main has {F["glslc"]} GLSLC compiler output magics (0x19866891): shader_split.py takes those that are whole outputs (13 Sentinels and the KTGL games compile their shaders into main; Unity\'s player has 11 of its own)')
     if rom['mags']['Trinity pack']: notes.append('Trinity packs (Game Freak, arc/data.trpfs with its index data.trpfd): the shaders are BNSH inside Oodle Mermaid streams; shader_split.py decodes them with the game\'s own OodleLZ_Decompress in the Mac host, which dm/host/hle_oodle.c finds by the library\'s name string')
     elif rom['mags']['PlatinumGames PKZ']: notes.append('PlatinumGames PKZ archives (pkzl; Bayonetta 3): DAT archives inside, compressed per entry (OodleKraken, ZStandard or None); the shaders are shader objects (VSO / PSO / CSO …) in DATs, which shader_split.py reads (pkz_entries, pg_shader), Oodle Kraken through the game\'s own streaming decoder in the Mac host (dm/host/hle_oodle.c)')