From de0dfe2e2704b69d66d19ab98ad3d70c72fafa4e Mon Sep 17 00:00:00 2001 From: seedds Date: Mon, 9 Nov 2026 18:46:29 -0800 Subject: [PATCH] Recognize bit-test guarded masked jump tables --- dm/tools/patch_nso.py | 52 ++++++++- dm/tools/tests/test_jump_tbnz.py | 181 +++++++++++++++++++++++++++++++ docs/agents/symptoms.md | 1 + docs/development.md | 2 +- docs/reference/rewriter.md | 10 +- 5 files changed, 243 insertions(+), 3 deletions(-) create mode 100644 dm/tools/tests/test_jump_tbnz.py diff --git a/dm/tools/patch_nso.py b/dm/tools/patch_nso.py index 40e77204d6..8b81252591 100644 --- a/dm/tools/patch_nso.py +++ b/dm/tools/patch_nso.py @@ -801,6 +801,7 @@ def func_of(a): if i >= 0: return (starts[i], starts[i + 1] if i + 1 < len(starts) else TEXT_END) return (max((e for s, e in DATA_BLOCKS if e <= a), default=0), starts[0]) if starts else (a, a + 4) done = set(); windows = []; jumps = {}; tabjumps = {}; tailcalls = set(); retries = [] +bit_test_protected = set() # accepted bit-test tables: entries here would bypass their base/guard proof; checked again after every table is resolved def guarded_count(ctx, kc, kl): """How many entries the compare ctx[kc] (cmp index,#N) lets the table load ctx[kl] read, or None when no branch on its flags guards the load. A compare bounds the index only through the branch that acts on it: cmp w18,#4; b.hi exit; cmp w18,#1; ldrsw x18,[x4,x18,lsl #2] has a nearer compare that bounds nothing, and taking it read 2 entries of a table of 5 @@ -928,6 +929,46 @@ def wname(r): return 'w' + r[1:] if page is None: return None tb = page + lo return [tb + struct.unpack_from(' 31 or masked.group(2) != check.group(1) or masked.group(1) != idx or int(masked.group(3), 0) != (1 << (bit + 1)) - 1: return None + if base in (ent, 'x' + idx) or add.op_str != f'x18, {ent}, {base}': return None + def control(p): + return p.mnemonic in ('b', 'bl', 'br', 'blr', 'ret', 'cbz', 'cbnz', 'tbz', 'tbnz', 'svc', 'hvc', 'smc', 'brk', 'hlt', 'eret', 'drps') or p.mnemonic.startswith('b.') + lo = None; head = None + for k in range(j - 5, -1, -1): + p = insl[k] + if control(p): return None # a lexical writer across a branch or call does not prove the reaching value + if not writes_reg(p, base): continue + low = re.fullmatch(rf'{base}, {base}, #(0x[0-9a-f]+|\d+)', p.op_str) if p.mnemonic == 'add' else None + if lo is None and low: lo = int(low.group(1), 0); continue + if lo is not None and p.mnemonic == 'adrp' and p.op_str.startswith(base + ','): head = k; break + return None # includes 32-bit writes and copies whose source is not proven here + if head is None or lo is None: return None + run = insl[head:j + 1] + if any(q.address != p.address + 4 for p, q in zip(run, run[1:])): return None + protected = {p.address for p in run[1:]} + for p in insl: + if p.mnemonic in ('b', 'bl', 'cbz', 'cbnz', 'tbz', 'tbnz') or p.mnemonic.startswith('b.'): + if p.operands[-1].imm in protected: return None + if any(protected.intersection(targets) for targets in tabjumps.values()): return None + if any(any(b <= t < b + 4 * width for t in protected) for b, width in jumps.values()): return None + tb = insl[head].operands[1].imm + lo; n = 1 << bit + if tb < 0 or tb + 4 * n > len(IMG): return None + targets = [tb + struct.unpack_from('> 24 @@ -1453,6 +1502,7 @@ def pieces(fa, fb): elif t & 0x7C == 0x14: d = w & 0x3FFFFFF; d -= (d & 0x2000000) << 1 else: continue dest = at + 4 * d + assert dest not in bit_test_protected, f'branch bypasses bit-test table proof {at:x}' if dest in new_mid: assert func_of(at) == func_of(dest), f'branch into window from another function {at:x}' def put_branch(at, to): struct.pack_into('> 5) << 31 | (((to - at) // 4) & 0x3fff) << 5 | reg +def mask(dst, src, bit): return 0x92400000 | bit << 10 | src << 5 | dst +def load(ent, base, idx): return 0xb8a07800 | idx << 16 | base << 5 | ent +def mov(dst, src, wide=True): return (0xaa0003e0 if wide else 0x2a0003e0) | src << 16 | dst + + +class BitTestTable(unittest.TestCase): + TABLE = 0x2040 + TARGETS = [0, 40, 44, 48] + + def setUp(self): + self.img = bytearray(0x4000) + for i, target in enumerate(self.TARGETS): struct.pack_into('` assertion where the index is a copy made before the check (`mov w0,w18`, then 41 instructions holding two other jump tables on `x0`, then `cmp w18,#4; b.hi; ldrsw x18,[x11,x0,lsl #2]; add x18,x18,x11; br x18`). → [Index copied before the bounds check](../reference/rewriter.md#index-copied-before-the-bounds-check) - `jump table ` assertion in `patch_nso.py` on an offset table whose index is checked with a 64-bit compare (`cmp x22, #0x10; …; b.hi; adrp x18 + add x18; mov x0, x18; ldrsw x18, [x0, x22, lsl #2]; add x18, x18, x0; br x18`). → [64-bit bounds check of the index](../reference/rewriter.md#64-bit-bounds-check-of-the-index) - `jump table ` assertion in `patch_nso.py` where the index is made with a 64-bit `and` by `0xffffffff` after the check (`cmp w8, #4; b.hi; and x18, x8, #0xffffffff; ldrsw x18, [x16, x18, lsl #2]; add x18, x18, x16; br x18`). → [Index zero-extended after a 32-bit bounds check](../reference/rewriter.md#index-zero-extended-after-a-32-bit-bounds-check) +- `jump table 590ee8` assertion in `patch_nso.py` on Ultra Street Fighter II: The Final Challengers (SDK 1.3.1). → [Bit-test bounded masked jump-table index](../reference/rewriter.md#bit-test-bounded-masked-jump-table-index) - `jump table ` assertion in a Unity / IL2CPP `main` whose dispatch checks a fresh mask of another register than the one it indexes with. → [Index masked in the block that branches to the dispatch](../reference/rewriter.md#index-masked-in-the-block-that-branches-to-the-dispatch) - `jump table 6eea88` or `jump table 6eeac0` assertion in `patch_nso.py` on Return of the Obra Dinn 1.0.1. → [Zero-guarded complemented jump-table index](../reference/rewriter.md#zero-guarded-complemented-jump-table-index) - `jump table ` assertion on an offset table whose base was set far back (`cmp w18,#5; b.hi; ldrsw x18,[x24,x18,lsl #2]; add x18,x18,x24; br x18`). → [Offset table base set far back in the function](../reference/rewriter.md#offset-table-base-set-far-back-in-the-function) diff --git a/docs/development.md b/docs/development.md index 955a0260d6..133990ec91 100644 --- a/docs/development.md +++ b/docs/development.md @@ -6,7 +6,7 @@ The README covers packing and playing. This page is for working on nxnative itse | Part | State | |---|---| -| Rewriter (`dm/tools/patch_nso.py`) | Runs unchanged on every module of every supported game. Per-build cuts are keyed by the module's build ID, in the profile's `patches.py` (a game's own modules) or `dm/tools/sdk_patches.py` (Nintendo's `sdk` modules). Profiles can opt verified cuts into [function matching](reference/rewriter.md#function-matched-cuts-for-unregistered-builds) across updates | +| Rewriter (`dm/tools/patch_nso.py`) | Runs unchanged on every module of every supported game. Offset tables can use a [bit-test bounded masked index](reference/rewriter.md#bit-test-bounded-masked-jump-table-index) when the base, guard and incoming paths prove the exact target set. Per-build cuts are keyed by the module's build ID, in the profile's `patches.py` (a game's own modules) or `dm/tools/sdk_patches.py` (Nintendo's `sdk` modules). Profiles can opt verified cuts into [function matching](reference/rewriter.md#function-matched-cuts-for-unregistered-builds) across updates | | Host (`dm/host`) | Stands in for the Horizon kernel and the system services the games call: threads and synchronization, memory, file system with transactional saves, HID, audio, movies (VP9 through libvpx, Vorbis sound through stb_vorbis), software keyboard, focus and lifecycle | | Graphics (`dm/gfx`) | NVN on Metal. Shaders are extracted by `shader_split.py` from Nintendo's containers and from Unity's `Shader` objects, then translated offline. Mario: 16,266 of 16,268 shaders compile. Engage: 9,268 of 9,269. Games that render with Vulkan (Hades) or OpenGL (Digimon Story Cyber Sleuth) go through layers of their own on Metal (`dm/gfx/vk`, `dm/gfx/gl`) | | Audio | Games that carry the SDK's audio renderer (Mario) run it in-process; games that use AudioOut directly (Engage: Wwise) feed it straight. With Core Audio enabled (default on iOS, `DM_AUDIO=1` on the Mac), the [silent sink keeps releasing PCM buffers while a device starts or retries](reference/audio.md#a-blocked-audio-device-start-freezes-the-game). | diff --git a/docs/reference/rewriter.md b/docs/reference/rewriter.md index 669804c35b..4bb0b89565 100644 --- a/docs/reference/rewriter.md +++ b/docs/reference/rewriter.md @@ -116,6 +116,14 @@ To enroll a cut for any game's own module: - **Check:** this is a separate fallback after every earlier recognizer fails. Adding the rule inside `jump_offsets` before its existing searches could change a previously recognized table's extent by selecting a nearer compare. The fallback preserves those earlier results; synthetic full-rewriter tests cover both the new five-entry form and an existing eight-entry match with a nearer five-entry check. - **Found on:** Mario Tennis Aces (0100BDE00862A000) `main+0x35837c` (1.0.0), a five-entry table whose targets all lie in the function ([PR #778](https://github.com/seedds/nxnative/pull/778)). The contributor reported that `main`, `subsdk0` and `sdk` rewrite with zero-extension recognition; gameplay validation remains open. +### Bit-test bounded masked jump-table index + +- **Symptom:** `jump table 590ee8` assertion in `patch_nso.py` on Ultra Street Fighter II: The Final Challengers (SDK 1.3.1). +- **Cause:** `tbnz wS,#B,exit; and xI,xS,#((1<<(B+1))-1); ldrsw xE,[xBase,xI,lsl #2]; add x18,xE,xBase; br x18` bounds the index without a compare. The branch excludes values with bit B set; the mask removes every higher bit, leaving exactly `2^B` table entries. Reading the mask plus one would include unreachable entries. +- **Fix:** `jump_tbnz_masked` is tried after the existing recognizers. It requires the adjacent dispatch instructions, the same tested and masked source, a full-width table base built by `adrp` / `add` on a straight-line path, and an in-image table whose targets are decoded instructions. Calls, traps, control-flow changes, register clobbers and undecodable gaps break the proof. No branch or table target may bypass the base initialization or guard; the complete module's direct branches and all resolved tables are checked again before rewriting, including tables encountered later. +- **Limits:** base copies and base initialization across control flow are not proven by this fallback. The separate long-context dispatch at `main+0x7d3000` and the game's SDK 1 audio support are outside this rule; recognizing this form alone does not establish full game compatibility. +- **Found on:** [PikaMan80's Ultra Street Fighter II report and source patch (#960)](https://github.com/seedds/nxnative/issues/960), `main+0x590ee8`, title ID `01007330027EE000`, update v131072. There is no repository profile for this title. + ### Index masked in the block that branches to the dispatch - **Symptom:** `jump table ` assertion in a Unity / IL2CPP `main` whose dispatch checks a fresh mask of another register than the one it indexes with. @@ -129,7 +137,7 @@ To enroll a cut for any game's own module: - **Symptom:** `jump table 6eea88` or `jump table 6eeac0` assertion in `patch_nso.py` on Return of the Obra Dinn 1.0.1. - **Cause:** Unity's IL2CPP output uses `tst w15,#7; b.eq exit; mvn w15,w15; and x15,x15,#7; ldrsw x15,[xB,x15,lsl #2]; add x18,x15,xB; br x18`. The zero test excludes one masked value before complementing it: the reachable indices are 0–6, so the table holds seven entries. Treating the `tst` immediate as a `cmp` bound reads an eighth word from the next table, introducing a target the guest cannot reach through this dispatch. -- **Fix:** the last fallback, `jump_masked_not`, accepts this contiguous guard / complement / low-bit-mask form, reads exactly the mask's number of entries, follows the table base through `copied_base`, and requires decoded instruction targets in the function. It rejects intervening register writes or control flow, direct branches / calls and already resolved tables entering after the guard, and its own table targets bypassing the guard. Earlier recognizers are unchanged. +- **Fix:** a fallback, `jump_masked_not`, accepts this contiguous guard / complement / low-bit-mask form, reads exactly the mask's number of entries, follows the table base through `copied_base`, and requires decoded instruction targets in the function. It rejects intervening register writes or control flow, direct branches / calls and already resolved tables entering after the guard, and its own table targets bypassing the guard. Earlier recognizers are unchanged. - **Found on:** [Return of the Obra Dinn](../../dm/games/010032E00E6E2000/README.md) 1.0.1 (`main` build `9e93a6566aaa27418dd2435d8d5dffe5`), two dispatches in the function at `main+0x6ee8c0`. - **Check:** `dm/tools/tests/test_jump_copies.py` exercises the exact seven-entry extent with an invalid next word, mismatched guards / masks / sources, register clobbers, incoming edges, undecodable words and invalid targets. The extracted update's two sites resolve to seven targets each; this is static verification, with gameplay validation recorded in the game note.