Enable compiler optimizations and LTO for macOS host, libvpx and app launcher

dm/gfx/nvn/dmsh_chain.h

@@ -1,3 +1,3 @@
 // Generated file (dm/gfx/tools/dmsh_chain.py --header): the B shader library translation chain version this source tree expects (default options), one for every game;
 // the host loads only <library dir>/b-<V>/. Run dmsh_chain.py --header after changing the chain's tools and commit this file with the change
-#define DMSH_CHAIN "acb723a57d06"
+#define DMSH_CHAIN "38dd7cf7a7ff"

dm/gfx/tools/dmsh_build.py

@@ -33,7 +33,6 @@
 if 'ios' in TARGETS and not IOSOUT: sys.exit('[RED] DMSH_TARGETS asks for iOS libraries without IOSOUT')
 if 'ios' not in TARGETS: IOSOUT = None   # every side library, cache, cleanup and stamp follows the same selection
 if IOSOUT:
-    sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), '../../ios/tools')); import ios_metal
     IOSOUT = os.path.join(IOSOUT, 'b-' + CHAIN); os.makedirs(IOSOUT, exist_ok=True)
 for _d in [out] + ([IOSOUT] if IOSOUT else []):   # one chain version serves every game, so b-<V>/ alone does not keep two games' libraries apart
     _g = dmsh_chain.read_stamp(_d).get('game')
@@ -162,17 +161,15 @@ def width(ty):   # argument buffer entries a member takes
 PRE_STATS = collections.Counter()
 # -fpreserve-invariance: vertex positions are marked [[invariant]], but without this flag Metal does not promise that two different shaders compute the same expression to the same value; on Maxwell the same multiply-adds always agree.
 # Super Mario Odyssey draws a depth pre-pass with alRenderDepthOnly, then the G-buffer with another vertex shader and depth test less-or-equal: a hair off and the test failed, leaving those pixels with old content (Cap Kingdom's ground in 32x32 blocks)
-# -O1: the front end's -O2 (CorrelatedValuePropagation, then SimplifyCFG) folds the translated shaders' repeated tests of one value into switches whose cases jump past their own join.
-# Apple's GPU backend then exceeds its control-flow ID budget ("Max Unstruct ID 758 ... exceeds supported limit", a note, not an error) and falls back to a slower lowering:
-# Odyssey's alRenderMaterialUber took 25 minutes and 17.3 MB of GPU code, 31 s and 1.3 MB at -O1. Ordinary shaders come out within a few percent either way (the backend optimizes again)
-MAC_METAL = ['xcrun', '-sdk', 'macosx', 'metal', '-c', '-O1', '-fpreserve-invariance']
-MAC_RECORD = ' '.join(MAC_METAL) + '; xcrun -sdk macosx metallib'
+# Use the shared O3/CVP workaround and late vector combining from ios_metal on both platforms.
+sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), '../../ios/tools')); import ios_metal
+MAC_METAL = ['xcrun', '-sdk', 'macosx', 'metal', '-c', '-O3', '-fpreserve-invariance', '-Xclang', '-disable-llvm-passes']
+MAC_RECORD = ' '.join(MAC_METAL) + '; xcrun -sdk macosx metallib' + ios_metal.OPT_RECORD
 # The outer shader and fetch-variant caches restore compiled libraries directly,
 # before the MSL compile cache can check their compiler. Keep override builds separate there too.
 METAL_FP = dmsh_mtlcache.compiler_id('macosx') + ('\0iphoneos\0' + dmsh_mtlcache.compiler_id('iphoneos') if IOSOUT else '')
 def mac_compile(msl, air, lib):   # same interface as ios_metal.compile
-    subprocess.run([*MAC_METAL, msl, '-o', air], check=True)
-    subprocess.run(['xcrun', '-sdk', 'macosx', 'metallib', air, '-o', lib], check=True)
+    ios_metal.compile(msl, air, lib, MAC_METAL)
 # The NVN driver's constant segments have one layout in every game, which dm/gfx/nvn/dmsh_abi.h holds for the host and this build (through dmsh_abi.py); nvn_mtl.m fills them the same way at run time.
 #   c2 C2_TEX + 8 r: handles bound with nvnCommandBufferBindTexture (combined texture and sampler), slot r. The driver has 32 such slots, but a game may bind higher ones, whose words lie in the
 #     image table (Hades II binds slot 34, and no image), so a texture instruction's handle anywhere below the separate texture table names a combined slot. In the image
@@ -843,7 +840,7 @@ def unfillable(dr, stage):
 # CODE_FP hashes the code form of this file and its helper modules (dmsh_chain.code_form): editing a comment, a docstring or the layout does not change any shader's key.
 # The library ABI enters as its values (dmsh_abi.VALUES), so a comment in dmsh_abi.h does not change a key either.
 # The first-stage code fingerprint does not include dmsh_fetch.py; readable-tile variants get a separate second-stage cache keyed on the generated MSL.
-CODE_FP = hashlib.sha256(source_without_rule_bodies() + b''.join(b'\0' + dmsh_chain.code_form(open(m.__file__, 'r').read()) for m in (dmsh_chain, dmsh_geometry, dmsh_atomic, abi) + ((ios_metal,) if IOSOUT else ()))
+CODE_FP = hashlib.sha256(source_without_rule_bodies() + b''.join(b'\0' + dmsh_chain.code_form(open(m.__file__, 'r').read()) for m in (dmsh_chain, dmsh_geometry, dmsh_atomic, abi, ios_metal))
                          + b'\0abi\0' + repr(sorted(abi.VALUES.items())).encode()).hexdigest()
 TOOL_FP = tool_fingerprint().encode()
 

dm/host/Makefile

@@ -15,7 +15,10 @@ endif
 endif
 GAME_H = gen/$(GAME)/game_profile.h
 include ../mac/min_os.mk
-CFLAGS=-O1 -g -Wall -Wno-unused-function -mmacosx-version-min=$(MACOS_MIN)
+# Host C / Objective-C and libvpx: guest ARM64 instructions and offline Metal keep their own policies.
+NATIVE_OPT ?= -O3 -flto=full
+VPX_OPT ?= $(NATIVE_OPT)
+CFLAGS=$(NATIVE_OPT) -g -Wall -Wno-unused-function -mmacosx-version-min=$(MACOS_MIN)
 # Service-layer takeover hle_*.c files are included by wildcard: adding a file does not touch this line (avoids merge conflicts)
 SRC=main.c loader.c hle.c crash.c probe.c thread.c save_sync.c memstat.m haptics_apple.m $(sort $(wildcard hle_*.c)) front_mac.m entry.S
 # NVN null device + nn::vi + the Vulkan layer (dm/gfx/)
@@ -27,9 +30,9 @@ VPX=../third_party/libvpx
 VPX_SRC=$(sort $(shell find $(VPX) -name '*.c'))
 VPX_OBJ=$(patsubst $(VPX)/%.c,gen/vpx/%.o,$(VPX_SRC))
 VPX_LIB=gen/vpx/libvpx.a
-gen/vpx/%.o: $(VPX)/%.c $(wildcard $(VPX)/config/*.h) ../mac/min_os.mk
+gen/vpx/%.o: $(VPX)/%.c $(wildcard $(VPX)/config/*.h) Makefile ../mac/min_os.mk
 	@mkdir -p $(dir $@)
-	clang -O2 -g -mmacosx-version-min=$(MACOS_MIN) -I$(VPX) -I$(VPX)/config -c $< -o $@
+	clang $(VPX_OPT) -g -mmacosx-version-min=$(MACOS_MIN) -I$(VPX) -I$(VPX)/config -c $< -o $@
 $(VPX_LIB): $(VPX_OBJ)
 	rm -f $@; ar rcs $@ $^
 # The host's own Metal shaders (dm/gfx/msl/*.metal) compiled for MSL_SDK into gen/msl-<sdk>/host_msl_gen.h (dm/gfx/tools/host_msl.py; dm/gfx/nvn/host_msl.m loads them), so the host
@@ -63,7 +66,7 @@ endif
 endif
 .PHONY: FORCE
 FORCE:
-dmhost: $(SRC) $(GFX) ../third_party/stb_vorbis/stb_vorbis.c dm.h hle.h romfs_image.h save_sync.h dm_input.h pad_slots.h hle_haptics.h haptics_apple.h motion_fusion.h motion_apple.h motion_pro.h motion_pro_mac.h pad_apple.h pad_hid.h pad_hid_mac.h front.h $(GFX_H) $(GAME_H) $(GAME_STAMP) $(VPX_LIB) $(MSL_H)
+dmhost: Makefile $(SRC) $(GFX) ../third_party/stb_vorbis/stb_vorbis.c dm.h hle.h romfs_image.h save_sync.h dm_input.h pad_slots.h hle_haptics.h haptics_apple.h motion_fusion.h motion_apple.h motion_pro.h motion_pro_mac.h pad_apple.h pad_hid.h pad_hid_mac.h front.h $(GFX_H) $(GAME_H) $(GAME_STAMP) $(VPX_LIB) $(MSL_H)
 	clang $(CFLAGS) -I../gfx/nvn/include -I$(dir $(MSL_H)) -I../third_party/vulkan-headers/include -I$(dir $(GAME_H)) -I$(VPX) -I../third_party/stb_vorbis -fobjc-arc -o $@ $(SRC) $(GFX) $(VPX_LIB) -framework Cocoa -framework QuartzCore -framework Metal -framework MetalFX -framework GameController -framework CoreHaptics -framework IOKit -framework AudioToolbox -framework VideoToolbox -framework CoreMedia -framework CoreVideo -framework ImageIO -framework UniformTypeIdentifiers
 clean:   # the host and everything make generates for it (gen/: the profile headers, the game stamp and the libvpx build; dmhost_* and *.dSYM beside them, as in .gitignore)
 	rm -rf gen dmhost dmhost_* *.dSYM

dm/host/hle_bigstack.c

@@ -77,6 +77,7 @@ __asm__(
     "    blr x16\n"
     ".globl _dm_bigstack_ret\n"
     ".private_extern _dm_bigstack_ret\n"
+    ".alt_entry _dm_bigstack_ret\n"   // Keep LTO from separating this return path from the entry.
     "_dm_bigstack_ret:\n"
     "    ldr x9, [sp]\n"
     "    mov sp, x9\n"

dm/ios/tools/ios_metal.py

@@ -1,14 +1,53 @@
 # The single compile standard for the iOS shader packaging chain: MSL from any source (PC chain / constant patches / in future, the Switch original translated via Ryujinx) → iphoneos metallib.
 # Compile parameters are defined only once, here; the chain.metal record in the reflection json is generated from the same parameters, record == actual.
-import subprocess
+import functools, hashlib, re, subprocess
+from pathlib import Path
 # -fmetal-math-mode=relaxed: keep fast math's reassociation / contraction, but forbid assuming there are no NaN / Inf. The translated shaders reproduce Maxwell's NaN semantics with isnan selects (8337 / 12809 use them),
 # and the default fast mode lets each GPU's backend compiler treat them as always false; on the A20 Pro the opening course's odd frames had 727k NaN pixels in the main image (a few dozen with the same AIR on the M4 Pro), which became the black band / black-edged glow
 # -fpreserve-invariance: [[invariant]] position outputs only agree across shaders with this flag (as dmsh_build.py's MAC_METAL: a depth pre-pass followed by a less-or-equal G-buffer)
-# -O1: -O2's switch folding leaves a control-flow graph Apple's GPU backend can only lower slowly into very large code (as dmsh_build.py's MAC_METAL)
-METAL = ['xcrun', '-sdk', 'iphoneos', 'metal', '-c', '-O1', '-mios-version-min=18.0', '-fmetal-math-mode=relaxed', '-fpreserve-invariance']
+# O3 without CVP avoids Apple's pathological control-flow lowering; retain its AIR passes and add late vector combining / scalar cleanup.
+METAL = ['xcrun', '-sdk', 'iphoneos', 'metal', '-c', '-O3', '-mios-version-min=18.0', '-fmetal-math-mode=relaxed', '-fpreserve-invariance', '-Xclang', '-disable-llvm-passes']
 METALLIB = ['xcrun', '-sdk', 'iphoneos', 'metallib']
-RECORD = ' '.join(METAL) + '; ' + ' '.join(METALLIB)                                # Written into json chain.metal
+OPT_RECORD = '; metal-opt default<O3> -CVP +vector-combine,instcombine,early-cse ' + hashlib.sha256(Path(__file__).read_bytes()).hexdigest()[:12]
+RECORD = ' '.join(METAL) + '; ' + ' '.join(METALLIB) + OPT_RECORD                  # Written into json chain.metal
 
-def compile(msl, air, lib):
-    subprocess.run([*METAL, msl, '-o', air], check=True)
-    subprocess.run([*METALLIB, air, '-o', lib], check=True)
+@functools.lru_cache(None)
+def optimizer(metal):
+    tool = Path(subprocess.check_output([*metal[:3], '-f', 'metal'], text=True).strip()).parent / 'metal-opt'
+    driver = subprocess.check_output([*metal, '-###', '-x', 'metal', '/dev/null', '-o', '/dev/null'], stderr=subprocess.STDOUT, text=True)
+    target = re.search(r'"-triple" "([^"]+)"', driver)
+    if target is None: raise ValueError('Metal did not report its target triple')
+    options = ['-mtriple=' + target[1], '-vectorize-loops', '-vectorize-slp', '-extra-vectorizer-passes']
+    pipeline = subprocess.check_output([tool, *options, '-passes=default<O3>', '-print-pipeline-passes', '-disable-output'], input='', text=True).strip()
+    names = {
+        'AIRFunctionPredicateResolverPass': 'air-function-predicate-resolver',
+        'AIRDisabledEliminationPass': 'air-disabled-elimination',
+        'AIRPostTessellationOptPass': 'air-post-tessellation-opt',
+        'AIRAliasScopePropagationPass': 'air-alias-scope-propagation',
+        'AIRConditionalAddressSpacePass': 'air-conditional-address-space',
+        'AIRIntersectionQueryDCEPass': 'air-intersection-query-dce',
+        'GlobalCtorOptPass': 'global-ctors-opt',
+        'AIRVertexAmplifierPass': 'air-vertex-amplifier',
+        'AIRUnusedArgumentsPass': 'air-unused-args',
+        'AIRFixupMetadataPass': 'air-fixup-metadata',
+        'AIRArraysOfResourcesEmulationPass': 'air-arrays-of-resources-emulation',
+        'AIRMetadataPrunerPass': 'air-metadata-pruner',
+        'AIRCallGraphCheckerPass': 'air-callgraph-checker',
+        'AIRVerifierPass': 'air-verify',
+    }
+    pipeline = re.sub(r'air::(\w+)', lambda m: names[m[1]], pipeline)
+    if pipeline.count('correlated-propagation,') != 2 or pipeline.count('global-ctors-opt') != 1:
+        raise ValueError('unexpected Metal O3 pipeline')
+    pipeline = pipeline.replace('correlated-propagation,', '').replace('global-ctors-opt', 'function(vector-combine,instcombine,early-cse<>),global-ctors-opt')
+    if 'correlated-propagation' in pipeline: raise ValueError('unfiltered Metal CVP pass')
+    return [tool, *options, '-passes=' + pipeline]
+
+def compile(msl, air, lib, metal=None):
+    metal = tuple(METAL if metal is None else metal)
+    raw = Path(str(air) + '.raw')
+    try:
+        subprocess.run([*metal, '-fmodules-cache-path=' + str(Path(air).resolve().parent / 'metal-modules'), msl, '-o', raw], check=True)
+        subprocess.run([*optimizer(metal), raw, '-o', air], check=True)
+        subprocess.run([*metal[:3], 'metallib', air, '-o', lib], check=True)
+    finally:
+        raw.unlink(missing_ok=True)

dm/mac/pack_app.py

@@ -88,8 +88,8 @@ def build_launcher(out, minos, clang, swiftc, sdk):   # launcher.c's main and th
     with tempfile.TemporaryDirectory(dir=out.parents[3]) as t:
         objs = [Path(t) / 'launcher.o', Path(t) / 'save_sync.o']
         for obj, src in zip(objs, (HERE / 'launcher.c', HERE.parent / 'host/save_sync.c')):
-            run([clang, '-isysroot', sdk, '-arch', 'arm64', f'-mmacosx-version-min={minos}', '-O2', '-Wall', '-I', HERE.parent / 'host', '-c', '-o', obj, src])
-        run([swiftc, '-sdk', sdk, '-target', f'arm64-apple-macos{minos}', '-O', '-swift-version', '5', '-parse-as-library', '-module-cache-path', Path(t) / 'module-cache',
+            run([clang, '-isysroot', sdk, '-arch', 'arm64', f'-mmacosx-version-min={minos}', '-O3', '-flto=full', '-Wall', '-I', HERE.parent / 'host', '-c', '-o', obj, src])
+        run([swiftc, '-sdk', sdk, '-target', f'arm64-apple-macos{minos}', '-O', '-whole-module-optimization', '-lto=llvm-full', '-swift-version', '5', '-parse-as-library', '-module-cache-path', Path(t) / 'module-cache',
              '-o', out, *swift_sources(), *objs, '-framework', 'CoreFoundation', '-framework', 'AppKit', '-framework', 'SwiftUI'])