From b8d35b29bfbc59e2570cfe317d128e6de715cd0e Mon Sep 17 00:00:00 2001 From: Peiikai <339721950+peiikai@users.noreply.github.com> Date: Fri, 13 Nov 2026 23:10:57 -0600 Subject: [PATCH] Windows: start the generator with shadow stacks off too, and log where it got (#779) On Windows on ARM (x64 emulation, Windows 11 26H1) the game code generator still stops with 0xC0000409 although it runs with Control Flow Guard off. Both generators reach unicorn's first jump table emulation within seconds; the sdk one just has no log line before it, so its log stays empty. unicorn leaves its generated code with VCRUNTIME140's longjmp from a setjmp that saved no frame. That longjmp fast-fails for it when CFG is on (#337), and when a shadow stack is present it always unwinds with RtlUnwindEx, which can't walk the generated code. So the generator is now also started with user-mode shadow stacks off (falling back to CFG off alone where Windows refuses that policy). gen.py now logs a first line with the CPU it runs on (emulated or not) and whether CFG and shadow stacks are on in its process, and a line before its first unicorn run, so a failure's last log line shows how far it got. Co-Authored-By: Claude Opus 5.5 --- runtime/gamebuild.c | 10 +++-- runtime/platform.h | 9 ++-- runtime/platform/windows.c | 79 ++++++++++++++++++++------------- tools/gen.py | 34 ++++++++++++++ tools/test_gamebuild_ios.py | 2 +- tools/test_gamebuild_resume.py | 2 +- tools/test_windows_processes.py | 14 +++--- 7 files changed, 104 insertions(+), 46 deletions(-) diff --git a/runtime/gamebuild.c b/runtime/gamebuild.c index 0e80865d4..1f490bed8 100644 --- a/runtime/gamebuild.c +++ b/runtime/gamebuild.c @@ -75,14 +75,16 @@ static PlatProc spawn_proc(char **argv, const char *log, bool generator) { PlatProc p = 0; if (!g_quitting) { #ifdef _WIN32 - // The generator runs with Control Flow Guard off even when Windows forces it on for every program (#337) + // The generator runs with Control Flow Guard and shadow stacks off even when Windows forces them on for every + // program (#337, #779) if (generator) { - unsigned long error = 0; - p = plat_spawn_no_cfg((const char *const *)argv, log, false, &error); + unsigned long error = 0, ss_error = 0; + p = plat_spawn_no_cfg((const char *const *)argv, log, false, &error, &ss_error); static bool told; if (p && !told) { told = true; - if (!error) LOG("[gamebuild] generator started with Control Flow Guard off\n"); + if (!error && !ss_error) LOG("[gamebuild] generator started with Control Flow Guard and shadow stacks off\n"); + else if (!error) LOG("[gamebuild] generator started with Control Flow Guard off: shadow stacks could not be turned off for it (error %lu)\n", ss_error); else LOG("[gamebuild] generator started normally: Control Flow Guard could not be turned off for it (error %lu)\n", error); } } else diff --git a/runtime/platform.h b/runtime/platform.h index b1e054895..e1f62a915 100644 --- a/runtime/platform.h +++ b/runtime/platform.h @@ -160,9 +160,12 @@ int plat_proc_wait(PlatProc p, bool block); // exit code, or -1 while still run #define PLAT_PROC_SIGNALED 256 // + N: ended by signal N instead of exiting (not on Windows) #define PLAT_PROC_FAST_FAIL 512 // Windows: ended by a security check (0xC0000409, e.g. Control Flow Guard) #ifdef _WIN32 -// plat_spawn with Control Flow Guard off for the child (the generator: unicorn's longjmp fails CFG's check, #337). -// *error: 0 = started that way; else why not (then it was started normally, or not at all if that failed too) -PlatProc plat_spawn_no_cfg(const char *const argv[], const char *log_path, bool append_log, unsigned long *error); +// plat_spawn with Control Flow Guard and shadow stacks off for the child (the generator: unicorn's longjmp fails +// CFG's check, #337, and can't unwind with a shadow stack, #779). +// *error: 0 = started with CFG off; else why not (then it was started normally, or not at all if that failed too). +// *shadow_stack_error: the same for shadow stacks (when only that is refused, it was started with CFG off alone). +PlatProc plat_spawn_no_cfg(const char *const argv[], const char *log_path, bool append_log, unsigned long *error, + unsigned long *shadow_stack_error); #endif void plat_proc_kill(PlatProc p); // end it and what it started (then it's gone: no wait) bool plat_remove_tree(const char *dir); // rm -rf diff --git a/runtime/platform/windows.c b/runtime/platform/windows.c index 24a454e15..c064ae5fc 100644 --- a/runtime/platform/windows.c +++ b/runtime/platform/windows.c @@ -767,18 +767,26 @@ ssize_t pwrite(int fd, const void *buf, size_t n, off_t off) { return prw(fd, (v // ---- child processes typedef struct { HANDLE process, job; } WinProcess; -// Control Flow Guard off for one child (plat_spawn_no_cfg), as a process mitigation policy: it wins over the -// system-wide Exploit protection setting, which the exe's header (tools/pe_no_cfg.py) does not (#337) +// Control Flow Guard and user-mode shadow stacks off for one child (plat_spawn_no_cfg), as process mitigation +// policies: they win over the system-wide Exploit protection settings, which the exe's header (tools/pe_no_cfg.py) +// does not. unicorn (the generator's jump tables) leaves the code it generated with longjmp from a setjmp that saved +// no frame, and VCRUNTIME140's longjmp stops the process (0xC0000409) for that when CFG is on (#337); with a shadow +// stack it unwinds instead (RtlUnwindEx), through generated code that has no unwind data (#779). #ifndef PROC_THREAD_ATTRIBUTE_MITIGATION_POLICY #define PROC_THREAD_ATTRIBUTE_MITIGATION_POLICY 0x00020007 #endif #ifndef PROCESS_CREATION_MITIGATION_POLICY_CONTROL_FLOW_GUARD_ALWAYS_OFF #define PROCESS_CREATION_MITIGATION_POLICY_CONTROL_FLOW_GUARD_ALWAYS_OFF (0x00000002ull << 40) #endif +#ifndef PROCESS_CREATION_MITIGATION_POLICY2_CET_USER_SHADOW_STACKS_ALWAYS_OFF +#define PROCESS_CREATION_MITIGATION_POLICY2_CET_USER_SHADOW_STACKS_ALWAYS_OFF (0x00000002ull << 28) +#endif // no_cfg: NULL = a plain start; else *no_cfg = 0 when the child runs with CFG off, or the error that prevented it -// (then it is started without the policy) -static PlatProc spawn_child(const char *const argv[], const char *log_path, bool append_log, unsigned long *no_cfg) { +// (then it is started without the policy). *no_ss likewise for shadow stacks: a Windows that doesn't take that +// policy gets CFG off alone. +static PlatProc spawn_child(const char *const argv[], const char *log_path, bool append_log, unsigned long *no_cfg, + unsigned long *no_ss) { static __thread char cmd[32768]; cmd[0] = 0; for (int i = 0; argv[i]; i++) append_arg(cmd, sizeof cmd, argv[i]); @@ -800,29 +808,36 @@ static PlatProc spawn_child(const char *const argv[], const char *log_path, bool const wchar_t *app = GetFullPathNameW(wprog, MAX_PATH, wfull, NULL) ? wfull : NULL; BOOL ok = FALSE; if (no_cfg) { - *no_cfg = ERROR_NOT_SUPPORTED; - STARTUPINFOEXW six = {.StartupInfo = si}; - six.StartupInfo.cb = sizeof six; - DWORD64 policy[2] = {PROCESS_CREATION_MITIGATION_POLICY_CONTROL_FLOW_GUARD_ALWAYS_OFF, 0}; - SIZE_T size = 0; - InitializeProcThreadAttributeList(NULL, 1, 0, &size); - six.lpAttributeList = size ? (LPPROC_THREAD_ATTRIBUTE_LIST)malloc(size) : NULL; - if (!six.lpAttributeList || !InitializeProcThreadAttributeList(six.lpAttributeList, 1, 0, &size)) { - *no_cfg = six.lpAttributeList ? GetLastError() : ERROR_NOT_ENOUGH_MEMORY; - free(six.lpAttributeList); - six.lpAttributeList = NULL; - } else if (!UpdateProcThreadAttribute(six.lpAttributeList, 0, PROC_THREAD_ATTRIBUTE_MITIGATION_POLICY, policy, - sizeof policy, NULL, NULL)) { - *no_cfg = GetLastError(); - } else { - ok = CreateProcessW(app, wcmd, NULL, NULL, TRUE, - CREATE_NO_WINDOW | CREATE_SUSPENDED | EXTENDED_STARTUPINFO_PRESENT, NULL, NULL, - &six.StartupInfo, &pi); - *no_cfg = ok ? 0 : GetLastError(); // (older Windows: ERROR_INVALID_PARAMETER; or a policy refuses it) - } - if (six.lpAttributeList) { - DeleteProcThreadAttributeList(six.lpAttributeList); - free(six.lpAttributeList); + *no_cfg = *no_ss = ERROR_NOT_SUPPORTED; + // first both policies, then CFG alone (a Windows without shadow stack support may refuse the second word) + for (int both = 1; both >= 0 && !ok; both--) { + STARTUPINFOEXW six = {.StartupInfo = si}; + six.StartupInfo.cb = sizeof six; + DWORD64 policy[2] = {PROCESS_CREATION_MITIGATION_POLICY_CONTROL_FLOW_GUARD_ALWAYS_OFF, + both ? PROCESS_CREATION_MITIGATION_POLICY2_CET_USER_SHADOW_STACKS_ALWAYS_OFF : 0}; + unsigned long e; + SIZE_T size = 0; + InitializeProcThreadAttributeList(NULL, 1, 0, &size); + six.lpAttributeList = size ? (LPPROC_THREAD_ATTRIBUTE_LIST)malloc(size) : NULL; + if (!six.lpAttributeList || !InitializeProcThreadAttributeList(six.lpAttributeList, 1, 0, &size)) { + e = six.lpAttributeList ? GetLastError() : ERROR_NOT_ENOUGH_MEMORY; + free(six.lpAttributeList); + six.lpAttributeList = NULL; + } else if (!UpdateProcThreadAttribute(six.lpAttributeList, 0, PROC_THREAD_ATTRIBUTE_MITIGATION_POLICY, + policy, sizeof policy, NULL, NULL)) { + e = GetLastError(); + } else { + ok = CreateProcessW(app, wcmd, NULL, NULL, TRUE, + CREATE_NO_WINDOW | CREATE_SUSPENDED | EXTENDED_STARTUPINFO_PRESENT, NULL, NULL, + &six.StartupInfo, &pi); + e = ok ? 0 : GetLastError(); // (older Windows: ERROR_INVALID_PARAMETER; or a policy refuses it) + } + if (six.lpAttributeList) { + DeleteProcThreadAttributeList(six.lpAttributeList); + free(six.lpAttributeList); + } + if (both) *no_ss = e; + if (ok || !both) *no_cfg = e; } } if (!ok) ok = CreateProcessW(app, wcmd, NULL, NULL, TRUE, CREATE_NO_WINDOW | CREATE_SUSPENDED, NULL, NULL, &si, &pi); @@ -855,13 +870,15 @@ static PlatProc spawn_child(const char *const argv[], const char *log_path, bool } PlatProc plat_spawn(const char *const argv[], const char *log_path, bool append_log) { - return spawn_child(argv, log_path, append_log, NULL); + return spawn_child(argv, log_path, append_log, NULL, NULL); } -PlatProc plat_spawn_no_cfg(const char *const argv[], const char *log_path, bool append_log, unsigned long *error) { - unsigned long e = 0; - PlatProc p = spawn_child(argv, log_path, append_log, &e); +PlatProc plat_spawn_no_cfg(const char *const argv[], const char *log_path, bool append_log, unsigned long *error, + unsigned long *shadow_stack_error) { + unsigned long e = 0, ss = 0; + PlatProc p = spawn_child(argv, log_path, append_log, &e, &ss); if (error) *error = e; + if (shadow_stack_error) *shadow_stack_error = ss; return p; } diff --git a/tools/gen.py b/tools/gen.py index 790aa490b..af1f1c816 100644 --- a/tools/gen.py +++ b/tools/gen.py @@ -89,6 +89,8 @@ def boundaries(self): def emu(self): if self.mu is None: + # (the first unicorn run, for the first jump table: a generator stopped by Windows ends here, #337, #779) + print(f'{self.name}: starting the jump table emulator', file=sys.stderr, flush=True) mu = uc.Uc(uc.UC_ARCH_ARM64, uc.UC_MODE_ARM) n = self.nso mu.mem_map(self.base, len(n.image)) @@ -533,6 +535,35 @@ def write_if_changed(path, text): f.write(text) +def windows_mitigations(): + """Windows: the CPU this runs on and whether Control Flow Guard and shadow stacks are on in this process, for the + log (unicorn's longjmp fails with either: runtime/platform/windows.c spawn_child, #337, #779). '' elsewhere.""" + if os.name != 'nt': + return '' + try: + import ctypes + from ctypes import wintypes + k = ctypes.WinDLL('kernel32', use_last_error=True) + k.GetCurrentProcess.restype = wintypes.HANDLE + me = k.GetCurrentProcess() + machine = {0x8664: 'x64', 0xAA64: 'arm64'} + proc, native = wintypes.USHORT(), wintypes.USHORT() + host = 'x64' + if hasattr(k, 'IsWow64Process2') and k.IsWow64Process2(me, ctypes.byref(proc), ctypes.byref(native)): + host = machine.get(native.value, hex(native.value)) + this = machine.get(0x8664 if platform.machine().lower() in ('amd64', 'x86_64') else 0xAA64) + out = [f'{this} on {host}' + (' (emulated)' if this != host else '')] + for name, policy, on in (('Control Flow Guard', 7, 1), ('shadow stacks', 17, 1), ('strict shadow stacks', 17, 0x10)): + flags = wintypes.DWORD() + if k.GetProcessMitigationPolicy(me, policy, ctypes.byref(flags), ctypes.sizeof(flags)): + out.append(f'{name} {"on" if flags.value & on else "off"}') + else: + out.append(f'{name} unknown (error {ctypes.get_last_error()})') + return ', '.join(out) + except Exception as e: # (only for the log) + return f'mitigations unknown ({e})' + + def main(): ap = argparse.ArgumentParser() ap.add_argument('name') @@ -553,6 +584,9 @@ def main(): help='Android: guest addresses relative to the window the runtime picks (lifter.gaddr)') args = ap.parse_args() lifter.DYN_BASE = args.dyn_base + # (first: a generator that ends early still shows it started, and where it ran) + where = windows_mitigations() + print(f'{args.name}: started' + (f' ({where})' if where else ''), file=sys.stderr, flush=True) t0 = time.time() m = Module(args.name, args.nso, args.base, args.simd) diff --git a/tools/test_gamebuild_ios.py b/tools/test_gamebuild_ios.py index ac403c5a3..61a79a4af 100644 --- a/tools/test_gamebuild_ios.py +++ b/tools/test_gamebuild_ios.py @@ -54,7 +54,7 @@ extern "C" uint64_t plat_now_ns() { return std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count(); } extern "C" void plat_sleep_until_ns(uint64_t t) { auto now=plat_now_ns(); if(t>now)std::this_thread::sleep_for(std::chrono::nanoseconds(t-now)); } extern "C" intptr_t plat_spawn(const char *const *a,const char *l,bool append) { return start_cb(a,l,append); } -extern "C" intptr_t plat_spawn_no_cfg(const char *const *a,const char *l,bool append,unsigned long *e) { if(e)*e=0; return start_cb(a,l,append); } +extern "C" intptr_t plat_spawn_no_cfg(const char *const *a,const char *l,bool append,unsigned long *e,unsigned long *ss) { if(e)*e=0; if(ss)*ss=0; return start_cb(a,l,append); } extern "C" int plat_proc_wait(intptr_t p,bool block) { return wait_cb(p,block); } extern "C" void plat_proc_kill(intptr_t p) { kill_cb(p); } ''' diff --git a/tools/test_gamebuild_resume.py b/tools/test_gamebuild_resume.py index b6398e257..d2c1bb7b5 100644 --- a/tools/test_gamebuild_resume.py +++ b/tools/test_gamebuild_resume.py @@ -44,7 +44,7 @@ extern "C" uint64_t plat_now_ns() { return std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count(); } extern "C" void plat_sleep_until_ns(uint64_t t) { auto now=plat_now_ns(); if(t>now)std::this_thread::sleep_for(std::chrono::nanoseconds(t-now)); } extern "C" intptr_t plat_spawn(const char *const *a,const char *l,bool append) { return start_cb(a,l,append); } -extern "C" intptr_t plat_spawn_no_cfg(const char *const *a,const char *l,bool append,unsigned long *e) { if(e)*e=0; return start_cb(a,l,append); } +extern "C" intptr_t plat_spawn_no_cfg(const char *const *a,const char *l,bool append,unsigned long *e,unsigned long *ss) { if(e)*e=0; if(ss)*ss=0; return start_cb(a,l,append); } extern "C" int plat_proc_wait(intptr_t p,bool block) { return wait_cb(p,block); } extern "C" void plat_proc_kill(intptr_t p) { kill_cb(p); } ''' diff --git a/tools/test_windows_processes.py b/tools/test_windows_processes.py index a25df7fe6..7c647d1c0 100644 --- a/tools/test_windows_processes.py +++ b/tools/test_windows_processes.py @@ -45,10 +45,12 @@ CloseHandle(pi.hThread);CloseHandle(pi.hProcess);Sleep(30000);return 0; } if(argc>1&&!strcmp(argv[1],"--fastfail")) { ExitProcess(0xC0000409u); } - if(argc>1&&!strcmp(argv[1],"--cfg")) { // 0: Control Flow Guard off in this process + if(argc>1&&!strcmp(argv[1],"--cfg")) { // 0: Control Flow Guard and shadow stacks off in this process PROCESS_MITIGATION_CONTROL_FLOW_GUARD_POLICY p={0}; if(!GetProcessMitigationPolicy(GetCurrentProcess(),ProcessControlFlowGuardPolicy,&p,sizeof p)) return 3; - return p.EnableControlFlowGuard?1:0; + DWORD ss=0; // (ProcessUserShadowStackPolicy = 17: bit 0 EnableUserShadowStack) + if(!GetProcessMitigationPolicy(GetCurrentProcess(),(PROCESS_MITIGATION_POLICY)17,&ss,sizeof ss)) return 4; + return p.EnableControlFlowGuard?1:(ss&1)?2:0; } assert(argc==2); char a[1200],b[1200];snprintf(a,sizeof a,"%s/first.pid",argv[1]);snprintf(b,sizeof b,"%s/second.pid",argv[1]); @@ -66,16 +68,16 @@ assert(plat_proc_wait(plat_spawn(crash,NULL,false),true)==255); const char *fastfail[]={exe,"--fastfail",NULL},*cfg[]={exe,"--cfg",NULL}; assert(plat_proc_wait(plat_spawn(fastfail,NULL,false),true)==PLAT_PROC_FAST_FAIL); - unsigned long cfg_error=1; - PlatProc nocfg=plat_spawn_no_cfg(cfg,NULL,false,&cfg_error); - assert(nocfg&&cfg_error==0&&plat_proc_wait(nocfg,true)==0); + unsigned long cfg_error=1,ss_error=1; + PlatProc nocfg=plat_spawn_no_cfg(cfg,NULL,false,&cfg_error,&ss_error); + assert(nocfg&&cfg_error==0&&ss_error==0&&plat_proc_wait(nocfg,true)==0); reject_job=true; assert(plat_proc_wait(plat_spawn(code,NULL,false),true)==7); const char *sleeper[]={exe,"--sleep",NULL}; PlatProc fallback=plat_spawn(sleeper,NULL,false); assert(fallback && !((WinProcess *)fallback)->job && plat_proc_wait(fallback,false)==-1); plat_proc_kill(fallback); - puts("Windows process trees: isolated cancellation, crash status, CFG-off start and fallback without job assignment passed"); + puts("Windows process trees: isolated cancellation, crash status, CFG- and shadow-stack-off start and fallback without job assignment passed"); return 0; } '''