diff --git a/runtime/gamebuild.c b/runtime/gamebuild.c index 0e80865d4..1f490bed8 100644 --- a/runtime/gamebuild.c +++ b/runtime/gamebuild.c @@ -75,14 +75,16 @@ static PlatProc spawn_proc(char **argv, const char *log, bool generator) { PlatProc p = 0; if (!g_quitting) { #ifdef _WIN32 - // The generator runs with Control Flow Guard off even when Windows forces it on for every program (#337) + // The generator runs with Control Flow Guard and shadow stacks off even when Windows forces them on for every + // program (#337, #779) if (generator) { - unsigned long error = 0; - p = plat_spawn_no_cfg((const char *const *)argv, log, false, &error); + unsigned long error = 0, ss_error = 0; + p = plat_spawn_no_cfg((const char *const *)argv, log, false, &error, &ss_error); static bool told; if (p && !told) { told = true; - if (!error) LOG("[gamebuild] generator started with Control Flow Guard off\n"); + if (!error && !ss_error) LOG("[gamebuild] generator started with Control Flow Guard and shadow stacks off\n"); + else if (!error) LOG("[gamebuild] generator started with Control Flow Guard off: shadow stacks could not be turned off for it (error %lu)\n", ss_error); else LOG("[gamebuild] generator started normally: Control Flow Guard could not be turned off for it (error %lu)\n", error); } } else diff --git a/runtime/platform.h b/runtime/platform.h index b1e054895..e1f62a915 100644 --- a/runtime/platform.h +++ b/runtime/platform.h @@ -160,9 +160,12 @@ int plat_proc_wait(PlatProc p, bool block); // exit code, or -1 while still run #define PLAT_PROC_SIGNALED 256 // + N: ended by signal N instead of exiting (not on Windows) #define PLAT_PROC_FAST_FAIL 512 // Windows: ended by a security check (0xC0000409, e.g. Control Flow Guard) #ifdef _WIN32 -// plat_spawn with Control Flow Guard off for the child (the generator: unicorn's longjmp fails CFG's check, #337). -// *error: 0 = started that way; else why not (then it was started normally, or not at all if that failed too) -PlatProc plat_spawn_no_cfg(const char *const argv[], const char *log_path, bool append_log, unsigned long *error); +// plat_spawn with Control Flow Guard and shadow stacks off for the child (the generator: unicorn's longjmp fails +// CFG's check, #337, and can't unwind with a shadow stack, #779). +// *error: 0 = started with CFG off; else why not (then it was started normally, or not at all if that failed too). +// *shadow_stack_error: the same for shadow stacks (when only that is refused, it was started with CFG off alone). +PlatProc plat_spawn_no_cfg(const char *const argv[], const char *log_path, bool append_log, unsigned long *error, + unsigned long *shadow_stack_error); #endif void plat_proc_kill(PlatProc p); // end it and what it started (then it's gone: no wait) bool plat_remove_tree(const char *dir); // rm -rf diff --git a/runtime/platform/windows.c b/runtime/platform/windows.c index 24a454e15..c064ae5fc 100644 --- a/runtime/platform/windows.c +++ b/runtime/platform/windows.c @@ -767,18 +767,26 @@ ssize_t pwrite(int fd, const void *buf, size_t n, off_t off) { return prw(fd, (v // ---- child processes typedef struct { HANDLE process, job; } WinProcess; -// Control Flow Guard off for one child (plat_spawn_no_cfg), as a process mitigation policy: it wins over the -// system-wide Exploit protection setting, which the exe's header (tools/pe_no_cfg.py) does not (#337) +// Control Flow Guard and user-mode shadow stacks off for one child (plat_spawn_no_cfg), as process mitigation +// policies: they win over the system-wide Exploit protection settings, which the exe's header (tools/pe_no_cfg.py) +// does not. unicorn (the generator's jump tables) leaves the code it generated with longjmp from a setjmp that saved +// no frame, and VCRUNTIME140's longjmp stops the process (0xC0000409) for that when CFG is on (#337); with a shadow +// stack it unwinds instead (RtlUnwindEx), through generated code that has no unwind data (#779). #ifndef PROC_THREAD_ATTRIBUTE_MITIGATION_POLICY #define PROC_THREAD_ATTRIBUTE_MITIGATION_POLICY 0x00020007 #endif #ifndef PROCESS_CREATION_MITIGATION_POLICY_CONTROL_FLOW_GUARD_ALWAYS_OFF #define PROCESS_CREATION_MITIGATION_POLICY_CONTROL_FLOW_GUARD_ALWAYS_OFF (0x00000002ull << 40) #endif +#ifndef PROCESS_CREATION_MITIGATION_POLICY2_CET_USER_SHADOW_STACKS_ALWAYS_OFF +#define PROCESS_CREATION_MITIGATION_POLICY2_CET_USER_SHADOW_STACKS_ALWAYS_OFF (0x00000002ull << 28) +#endif // no_cfg: NULL = a plain start; else *no_cfg = 0 when the child runs with CFG off, or the error that prevented it -// (then it is started without the policy) -static PlatProc spawn_child(const char *const argv[], const char *log_path, bool append_log, unsigned long *no_cfg) { +// (then it is started without the policy). *no_ss likewise for shadow stacks: a Windows that doesn't take that +// policy gets CFG off alone. +static PlatProc spawn_child(const char *const argv[], const char *log_path, bool append_log, unsigned long *no_cfg, + unsigned long *no_ss) { static __thread char cmd[32768]; cmd[0] = 0; for (int i = 0; argv[i]; i++) append_arg(cmd, sizeof cmd, argv[i]); @@ -800,29 +808,36 @@ static PlatProc spawn_child(const char *const argv[], const char *log_path, bool const wchar_t *app = GetFullPathNameW(wprog, MAX_PATH, wfull, NULL) ? wfull : NULL; BOOL ok = FALSE; if (no_cfg) { - *no_cfg = ERROR_NOT_SUPPORTED; - STARTUPINFOEXW six = {.StartupInfo = si}; - six.StartupInfo.cb = sizeof six; - DWORD64 policy[2] = {PROCESS_CREATION_MITIGATION_POLICY_CONTROL_FLOW_GUARD_ALWAYS_OFF, 0}; - SIZE_T size = 0; - InitializeProcThreadAttributeList(NULL, 1, 0, &size); - six.lpAttributeList = size ? (LPPROC_THREAD_ATTRIBUTE_LIST)malloc(size) : NULL; - if (!six.lpAttributeList || !InitializeProcThreadAttributeList(six.lpAttributeList, 1, 0, &size)) { - *no_cfg = six.lpAttributeList ? GetLastError() : ERROR_NOT_ENOUGH_MEMORY; - free(six.lpAttributeList); - six.lpAttributeList = NULL; - } else if (!UpdateProcThreadAttribute(six.lpAttributeList, 0, PROC_THREAD_ATTRIBUTE_MITIGATION_POLICY, policy, - sizeof policy, NULL, NULL)) { - *no_cfg = GetLastError(); - } else { - ok = CreateProcessW(app, wcmd, NULL, NULL, TRUE, - CREATE_NO_WINDOW | CREATE_SUSPENDED | EXTENDED_STARTUPINFO_PRESENT, NULL, NULL, - &six.StartupInfo, &pi); - *no_cfg = ok ? 0 : GetLastError(); // (older Windows: ERROR_INVALID_PARAMETER; or a policy refuses it) - } - if (six.lpAttributeList) { - DeleteProcThreadAttributeList(six.lpAttributeList); - free(six.lpAttributeList); + *no_cfg = *no_ss = ERROR_NOT_SUPPORTED; + // first both policies, then CFG alone (a Windows without shadow stack support may refuse the second word) + for (int both = 1; both >= 0 && !ok; both--) { + STARTUPINFOEXW six = {.StartupInfo = si}; + six.StartupInfo.cb = sizeof six; + DWORD64 policy[2] = {PROCESS_CREATION_MITIGATION_POLICY_CONTROL_FLOW_GUARD_ALWAYS_OFF, + both ? PROCESS_CREATION_MITIGATION_POLICY2_CET_USER_SHADOW_STACKS_ALWAYS_OFF : 0}; + unsigned long e; + SIZE_T size = 0; + InitializeProcThreadAttributeList(NULL, 1, 0, &size); + six.lpAttributeList = size ? (LPPROC_THREAD_ATTRIBUTE_LIST)malloc(size) : NULL; + if (!six.lpAttributeList || !InitializeProcThreadAttributeList(six.lpAttributeList, 1, 0, &size)) { + e = six.lpAttributeList ? GetLastError() : ERROR_NOT_ENOUGH_MEMORY; + free(six.lpAttributeList); + six.lpAttributeList = NULL; + } else if (!UpdateProcThreadAttribute(six.lpAttributeList, 0, PROC_THREAD_ATTRIBUTE_MITIGATION_POLICY, + policy, sizeof policy, NULL, NULL)) { + e = GetLastError(); + } else { + ok = CreateProcessW(app, wcmd, NULL, NULL, TRUE, + CREATE_NO_WINDOW | CREATE_SUSPENDED | EXTENDED_STARTUPINFO_PRESENT, NULL, NULL, + &six.StartupInfo, &pi); + e = ok ? 0 : GetLastError(); // (older Windows: ERROR_INVALID_PARAMETER; or a policy refuses it) + } + if (six.lpAttributeList) { + DeleteProcThreadAttributeList(six.lpAttributeList); + free(six.lpAttributeList); + } + if (both) *no_ss = e; + if (ok || !both) *no_cfg = e; } } if (!ok) ok = CreateProcessW(app, wcmd, NULL, NULL, TRUE, CREATE_NO_WINDOW | CREATE_SUSPENDED, NULL, NULL, &si, &pi); @@ -855,13 +870,15 @@ static PlatProc spawn_child(const char *const argv[], const char *log_path, bool } PlatProc plat_spawn(const char *const argv[], const char *log_path, bool append_log) { - return spawn_child(argv, log_path, append_log, NULL); + return spawn_child(argv, log_path, append_log, NULL, NULL); } -PlatProc plat_spawn_no_cfg(const char *const argv[], const char *log_path, bool append_log, unsigned long *error) { - unsigned long e = 0; - PlatProc p = spawn_child(argv, log_path, append_log, &e); +PlatProc plat_spawn_no_cfg(const char *const argv[], const char *log_path, bool append_log, unsigned long *error, + unsigned long *shadow_stack_error) { + unsigned long e = 0, ss = 0; + PlatProc p = spawn_child(argv, log_path, append_log, &e, &ss); if (error) *error = e; + if (shadow_stack_error) *shadow_stack_error = ss; return p; } diff --git a/tools/gen.py b/tools/gen.py index 790aa490b..af1f1c816 100644 --- a/tools/gen.py +++ b/tools/gen.py @@ -89,6 +89,8 @@ def boundaries(self): def emu(self): if self.mu is None: + # (the first unicorn run, for the first jump table: a generator stopped by Windows ends here, #337, #779) + print(f'{self.name}: starting the jump table emulator', file=sys.stderr, flush=True) mu = uc.Uc(uc.UC_ARCH_ARM64, uc.UC_MODE_ARM) n = self.nso mu.mem_map(self.base, len(n.image)) @@ -533,6 +535,35 @@ def write_if_changed(path, text): f.write(text) +def windows_mitigations(): + """Windows: the CPU this runs on and whether Control Flow Guard and shadow stacks are on in this process, for the + log (unicorn's longjmp fails with either: runtime/platform/windows.c spawn_child, #337, #779). '' elsewhere.""" + if os.name != 'nt': + return '' + try: + import ctypes + from ctypes import wintypes + k = ctypes.WinDLL('kernel32', use_last_error=True) + k.GetCurrentProcess.restype = wintypes.HANDLE + me = k.GetCurrentProcess() + machine = {0x8664: 'x64', 0xAA64: 'arm64'} + proc, native = wintypes.USHORT(), wintypes.USHORT() + host = 'x64' + if hasattr(k, 'IsWow64Process2') and k.IsWow64Process2(me, ctypes.byref(proc), ctypes.byref(native)): + host = machine.get(native.value, hex(native.value)) + this = machine.get(0x8664 if platform.machine().lower() in ('amd64', 'x86_64') else 0xAA64) + out = [f'{this} on {host}' + (' (emulated)' if this != host else '')] + for name, policy, on in (('Control Flow Guard', 7, 1), ('shadow stacks', 17, 1), ('strict shadow stacks', 17, 0x10)): + flags = wintypes.DWORD() + if k.GetProcessMitigationPolicy(me, policy, ctypes.byref(flags), ctypes.sizeof(flags)): + out.append(f'{name} {"on" if flags.value & on else "off"}') + else: + out.append(f'{name} unknown (error {ctypes.get_last_error()})') + return ', '.join(out) + except Exception as e: # (only for the log) + return f'mitigations unknown ({e})' + + def main(): ap = argparse.ArgumentParser() ap.add_argument('name') @@ -553,6 +584,9 @@ def main(): help='Android: guest addresses relative to the window the runtime picks (lifter.gaddr)') args = ap.parse_args() lifter.DYN_BASE = args.dyn_base + # (first: a generator that ends early still shows it started, and where it ran) + where = windows_mitigations() + print(f'{args.name}: started' + (f' ({where})' if where else ''), file=sys.stderr, flush=True) t0 = time.time() m = Module(args.name, args.nso, args.base, args.simd) diff --git a/tools/test_gamebuild_ios.py b/tools/test_gamebuild_ios.py index ac403c5a3..61a79a4af 100644 --- a/tools/test_gamebuild_ios.py +++ b/tools/test_gamebuild_ios.py @@ -54,7 +54,7 @@ extern "C" uint64_t plat_now_ns() { return std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count(); } extern "C" void plat_sleep_until_ns(uint64_t t) { auto now=plat_now_ns(); if(t>now)std::this_thread::sleep_for(std::chrono::nanoseconds(t-now)); } extern "C" intptr_t plat_spawn(const char *const *a,const char *l,bool append) { return start_cb(a,l,append); } -extern "C" intptr_t plat_spawn_no_cfg(const char *const *a,const char *l,bool append,unsigned long *e) { if(e)*e=0; return start_cb(a,l,append); } +extern "C" intptr_t plat_spawn_no_cfg(const char *const *a,const char *l,bool append,unsigned long *e,unsigned long *ss) { if(e)*e=0; if(ss)*ss=0; return start_cb(a,l,append); } extern "C" int plat_proc_wait(intptr_t p,bool block) { return wait_cb(p,block); } extern "C" void plat_proc_kill(intptr_t p) { kill_cb(p); } ''' diff --git a/tools/test_gamebuild_resume.py b/tools/test_gamebuild_resume.py index b6398e257..d2c1bb7b5 100644 --- a/tools/test_gamebuild_resume.py +++ b/tools/test_gamebuild_resume.py @@ -44,7 +44,7 @@ extern "C" uint64_t plat_now_ns() { return std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count(); } extern "C" void plat_sleep_until_ns(uint64_t t) { auto now=plat_now_ns(); if(t>now)std::this_thread::sleep_for(std::chrono::nanoseconds(t-now)); } extern "C" intptr_t plat_spawn(const char *const *a,const char *l,bool append) { return start_cb(a,l,append); } -extern "C" intptr_t plat_spawn_no_cfg(const char *const *a,const char *l,bool append,unsigned long *e) { if(e)*e=0; return start_cb(a,l,append); } +extern "C" intptr_t plat_spawn_no_cfg(const char *const *a,const char *l,bool append,unsigned long *e,unsigned long *ss) { if(e)*e=0; if(ss)*ss=0; return start_cb(a,l,append); } extern "C" int plat_proc_wait(intptr_t p,bool block) { return wait_cb(p,block); } extern "C" void plat_proc_kill(intptr_t p) { kill_cb(p); } ''' diff --git a/tools/test_windows_processes.py b/tools/test_windows_processes.py index a25df7fe6..7c647d1c0 100644 --- a/tools/test_windows_processes.py +++ b/tools/test_windows_processes.py @@ -45,10 +45,12 @@ CloseHandle(pi.hThread);CloseHandle(pi.hProcess);Sleep(30000);return 0; } if(argc>1&&!strcmp(argv[1],"--fastfail")) { ExitProcess(0xC0000409u); } - if(argc>1&&!strcmp(argv[1],"--cfg")) { // 0: Control Flow Guard off in this process + if(argc>1&&!strcmp(argv[1],"--cfg")) { // 0: Control Flow Guard and shadow stacks off in this process PROCESS_MITIGATION_CONTROL_FLOW_GUARD_POLICY p={0}; if(!GetProcessMitigationPolicy(GetCurrentProcess(),ProcessControlFlowGuardPolicy,&p,sizeof p)) return 3; - return p.EnableControlFlowGuard?1:0; + DWORD ss=0; // (ProcessUserShadowStackPolicy = 17: bit 0 EnableUserShadowStack) + if(!GetProcessMitigationPolicy(GetCurrentProcess(),(PROCESS_MITIGATION_POLICY)17,&ss,sizeof ss)) return 4; + return p.EnableControlFlowGuard?1:(ss&1)?2:0; } assert(argc==2); char a[1200],b[1200];snprintf(a,sizeof a,"%s/first.pid",argv[1]);snprintf(b,sizeof b,"%s/second.pid",argv[1]); @@ -66,16 +68,16 @@ assert(plat_proc_wait(plat_spawn(crash,NULL,false),true)==255); const char *fastfail[]={exe,"--fastfail",NULL},*cfg[]={exe,"--cfg",NULL}; assert(plat_proc_wait(plat_spawn(fastfail,NULL,false),true)==PLAT_PROC_FAST_FAIL); - unsigned long cfg_error=1; - PlatProc nocfg=plat_spawn_no_cfg(cfg,NULL,false,&cfg_error); - assert(nocfg&&cfg_error==0&&plat_proc_wait(nocfg,true)==0); + unsigned long cfg_error=1,ss_error=1; + PlatProc nocfg=plat_spawn_no_cfg(cfg,NULL,false,&cfg_error,&ss_error); + assert(nocfg&&cfg_error==0&&ss_error==0&&plat_proc_wait(nocfg,true)==0); reject_job=true; assert(plat_proc_wait(plat_spawn(code,NULL,false),true)==7); const char *sleeper[]={exe,"--sleep",NULL}; PlatProc fallback=plat_spawn(sleeper,NULL,false); assert(fallback && !((WinProcess *)fallback)->job && plat_proc_wait(fallback,false)==-1); plat_proc_kill(fallback); - puts("Windows process trees: isolated cancellation, crash status, CFG-off start and fallback without job assignment passed"); + puts("Windows process trees: isolated cancellation, crash status, CFG- and shadow-stack-off start and fallback without job assignment passed"); return 0; } '''