Windows: start the generator with shadow stacks off too, and log where it got (#779)

runtime/gamebuild.c

@@ -75,14 +75,16 @@ static PlatProc spawn_proc(char **argv, const char *log, bool generator) {
     PlatProc p = 0;
     if (!g_quitting) {
 #ifdef _WIN32
-        // The generator runs with Control Flow Guard off even when Windows forces it on for every program (#337)
+        // The generator runs with Control Flow Guard and shadow stacks off even when Windows forces them on for every
+        // program (#337, #779)
         if (generator) {
-            unsigned long error = 0;
-            p = plat_spawn_no_cfg((const char *const *)argv, log, false, &error);
+            unsigned long error = 0, ss_error = 0;
+            p = plat_spawn_no_cfg((const char *const *)argv, log, false, &error, &ss_error);
             static bool told;
             if (p && !told) {
                 told = true;
-                if (!error) LOG("[gamebuild] generator started with Control Flow Guard off\n");
+                if (!error && !ss_error) LOG("[gamebuild] generator started with Control Flow Guard and shadow stacks off\n");
+                else if (!error) LOG("[gamebuild] generator started with Control Flow Guard off: shadow stacks could not be turned off for it (error %lu)\n", ss_error);
                 else LOG("[gamebuild] generator started normally: Control Flow Guard could not be turned off for it (error %lu)\n", error);
             }
         } else

runtime/platform.h

@@ -160,9 +160,12 @@ int plat_proc_wait(PlatProc p, bool block);  // exit code, or -1 while still run
 #define PLAT_PROC_SIGNALED 256               // + N: ended by signal N instead of exiting (not on Windows)
 #define PLAT_PROC_FAST_FAIL 512              // Windows: ended by a security check (0xC0000409, e.g. Control Flow Guard)
 #ifdef _WIN32
-// plat_spawn with Control Flow Guard off for the child (the generator: unicorn's longjmp fails CFG's check, #337).
-// *error: 0 = started that way; else why not (then it was started normally, or not at all if that failed too)
-PlatProc plat_spawn_no_cfg(const char *const argv[], const char *log_path, bool append_log, unsigned long *error);
+// plat_spawn with Control Flow Guard and shadow stacks off for the child (the generator: unicorn's longjmp fails
+// CFG's check, #337, and can't unwind with a shadow stack, #779).
+// *error: 0 = started with CFG off; else why not (then it was started normally, or not at all if that failed too).
+// *shadow_stack_error: the same for shadow stacks (when only that is refused, it was started with CFG off alone).
+PlatProc plat_spawn_no_cfg(const char *const argv[], const char *log_path, bool append_log, unsigned long *error,
+                           unsigned long *shadow_stack_error);
 #endif
 void plat_proc_kill(PlatProc p);             // end it and what it started (then it's gone: no wait)
 bool plat_remove_tree(const char *dir);      // rm -rf

runtime/platform/windows.c

@@ -767,18 +767,26 @@ ssize_t pwrite(int fd, const void *buf, size_t n, off_t off) { return prw(fd, (v
 // ---- child processes
 typedef struct { HANDLE process, job; } WinProcess;
 
-// Control Flow Guard off for one child (plat_spawn_no_cfg), as a process mitigation policy: it wins over the
-// system-wide Exploit protection setting, which the exe's header (tools/pe_no_cfg.py) does not (#337)
+// Control Flow Guard and user-mode shadow stacks off for one child (plat_spawn_no_cfg), as process mitigation
+// policies: they win over the system-wide Exploit protection settings, which the exe's header (tools/pe_no_cfg.py)
+// does not. unicorn (the generator's jump tables) leaves the code it generated with longjmp from a setjmp that saved
+// no frame, and VCRUNTIME140's longjmp stops the process (0xC0000409) for that when CFG is on (#337); with a shadow
+// stack it unwinds instead (RtlUnwindEx), through generated code that has no unwind data (#779).
 #ifndef PROC_THREAD_ATTRIBUTE_MITIGATION_POLICY
 #define PROC_THREAD_ATTRIBUTE_MITIGATION_POLICY 0x00020007
 #endif
 #ifndef PROCESS_CREATION_MITIGATION_POLICY_CONTROL_FLOW_GUARD_ALWAYS_OFF
 #define PROCESS_CREATION_MITIGATION_POLICY_CONTROL_FLOW_GUARD_ALWAYS_OFF (0x00000002ull << 40)
 #endif
+#ifndef PROCESS_CREATION_MITIGATION_POLICY2_CET_USER_SHADOW_STACKS_ALWAYS_OFF
+#define PROCESS_CREATION_MITIGATION_POLICY2_CET_USER_SHADOW_STACKS_ALWAYS_OFF (0x00000002ull << 28)
+#endif
 
 // no_cfg: NULL = a plain start; else *no_cfg = 0 when the child runs with CFG off, or the error that prevented it
-// (then it is started without the policy)
-static PlatProc spawn_child(const char *const argv[], const char *log_path, bool append_log, unsigned long *no_cfg) {
+// (then it is started without the policy). *no_ss likewise for shadow stacks: a Windows that doesn't take that
+// policy gets CFG off alone.
+static PlatProc spawn_child(const char *const argv[], const char *log_path, bool append_log, unsigned long *no_cfg,
+                            unsigned long *no_ss) {
     static __thread char cmd[32768];
     cmd[0] = 0;
     for (int i = 0; argv[i]; i++) append_arg(cmd, sizeof cmd, argv[i]);
@@ -800,29 +808,36 @@ static PlatProc spawn_child(const char *const argv[], const char *log_path, bool
     const wchar_t *app = GetFullPathNameW(wprog, MAX_PATH, wfull, NULL) ? wfull : NULL;
     BOOL ok = FALSE;
     if (no_cfg) {
-        *no_cfg = ERROR_NOT_SUPPORTED;
-        STARTUPINFOEXW six = {.StartupInfo = si};
-        six.StartupInfo.cb = sizeof six;
-        DWORD64 policy[2] = {PROCESS_CREATION_MITIGATION_POLICY_CONTROL_FLOW_GUARD_ALWAYS_OFF, 0};
-        SIZE_T size = 0;
-        InitializeProcThreadAttributeList(NULL, 1, 0, &size);
-        six.lpAttributeList = size ? (LPPROC_THREAD_ATTRIBUTE_LIST)malloc(size) : NULL;
-        if (!six.lpAttributeList || !InitializeProcThreadAttributeList(six.lpAttributeList, 1, 0, &size)) {
-            *no_cfg = six.lpAttributeList ? GetLastError() : ERROR_NOT_ENOUGH_MEMORY;
-            free(six.lpAttributeList);
-            six.lpAttributeList = NULL;
-        } else if (!UpdateProcThreadAttribute(six.lpAttributeList, 0, PROC_THREAD_ATTRIBUTE_MITIGATION_POLICY, policy,
-                                              sizeof policy, NULL, NULL)) {
-            *no_cfg = GetLastError();
-        } else {
-            ok = CreateProcessW(app, wcmd, NULL, NULL, TRUE,
-                                CREATE_NO_WINDOW | CREATE_SUSPENDED | EXTENDED_STARTUPINFO_PRESENT, NULL, NULL,
-                                &six.StartupInfo, &pi);
-            *no_cfg = ok ? 0 : GetLastError();  // (older Windows: ERROR_INVALID_PARAMETER; or a policy refuses it)
-        }
-        if (six.lpAttributeList) {
-            DeleteProcThreadAttributeList(six.lpAttributeList);
-            free(six.lpAttributeList);
+        *no_cfg = *no_ss = ERROR_NOT_SUPPORTED;
+        // first both policies, then CFG alone (a Windows without shadow stack support may refuse the second word)
+        for (int both = 1; both >= 0 && !ok; both--) {
+            STARTUPINFOEXW six = {.StartupInfo = si};
+            six.StartupInfo.cb = sizeof six;
+            DWORD64 policy[2] = {PROCESS_CREATION_MITIGATION_POLICY_CONTROL_FLOW_GUARD_ALWAYS_OFF,
+                                 both ? PROCESS_CREATION_MITIGATION_POLICY2_CET_USER_SHADOW_STACKS_ALWAYS_OFF : 0};
+            unsigned long e;
+            SIZE_T size = 0;
+            InitializeProcThreadAttributeList(NULL, 1, 0, &size);
+            six.lpAttributeList = size ? (LPPROC_THREAD_ATTRIBUTE_LIST)malloc(size) : NULL;
+            if (!six.lpAttributeList || !InitializeProcThreadAttributeList(six.lpAttributeList, 1, 0, &size)) {
+                e = six.lpAttributeList ? GetLastError() : ERROR_NOT_ENOUGH_MEMORY;
+                free(six.lpAttributeList);
+                six.lpAttributeList = NULL;
+            } else if (!UpdateProcThreadAttribute(six.lpAttributeList, 0, PROC_THREAD_ATTRIBUTE_MITIGATION_POLICY,
+                                                  policy, sizeof policy, NULL, NULL)) {
+                e = GetLastError();
+            } else {
+                ok = CreateProcessW(app, wcmd, NULL, NULL, TRUE,
+                                    CREATE_NO_WINDOW | CREATE_SUSPENDED | EXTENDED_STARTUPINFO_PRESENT, NULL, NULL,
+                                    &six.StartupInfo, &pi);
+                e = ok ? 0 : GetLastError();  // (older Windows: ERROR_INVALID_PARAMETER; or a policy refuses it)
+            }
+            if (six.lpAttributeList) {
+                DeleteProcThreadAttributeList(six.lpAttributeList);
+                free(six.lpAttributeList);
+            }
+            if (both) *no_ss = e;
+            if (ok || !both) *no_cfg = e;
         }
     }
     if (!ok) ok = CreateProcessW(app, wcmd, NULL, NULL, TRUE, CREATE_NO_WINDOW | CREATE_SUSPENDED, NULL, NULL, &si, &pi);
@@ -855,13 +870,15 @@ static PlatProc spawn_child(const char *const argv[], const char *log_path, bool
 }
 
 PlatProc plat_spawn(const char *const argv[], const char *log_path, bool append_log) {
-    return spawn_child(argv, log_path, append_log, NULL);
+    return spawn_child(argv, log_path, append_log, NULL, NULL);
 }
 
-PlatProc plat_spawn_no_cfg(const char *const argv[], const char *log_path, bool append_log, unsigned long *error) {
-    unsigned long e = 0;
-    PlatProc p = spawn_child(argv, log_path, append_log, &e);
+PlatProc plat_spawn_no_cfg(const char *const argv[], const char *log_path, bool append_log, unsigned long *error,
+                           unsigned long *shadow_stack_error) {
+    unsigned long e = 0, ss = 0;
+    PlatProc p = spawn_child(argv, log_path, append_log, &e, &ss);
     if (error) *error = e;
+    if (shadow_stack_error) *shadow_stack_error = ss;
     return p;
 }
 

tools/gen.py

@@ -89,6 +89,8 @@ def boundaries(self):
 
     def emu(self):
         if self.mu is None:
+            # (the first unicorn run, for the first jump table: a generator stopped by Windows ends here, #337, #779)
+            print(f'{self.name}: starting the jump table emulator', file=sys.stderr, flush=True)
             mu = uc.Uc(uc.UC_ARCH_ARM64, uc.UC_MODE_ARM)
             n = self.nso
             mu.mem_map(self.base, len(n.image))
@@ -533,6 +535,35 @@ def write_if_changed(path, text):
         f.write(text)
 
 
+def windows_mitigations():
+    """Windows: the CPU this runs on and whether Control Flow Guard and shadow stacks are on in this process, for the
+    log (unicorn's longjmp fails with either: runtime/platform/windows.c spawn_child, #337, #779). '' elsewhere."""
+    if os.name != 'nt':
+        return ''
+    try:
+        import ctypes
+        from ctypes import wintypes
+        k = ctypes.WinDLL('kernel32', use_last_error=True)
+        k.GetCurrentProcess.restype = wintypes.HANDLE
+        me = k.GetCurrentProcess()
+        machine = {0x8664: 'x64', 0xAA64: 'arm64'}
+        proc, native = wintypes.USHORT(), wintypes.USHORT()
+        host = 'x64'
+        if hasattr(k, 'IsWow64Process2') and k.IsWow64Process2(me, ctypes.byref(proc), ctypes.byref(native)):
+            host = machine.get(native.value, hex(native.value))
+        this = machine.get(0x8664 if platform.machine().lower() in ('amd64', 'x86_64') else 0xAA64)
+        out = [f'{this} on {host}' + (' (emulated)' if this != host else '')]
+        for name, policy, on in (('Control Flow Guard', 7, 1), ('shadow stacks', 17, 1), ('strict shadow stacks', 17, 0x10)):
+            flags = wintypes.DWORD()
+            if k.GetProcessMitigationPolicy(me, policy, ctypes.byref(flags), ctypes.sizeof(flags)):
+                out.append(f'{name} {"on" if flags.value & on else "off"}')
+            else:
+                out.append(f'{name} unknown (error {ctypes.get_last_error()})')
+        return ', '.join(out)
+    except Exception as e:  # (only for the log)
+        return f'mitigations unknown ({e})'
+
+
 def main():
     ap = argparse.ArgumentParser()
     ap.add_argument('name')
@@ -553,6 +584,9 @@ def main():
                     help='Android: guest addresses relative to the window the runtime picks (lifter.gaddr)')
     args = ap.parse_args()
     lifter.DYN_BASE = args.dyn_base
+    # (first: a generator that ends early still shows it started, and where it ran)
+    where = windows_mitigations()
+    print(f'{args.name}: started' + (f' ({where})' if where else ''), file=sys.stderr, flush=True)
 
     t0 = time.time()
     m = Module(args.name, args.nso, args.base, args.simd)

tools/test_gamebuild_ios.py

@@ -54,7 +54,7 @@
 extern "C" uint64_t plat_now_ns() { return std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now().time_since_epoch()).count(); }
 extern "C" void plat_sleep_until_ns(uint64_t t) { auto now=plat_now_ns(); if(t>now)std::this_thread::sleep_for(std::chrono::nanoseconds(t-now)); }
 extern "C" intptr_t plat_spawn(const char *const *a,const char *l,bool append) { return start_cb(a,l,append); }
-extern "C" intptr_t plat_spawn_no_cfg(const char *const *a,const char *l,bool append,unsigned long *e) { if(e)*e=0; return start_cb(a,l,append); }
+extern "C" intptr_t plat_spawn_no_cfg(const char *const *a,const char *l,bool append,unsigned long *e,unsigned long *ss) { if(e)*e=0; if(ss)*ss=0; return start_cb(a,l,append); }
 extern "C" int plat_proc_wait(intptr_t p,bool block) { return wait_cb(p,block); }
 extern "C" void plat_proc_kill(intptr_t p) { kill_cb(p); }
 '''

tools/test_gamebuild_resume.py

@@ -44,7 +44,7 @@
 extern "C" uint64_t plat_now_ns() { return std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now().time_since_epoch()).count(); }
 extern "C" void plat_sleep_until_ns(uint64_t t) { auto now=plat_now_ns(); if(t>now)std::this_thread::sleep_for(std::chrono::nanoseconds(t-now)); }
 extern "C" intptr_t plat_spawn(const char *const *a,const char *l,bool append) { return start_cb(a,l,append); }
-extern "C" intptr_t plat_spawn_no_cfg(const char *const *a,const char *l,bool append,unsigned long *e) { if(e)*e=0; return start_cb(a,l,append); }
+extern "C" intptr_t plat_spawn_no_cfg(const char *const *a,const char *l,bool append,unsigned long *e,unsigned long *ss) { if(e)*e=0; if(ss)*ss=0; return start_cb(a,l,append); }
 extern "C" int plat_proc_wait(intptr_t p,bool block) { return wait_cb(p,block); }
 extern "C" void plat_proc_kill(intptr_t p) { kill_cb(p); }
 '''

tools/test_windows_processes.py

@@ -45,10 +45,12 @@
         CloseHandle(pi.hThread);CloseHandle(pi.hProcess);Sleep(30000);return 0;
     }
     if(argc>1&&!strcmp(argv[1],"--fastfail")) { ExitProcess(0xC0000409u); }
-    if(argc>1&&!strcmp(argv[1],"--cfg")) {  // 0: Control Flow Guard off in this process
+    if(argc>1&&!strcmp(argv[1],"--cfg")) {  // 0: Control Flow Guard and shadow stacks off in this process
         PROCESS_MITIGATION_CONTROL_FLOW_GUARD_POLICY p={0};
         if(!GetProcessMitigationPolicy(GetCurrentProcess(),ProcessControlFlowGuardPolicy,&p,sizeof p)) return 3;
-        return p.EnableControlFlowGuard?1:0;
+        DWORD ss=0;  // (ProcessUserShadowStackPolicy = 17: bit 0 EnableUserShadowStack)
+        if(!GetProcessMitigationPolicy(GetCurrentProcess(),(PROCESS_MITIGATION_POLICY)17,&ss,sizeof ss)) return 4;
+        return p.EnableControlFlowGuard?1:(ss&1)?2:0;
     }
     assert(argc==2);
     char a[1200],b[1200];snprintf(a,sizeof a,"%s/first.pid",argv[1]);snprintf(b,sizeof b,"%s/second.pid",argv[1]);
@@ -66,16 +68,16 @@
     assert(plat_proc_wait(plat_spawn(crash,NULL,false),true)==255);
     const char *fastfail[]={exe,"--fastfail",NULL},*cfg[]={exe,"--cfg",NULL};
     assert(plat_proc_wait(plat_spawn(fastfail,NULL,false),true)==PLAT_PROC_FAST_FAIL);
-    unsigned long cfg_error=1;
-    PlatProc nocfg=plat_spawn_no_cfg(cfg,NULL,false,&cfg_error);
-    assert(nocfg&&cfg_error==0&&plat_proc_wait(nocfg,true)==0);
+    unsigned long cfg_error=1,ss_error=1;
+    PlatProc nocfg=plat_spawn_no_cfg(cfg,NULL,false,&cfg_error,&ss_error);
+    assert(nocfg&&cfg_error==0&&ss_error==0&&plat_proc_wait(nocfg,true)==0);
     reject_job=true;
     assert(plat_proc_wait(plat_spawn(code,NULL,false),true)==7);
     const char *sleeper[]={exe,"--sleep",NULL};
     PlatProc fallback=plat_spawn(sleeper,NULL,false);
     assert(fallback && !((WinProcess *)fallback)->job && plat_proc_wait(fallback,false)==-1);
     plat_proc_kill(fallback);
-    puts("Windows process trees: isolated cancellation, crash status, CFG-off start and fallback without job assignment passed");
+    puts("Windows process trees: isolated cancellation, crash status, CFG- and shadow-stack-off start and fallback without job assignment passed");
     return 0;
 }
 '''