android: A57 — nice -20 through RLIMIT_NICE, without CAP_SYS_NICE

android/patches/gamescope/0053-A57-nice-via-rlimit-nice.patch

@@ -0,0 +1,174 @@
+A57 — nice -20 when RLIMIT_NICE allows it, without CAP_SYS_NICE
+
+gamescope only lowers its nice value when it holds CAP_SYS_NICE. An Android
+app never does, so every gamescope thread ran at nice 0 ("No CAP_SYS_NICE,
+falling back to regular-priority compute and threads") while the game's own
+threads ran at nice -5 and -2 and Wine's audio threads at -19. On a Retroid
+Pocket 6 (Android 13, Dave the Diver at a 60 cap, real-vsync daemon) about 1 %
+of frames missed their vblank: 23 of 25 late posts over 44 s were gamescope
+draw spikes, gamescope's own commit coming 3.9 to 10.8 ms late because
+gamescope-xwm was runnable but not scheduled for 1 to 8 ms.
+
+The same device grants RLIMIT_NICE = 40 (soft and hard) to the app and every
+child, which is how Linux lets an unprivileged process lower its nice value:
+setpriority succeeds down to 20 - rlim_cur, so nice -20 here. Realtime is not
+granted (RLIMIT_RTPRIO is 0).
+
+- CanSetNice(n): CAP_SYS_NICE, or an RLIMIT_NICE soft limit that reaches n
+  (20 - rlim_cur <= n, or RLIM_INFINITY). SetNice uses it.
+- main sets nice -20 on that path too. Threads created after it
+  (gamescope-xwm, -wl, -kms, -wait, -vrmon, the vblank thread) inherit it.
+  SetRealtime stays on CAP_SYS_NICE only.
+- nice() only changes the calling thread, and two waiter threads are started
+  by static constructors before main: g_ImageWaiter ("gamescope_img",
+  steamcompmgr.cpp) and g_LibEisWaiter ("gamescope-eis", wlserver.cpp). They
+  stayed at nice 0 (upstream has the same gap with CAP_SYS_NICE). On the RP6
+  (SteamWorld Dig 2, 30 s live capture) gamescope_img, the thread that marks
+  an Xwayland commit done, first ran 15.5 ms after the commit arrived, on a
+  CPU 67 % busy with app threads; the commit was not done at the next wakeup
+  and gamescope skipped a whole period. SetNice now also gives the new value
+  to every other thread of the process that is still at the old one.
+- RestoreNice no longer needs CAP_SYS_NICE: raising the value back needs no
+  permission, and SetNice only records values when it lowered them. The game
+  and the other children gamescope spawns still start at the nice gamescope
+  was started with. Xwayland is forked by wlroots (xwayland/server.c), not
+  through Process::SpawnProcess, so it inherits nice -20, as it does upstream
+  on a desktop with CAP_SYS_NICE.
+- The Vulkan queues are unchanged: VK_QUEUE_GLOBAL_PRIORITY_REALTIME is still
+  requested only with CAP_SYS_NICE (rendervulkan.cpp CreateDevice, with its
+  VK_ERROR_NOT_PERMITTED_KHR fallback), so without it both queues stay at
+  regular priority as before.
+
+Logs: "No CAP_SYS_NICE; RLIMIT_NICE permits nice -20 (threads at nice -20),
+realtime compute queue and scheduling unavailable." once at start, instead of
+the "falling back to regular-priority" line, and "Nice -20 also applied to
+N thread(s) started before it was set." when earlier threads were moved.
+
+diff --git a/src/Utils/Process.cpp b/src/Utils/Process.cpp
+index 1a82f57..b94ae48 100644
+--- a/src/Utils/Process.cpp
++++ b/src/Utils/Process.cpp
+@@ -900,12 +900,70 @@ namespace gamescope::Process
+ #endif
+     }
+
++    static bool RlimitNiceAllows( int nNice )
++    {
++#if defined(__linux__)
++        // Without CAP_SYS_NICE, setpriority may still lower the nice value down to
++        // 20 - RLIMIT_NICE (the soft limit). This is how an unprivileged process is
++        // granted a better priority without the capability.
++        struct rlimit niceLimit{};
++        if ( getrlimit( RLIMIT_NICE, &niceLimit ) != 0 )
++            return false;
++
++        if ( niceLimit.rlim_cur == RLIM_INFINITY )
++            return true;
++
++        return 20 - static_cast<long long>( niceLimit.rlim_cur ) <= nNice;
++#else
++        return false;
++#endif
++    }
++
++    bool CanSetNice( int nNice )
++    {
++        return HasCapSysNice() || RlimitNiceAllows( nNice );
++    }
++
++    // nice() only changes the calling thread. Threads that already exist keep the
++    // nice they were started with, and some are started by static constructors
++    // before main gets here (g_ImageWaiter "gamescope_img", "gamescope-eis").
++    // Give every other thread still at the old value the new one.
++    static void ApplyNiceToEarlierThreads( int nOldNice, int nNewNice )
++    {
++#if defined(__linux__)
++        DIR *pTaskDir = opendir( "/proc/self/task" );
++        if ( !pTaskDir )
++            return;
++        defer( closedir( pTaskDir ) );
++
++        const pid_t nSelf = gettid();
++        int nApplied = 0;
++        while ( struct dirent *pEntry = readdir( pTaskDir ) )
++        {
++            std::optional<pid_t> onTid = Parse<pid_t>( pEntry->d_name );
++            if ( !onTid || *onTid == nSelf )
++                continue;
++
++            errno = 0;
++            int nThreadNice = getpriority( PRIO_PROCESS, *onTid );
++            if ( errno != 0 || nThreadNice != nOldNice )
++                continue;
++
++            if ( setpriority( PRIO_PROCESS, *onTid, nNewNice ) == 0 )
++                nApplied++;
++        }
++
++        if ( nApplied )
++            s_ProcessLog.infof( "Nice %d also applied to %d thread(s) started before it was set.", nNewNice, nApplied );
++#endif
++    }
++
+     std::optional<int> g_oOldNice;
+     std::optional<int> g_oNewNice;
+     void SetNice( int nNice )
+     {
+ #if defined(__linux__)
+-        if ( !HasCapSysNice() )
++        if ( !CanSetNice( nNice ) )
+             return;
+
+         errno = 0;
+@@ -920,6 +978,9 @@ namespace gamescope::Process
+         if ( nNewNice != -1 || errno == 0 )
+         {
+             g_oNewNice = nNewNice;
++
++            if ( g_oOldNice && *g_oOldNice != nNewNice )
++                ApplyNiceToEarlierThreads( *g_oOldNice, nNewNice );
+         }
+ #endif
+     }
+@@ -927,9 +988,9 @@ namespace gamescope::Process
+     void RestoreNice()
+     {
+ #if defined(__linux__)
+-        if ( !HasCapSysNice() )
+-            return;
+-
++        // Raising the nice value back needs no permission, and SetNice only
++        // records values when it was allowed to lower it (by CAP_SYS_NICE or
++        // RLIMIT_NICE), so children must be restored either way.
+         if ( !g_oOldNice || !g_oNewNice )
+             return;
+
+diff --git a/src/Utils/Process.h b/src/Utils/Process.h
+index 1828bce..5456233 100644
+--- a/src/Utils/Process.h
++++ b/src/Utils/Process.h
+@@ -78,6 +78,7 @@ namespace gamescope::Process
+     pid_t SpawnProcessInWatchdog( char **argv, bool bRespawn = false, std::function<void()> fnPreambleInChild = nullptr );
+
+     bool HasCapSysNice();
++    bool CanSetNice( int nNice );
+     void SetNice( int nNice );
+     void RestoreNice();
+
+diff --git a/src/main.cpp b/src/main.cpp
+index 4b51a16..0d13d0b 100644
+--- a/src/main.cpp
++++ b/src/main.cpp
+@@ -885,6 +885,12 @@ int main(int argc, char **argv)
+ 		if ( g_bRt )
+ 			gamescope::Process::SetRealtime();
+ 	}
++	else if ( gamescope::Process::CanSetNice( -20 ) )
++	{
++		gamescope::Process::SetNice( -20 );
++
++		fprintf( stderr, "No CAP_SYS_NICE; RLIMIT_NICE permits nice -20 (threads at nice %d), realtime compute queue and scheduling unavailable.\n", getpriority( PRIO_PROCESS, 0 ) );
++	}
+ 	else
+ 	{
+ 		fprintf( stderr, "No CAP_SYS_NICE, falling back to regular-priority compute and threads.\nPerformance will be affected.\n" );