SurfaceFlinger renderer: RGBA_8888 scan-out slots under the daemon's layout rule

app/src/main/cpp/asurfacerenderer/ahbimage.c

@@ -11,6 +11,9 @@
 #define LOGD(...) __android_log_print(ANDROID_LOG_DEBUG,  LOG_TAG, __VA_ARGS__)
 
 #define AHARDWAREBUFFER_FORMAT_B8G8R8A8_UNORM 5
+/* HAL_PIXEL_FORMAT_RGBA_8888: the only 32-bit layout every SurfaceFlinger render engine can
+ * wrap; AHARDWAREBUFFER_FORMAT_R8G8B8A8_UNORM in the NDK header. */
+#define AHB_FORMAT_RGBA_8888 1
 
 static void dump_ahb_usage(uint64_t usage)
 {
@@ -170,10 +173,19 @@ Java_com_winlator_renderer_AHBImage_hardwareBufferFromSocket(
     return (jlong)(uintptr_t)ahb;
 }
 
+/* The X server writes B,G,R,A bytes; an RGBA_8888 slot wants R,G,B,A. */
+static void copy_row_swap_rb(uint32_t *dst, const uint32_t *src, size_t count)
+{
+    for (size_t i = 0; i < count; i++) {
+        uint32_t p = src[i];
+        dst[i] = (p & 0xff00ff00u) | ((p & 0x00ff0000u) >> 16) | ((p & 0x000000ffu) << 16);
+    }
+}
+
 JNIEXPORT jint JNICALL
 Java_com_winlator_renderer_AHBImage_copyHardwareBuffer(
         JNIEnv *env, jobject obj, jobject srcBuffer, jlong dstPtr,
-        jshort width, jshort height, jshort srcStride, jint waitFence)
+        jshort width, jshort height, jshort srcStride, jint waitFence, jboolean swapRB)
 {
     uint32_t* srcAddr = (uint32_t*)(*env)->GetDirectBufferAddress(env, srcBuffer);
     jlong srcCapacity = (*env)->GetDirectBufferCapacity(env, srcBuffer);
@@ -206,8 +218,19 @@ Java_com_winlator_renderer_AHBImage_copyHardwareBuffer(
         return -1;
     }
 
-    if ((uint32_t)width == (uint32_t)srcStride &&
-        (uint32_t)width == dstStride) {
+    if (swapRB) {
+        if ((uint32_t)width == (uint32_t)srcStride &&
+            (uint32_t)width == dstStride) {
+            copy_row_swap_rb(dstAddr, srcAddr, (size_t)width * (size_t)height);
+        } else {
+            for (int y = 0; y < height; y++) {
+                copy_row_swap_rb(dstAddr + (size_t)y * dstStride,
+                                 srcAddr + (size_t)y * (uint32_t)srcStride,
+                                 (size_t)width);
+            }
+        }
+    } else if ((uint32_t)width == (uint32_t)srcStride &&
+               (uint32_t)width == dstStride) {
         memcpy(dstAddr, srcAddr, (size_t)width * (size_t)height * 4);
     } else {
         for (int y = 0; y < height; y++) {
@@ -229,7 +252,7 @@ Java_com_winlator_renderer_AHBImage_copyHardwareBuffer(
 
 JNIEXPORT jlong JNICALL
 Java_com_winlator_renderer_AHBImage_createHardwareBuffer(
-        JNIEnv *env, jobject obj, jshort width, jshort height)
+        JNIEnv *env, jobject obj, jshort width, jshort height, jboolean rgba)
 {
     if (width <= 0 || height <= 0) {
         LOGE("nativeCreateHardwareBuffer: invalid dimensions %d x %d", width, height);
@@ -241,7 +264,7 @@ Java_com_winlator_renderer_AHBImage_createHardwareBuffer(
     desc.width  = (uint32_t)width;
     desc.height = (uint32_t)height;
     desc.layers = 1;
-    desc.format = AHARDWAREBUFFER_FORMAT_B8G8R8A8_UNORM;
+    desc.format = rgba ? AHB_FORMAT_RGBA_8888 : AHARDWAREBUFFER_FORMAT_B8G8R8A8_UNORM;
     desc.usage  = AHARDWAREBUFFER_USAGE_GPU_SAMPLED_IMAGE
                   | AHARDWAREBUFFER_USAGE_CPU_WRITE_OFTEN
                   | AHARDWAREBUFFER_USAGE_GPU_COLOR_OUTPUT

app/src/main/java/app/gamenative/ui/screen/xserver/XServerScreen.kt

@@ -217,6 +217,7 @@ import com.winlator.widget.FrameRating
 import com.winlator.widget.InputControlsView
 import com.winlator.widget.TouchpadView
 import com.winlator.renderer.ASurfaceRenderer
+import com.winlator.renderer.AHBImage
 import com.winlator.renderer.GLRenderer
 import com.winlator.renderer.VulkanRenderer
 import com.winlator.widget.XServerRendererView
@@ -262,6 +263,8 @@ import kotlinx.coroutines.withTimeoutOrNull
 import org.json.JSONException
 import org.json.JSONObject
 import timber.log.Timber
+import android.system.Os
+import android.system.ErrnoException
 import java.io.File
 import java.io.IOException
 import java.nio.file.Files
@@ -2128,6 +2131,9 @@ fun XServerScreen(
             // other containers as well. Uncheck the per-container useLegacyRenderer
             // setting to switch to the Vulkan renderer.
             val useGLRenderer = isX11Display && (container.graphicsDriver == "virgl" || container.displayRenderer.equals("gl", true))
+            // Before the first drawable: the SurfaceFlinger display renderer hands its window buffers to
+            // surfaceflinger, so they follow the same scan-out layout rule as the gamescope daemon's pool.
+            if (isX11Display) AHBImage.setCpuScanoutRgba(resolveScanoutLayout(container) == "rgba")
             val xServerViewInstance: XServerRendererView = if (useGLRenderer) {
                 XServerViewGL(context, xServerToUse)
             } else {
@@ -6077,6 +6083,29 @@ internal fun resolveDisplayX11(container: Container?): Boolean {
     return container == null || Container.isX11Display(container.containerVariant, container.displayServer)
 }
 
+/**
+ * The scan-out layout for buffers this process hands to SurfaceFlinger, "bgra" or "rgba": the container's
+ * GNDRM_SCANOUT_LAYOUT override, else the gndrm daemon's device rule (RGBA on Android 16 with a GLES render
+ * engine, which aborts surfaceflinger on a BGRA_8888 layer). The override is put in this process's
+ * environment first, the way the daemon reads it, and cleared otherwise so an earlier session's does not linger.
+ */
+internal fun resolveScanoutLayout(container: Container): String {
+    val override = EnvVars(container.envVars).get("GNDRM_SCANOUT_LAYOUT")
+    try {
+        if (override.isNotEmpty()) Os.setenv("GNDRM_SCANOUT_LAYOUT", override, true) else Os.unsetenv("GNDRM_SCANOUT_LAYOUT")
+    } catch (e: ErrnoException) {
+        Timber.w(e, "Could not set GNDRM_SCANOUT_LAYOUT")
+    }
+    val layout = try {
+        GameNativeDrm.nativeScanoutLayout() ?: "bgra"
+    } catch (e: LinkageError) {
+        Timber.w(e, "Scan-out layout rule unavailable, assuming bgra")
+        "bgra"
+    }
+    Timber.i("X11 scan-out layout %s (%s)", layout, if (override.isNotEmpty()) "container override" else "device rule")
+    return layout
+}
+
 private fun refreshComponentsFiles(context: Context) {
     val extractionPairs = listOf(
         "pulseaudio-gamenative-20260919.tzst" to File(context.filesDir, "pulseaudio"),

app/src/main/java/com/winlator/renderer/AHBImage.java

@@ -18,13 +18,37 @@ public class AHBImage extends NativeTexture {
     private int swapchainIndex = 0;
     private long preparedRendererGeneration = -1;
     private static boolean supported = false;
+    /**
+     * Scan-out layout of the CPU swapchain slots handed to SurfaceFlinger, the same rule the gndrm daemon
+     * applies to its pool: RGBA_8888 where the render engine cannot wrap a BGRA_8888 buffer (Android 16 on
+     * Skia GLES aborts surfaceflinger on one), BGRA_8888 elsewhere. The X server keeps writing B,G,R,A bytes
+     * into the locked front buffer; an RGBA slot gets them swapped on the copy.
+     */
+    private static volatile boolean cpuScanoutRgba = false;
+    private final boolean scanoutRgba;
+
+    /** Set before a session's first drawable: every AHBImage created afterwards follows it. */
+    public static void setCpuScanoutRgba(boolean rgba) {
+        cpuScanoutRgba = rgba;
+    }
+
+    public static boolean isCpuScanoutRgba() {
+        return cpuScanoutRgba;
+    }
+
+    /** Whether this image's scan-out slots are RGBA_8888 with R,G,B,A bytes (no conversion needed). */
+    public boolean isScanoutRgba() {
+        return scanoutRgba;
+    }
 
     static {
         System.loadLibrary("ahbimage");
     }
 
     public AHBImage(short width, short height) {
-        hardwareBufferPtr = createHardwareBuffer(width, height);
+        scanoutRgba = cpuScanoutRgba;
+        // The front buffer is only ever CPU-written and read back; its declared format does not matter.
+        hardwareBufferPtr = createHardwareBuffer(width, height, false);
         if (hardwareBufferPtr == 0) {
             System.err.println("Error: Failed to create hardware buffer");
             return;
@@ -43,7 +67,7 @@ public AHBImage(short width, short height) {
 
         boolean swapchainOk = true;
         for (int i = 0; i < swapchainAhbs.length; i++) {
-            swapchainAhbs[i] = createHardwareBuffer(width, height);
+            swapchainAhbs[i] = createHardwareBuffer(width, height, scanoutRgba);
             if (swapchainAhbs[i] == 0) swapchainOk = false;
         }
 
@@ -59,6 +83,7 @@ public AHBImage(short width, short height) {
     }
 
     public AHBImage(int socketFd) {
+        scanoutRgba = false;
         hardwareBufferPtr = hardwareBufferFromSocket(socketFd);
         if (hardwareBufferPtr != 0) {
             width = nativeGetWidth(hardwareBufferPtr);
@@ -122,7 +147,7 @@ public long getScanoutHardwareBufferPtr() {
             nativeCloseFd(lastAcquireFence);
         }
 
-        lastAcquireFence = copyHardwareBuffer(virtualData, targetAhb, (short)width, (short)height, stride, waitFence);
+        lastAcquireFence = copyHardwareBuffer(virtualData, targetAhb, (short)width, (short)height, stride, waitFence, scanoutRgba);
 
         lastUsedSlot = swapchainIndex;
         swapchainIndex = (swapchainIndex + 1) % swapchainAhbs.length;
@@ -205,15 +230,15 @@ public long getHardwareBufferPtr() {
 
     private native long hardwareBufferFromSocket(int fd);
 
-    private native long createHardwareBuffer(short width, short height);
+    private native long createHardwareBuffer(short width, short height, boolean rgba);
 
     private native void destroyHardwareBuffer(long hardwareBufferPtr);
 
     private native ByteBuffer lockHardwareBuffer(long hardwareBufferPtr);
 
     private native int unlockHardwareBuffer(long hardwareBufferPtr);
 
-    private native int copyHardwareBuffer(ByteBuffer srcBuffer, long dstPtr, short width, short height, short srcStride, int waitFence);
+    private native int copyHardwareBuffer(ByteBuffer srcBuffer, long dstPtr, short width, short height, short srcStride, int waitFence, boolean swapRB);
 
     public static native short nativeGetWidth(long ptr);
 

app/src/main/java/com/winlator/renderer/ASurfaceRenderer.java

@@ -476,8 +476,11 @@ private void pushCpuImageToNative(int windowId, Drawable drawable) {
                 long ahbPtr = g.getScanoutHardwareBufferPtr();
                 if (ahbPtr != 0) {
                     int acquireFence = g.consumeAcquireFence();
-                    // Disable swap R/B in cpu path as it is handled with drawable
-                    nativeSetWindowBuffer(windowId, ahbPtr, acquireFence, 0, 0, g, g.getLastUsedSlot(), sfCompatMode);
+                    // Disable swap R/B in cpu path as it is handled with drawable.
+                    // An RGBA_8888 slot already holds R,G,B,A bytes: SurfaceFlinger takes it as is, and the
+                    // BGRA-to-RGBA blit (which swaps an RGBA-declared source) must not touch it.
+                    boolean convert = sfCompatMode && !g.isScanoutRgba();
+                    nativeSetWindowBuffer(windowId, ahbPtr, acquireFence, 0, 0, g, g.getLastUsedSlot(), convert);
                     if (hudRef != null && skipFPSCount.get() >= 1) {
                         hudRef.update();
                         skipFPSCount.set(0);