egl/x11: Handle both depth 30 formats for eglCreateImage(). (v4)

We need to distinguish if the backing storage of a pixmap
is XRGB2101010 or XBGR2101010, as different gpu hw supports
different formats. NVidia hw prefers XBGR, whereas AMD and
Intel are happy with XRGB.

Use the red channel mask of the first depth 30 visual of
the x-screen to distinguish which hw format to choose.

This fixes desktop composition of color depth 30 windows
when the X11 compositor uses EGL.

v2: Switch from using the visual of the root window to simply
    using the first depth 30 visual for the x-screen, as testing
    shows that each driver only exports either xrgb ordering or
    xbgr ordering for the channel masks of its depth 30 visuals,
    so this should be unambiguous and avoid trouble if X ever
    supports depth 30 pixmaps on screens with a non-depth 30 root
    window visual. This per Michels suggestion.

v3: No change to v2, but spent some time testing this more on
    AMD hw, with my software hacked up to intentionally choose
    pixel formats/visual with the non-preferred xBGR2101010
    ordering on the ati-ddx, also with a standard non-OpenGL
    X-Window with depth 30 visual, to make sure that things show
    up properly with the right colors on the screen when going
    through EGL+OpenGL based compositing on KDE-5. Iow. to confirm
    that my explanation to the v2 patch on the mailing list of why
    it should work and the actual practice agree (or possibly that
    i am good at fooling myself during testing ;).

v4: Drop the local `red_mask` and just `return visual->red_mask`/
    `return 0`, as suggested by Eric Engestrom.

    Rebased onto current master, to take the cleanup via the new
    function dri2_format_for_depth() into account.

Signed-off-by: Mario Kleiner <[email protected]>
Reviewed-by: Eric Engestrom <[email protected]>

4 files changed, 48 insertions, 9 deletions

diff --git a/src/egl/drivers/dri2/egl_dri2.h b/src/egl/drivers/dri2/egl_dri2.h
index 5d8fbfa2356..f8001ec4b66 100644
--- a/src/egl/drivers/dri2/egl_dri2.h
+++ b/src/egl/drivers/dri2/egl_dri2.h
@@ -411,6 +411,8 @@ EGLBoolean
 dri2_initialize_x11(_EGLDriver *drv, _EGLDisplay *disp);
 void
 dri2_teardown_x11(struct dri2_egl_display *dri2_dpy);
+unsigned int
+dri2_x11_get_red_mask_for_depth(struct dri2_egl_display *dri2_dpy, int depth);
 #else
 static inline EGLBoolean
 dri2_initialize_x11(_EGLDriver *drv, _EGLDisplay *disp)
@@ -419,6 +421,11 @@ dri2_initialize_x11(_EGLDriver *drv, _EGLDisplay *disp)
 }
 static inline void
 dri2_teardown_x11(struct dri2_egl_display *dri2_dpy) {}
+static inline unsigned int
+dri2_x11_get_red_mask_for_depth(struct dri2_egl_display *dri2_dpy, int depth)
+{
+   return 0;
+}
 #endif
 
 #ifdef HAVE_DRM_PLATFORM
diff --git a/src/egl/drivers/dri2/platform_x11.c b/src/egl/drivers/dri2/platform_x11.c
index ea9b0cc6d6f..cfa5c4aa2bf 100644
--- a/src/egl/drivers/dri2/platform_x11.c
+++ b/src/egl/drivers/dri2/platform_x11.c
@@ -56,7 +56,7 @@ dri2_x11_swap_interval(_EGLDriver *drv, _EGLDisplay *disp, _EGLSurface *surf,
                        EGLint interval);
 
 uint32_t
-dri2_format_for_depth(uint32_t depth);
+dri2_format_for_depth(struct dri2_egl_display *dri2_dpy, uint32_t depth);
 
 static void
 swrastCreateDrawable(struct dri2_egl_display * dri2_dpy,
@@ -212,6 +212,36 @@ get_xcb_screen(xcb_screen_iterator_t iter, int screen)
     return NULL;
 }
 
+static xcb_visualtype_t *
+get_xcb_visualtype_for_depth(struct dri2_egl_display *dri2_dpy, int depth)
+{
+   xcb_visualtype_iterator_t visual_iter;
+   xcb_screen_t *screen = dri2_dpy->screen;
+   xcb_depth_iterator_t depth_iter = xcb_screen_allowed_depths_iterator(screen);
+
+   for (; depth_iter.rem; xcb_depth_next(&depth_iter)) {
+      if (depth_iter.data->depth != depth)
+         continue;
+
+      visual_iter = xcb_depth_visuals_iterator(depth_iter.data);
+      if (visual_iter.rem)
+         return visual_iter.data;
+   }
+
+   return NULL;
+}
+
+/* Get red channel mask for given depth. */
+unsigned int
+dri2_x11_get_red_mask_for_depth(struct dri2_egl_display *dri2_dpy, int depth)
+{
+   xcb_visualtype_t *visual = get_xcb_visualtype_for_depth(dri2_dpy, depth);
+
+   if (visual)
+      return visual->red_mask;
+
+   return 0;
+}
 
 /**
  * Called via eglCreateWindowSurface(), drv->API.CreateWindowSurface().
@@ -1010,7 +1040,7 @@ dri2_x11_copy_buffers(_EGLDriver *drv, _EGLDisplay *disp, _EGLSurface *surf,
 }
 
 uint32_t
-dri2_format_for_depth(uint32_t depth)
+dri2_format_for_depth(struct dri2_egl_display *dri2_dpy, uint32_t depth)
 {
    switch (depth) {
    case 16:
@@ -1018,7 +1048,11 @@ dri2_format_for_depth(uint32_t depth)
    case 24:
       return __DRI_IMAGE_FORMAT_XRGB8888;
    case 30:
-      return __DRI_IMAGE_FORMAT_XRGB2101010;
+      /* Different preferred formats for different hw */
+      if (dri2_x11_get_red_mask_for_depth(dri2_dpy, 30) == 0x3ff)
+         return __DRI_IMAGE_FORMAT_XBGR2101010;
+      else
+         return __DRI_IMAGE_FORMAT_XRGB2101010;
    case 32:
       return __DRI_IMAGE_FORMAT_ARGB8888;
    default:
@@ -1026,7 +1060,6 @@ dri2_format_for_depth(uint32_t depth)
    }
 }
 
-
 static _EGLImage *
 dri2_create_image_khr_pixmap(_EGLDisplay *disp, _EGLContext *ctx,
 			     EGLClientBuffer buffer, const EGLint *attr_list)
@@ -1071,8 +1104,7 @@ dri2_create_image_khr_pixmap(_EGLDisplay *disp, _EGLContext *ctx,
       return NULL;
    }
 
-   format = dri2_format_for_depth(geometry_reply->depth);
-
+   format = dri2_format_for_depth(dri2_dpy, geometry_reply->depth);
    if (format == __DRI_IMAGE_FORMAT_NONE) {
       _eglError(EGL_BAD_PARAMETER,
 		"dri2_create_image_khr: unsupported pixmap depth");
diff --git a/src/egl/drivers/dri2/platform_x11_dri3.c b/src/egl/drivers/dri2/platform_x11_dri3.c
index 0d5a9d037f5..c3c9c2dd45d 100644
--- a/src/egl/drivers/dri2/platform_x11_dri3.c
+++ b/src/egl/drivers/dri2/platform_x11_dri3.c
@@ -281,7 +281,7 @@ dri3_create_image_khr_pixmap(_EGLDisplay *disp, _EGLContext *ctx,
       return NULL;
    }
 
-   format = dri2_format_for_depth(bp_reply->depth);
+   format = dri2_format_for_depth(dri2_dpy, bp_reply->depth);
    if (format == __DRI_IMAGE_FORMAT_NONE) {
       _eglError(EGL_BAD_PARAMETER,
                 "dri3_create_image_khr: unsupported pixmap depth");
@@ -333,7 +333,7 @@ dri3_create_image_khr_pixmap_from_buffers(_EGLDisplay *disp, _EGLContext *ctx,
       return EGL_NO_IMAGE_KHR;
    }
 
-   format = dri2_format_for_depth(bp_reply->depth);
+   format = dri2_format_for_depth(dri2_dpy, bp_reply->depth);
    if (format == __DRI_IMAGE_FORMAT_NONE) {
       _eglError(EGL_BAD_PARAMETER,
                 "dri3_create_image_khr: unsupported pixmap depth");
diff --git a/src/egl/drivers/dri2/platform_x11_dri3.h b/src/egl/drivers/dri2/platform_x11_dri3.h
index e6fd0136697..f60d1df641a 100644
--- a/src/egl/drivers/dri2/platform_x11_dri3.h
+++ b/src/egl/drivers/dri2/platform_x11_dri3.h
@@ -39,6 +39,6 @@ EGLBoolean
 dri3_x11_connect(struct dri2_egl_display *dri2_dpy);
 
 uint32_t
-dri2_format_for_depth(uint32_t depth);
+dri2_format_for_depth(struct dri2_egl_display *dri2_dpy, uint32_t depth);
 
 #endif