st/xorg: Fix comp alpha code and deal with luminance masks

There are two fixes in here one is a one liner that fixes
component alpha logic. The other deals better with luminance
formats used for masks, sources not yet implemented.

Fixes component alpha text and icons in gnome. There are a one
or two cases that this code misses. Like if src_luminance is set
but no mask image is given.

4 files changed, 62 insertions, 6 deletions

diff --git a/src/gallium/state_trackers/xorg/xorg_composite.c b/src/gallium/state_trackers/xorg/xorg_composite.c
index 1ff19a2a5c0..93a3e1b8cf0 100644
--- a/src/gallium/state_trackers/xorg/xorg_composite.c
+++ b/src/gallium/state_trackers/xorg/xorg_composite.c
@@ -235,8 +235,11 @@ picture_format_fixups(struct exa_pixmap_priv *pSrc, PicturePtr pSrcPicture, bool
    boolean swizzle = FALSE;
    unsigned ret = 0;
 
-   if (pSrc->picture_format == pSrcPicture->format)
+   if (pSrc->picture_format == pSrcPicture->format) {
+      if (pSrc->picture_format == PICT_a8)
+         return mask ? FS_MASK_LUMINANCE : FS_MASK_LUMINANCE;
       return 0;
+   }
 
    if (pSrc->picture_format != PICT_a8r8g8b8) {
       assert(!"can not handle formats");
diff --git a/src/gallium/state_trackers/xorg/xorg_exa.c b/src/gallium/state_trackers/xorg/xorg_exa.c
index 25c9fce2547..6fa274eb0ab 100644
--- a/src/gallium/state_trackers/xorg/xorg_exa.c
+++ b/src/gallium/state_trackers/xorg/xorg_exa.c
@@ -551,6 +551,9 @@ ExaPrepareComposite(int op, PicturePtr pSrcPicture,
          XORG_FALLBACK("pSrc pic_format: %s != %s",
                        render_format_name(priv->picture_format),
                        render_format_name(pSrcPicture->format));
+
+      if (priv->picture_format == PICT_a8)
+         XORG_FALLBACK("pSrc pic_format == PICT_a8");
    }
 
    if (pMask) {
diff --git a/src/gallium/state_trackers/xorg/xorg_exa_tgsi.c b/src/gallium/state_trackers/xorg/xorg_exa_tgsi.c
index cbb9ec137ea..3bf64b6331d 100644
--- a/src/gallium/state_trackers/xorg/xorg_exa_tgsi.c
+++ b/src/gallium/state_trackers/xorg/xorg_exa_tgsi.c
@@ -43,6 +43,38 @@
  * OUT[0] = color
  */
 
+static void
+print_fs_traits(int fs_traits)
+{
+   const char *strings[] = {
+      "FS_COMPOSITE",       // = 1 << 0,
+      "FS_MASK",            // = 1 << 1,
+      "FS_SOLID_FILL",      // = 1 << 2,
+      "FS_LINGRAD_FILL",    // = 1 << 3,
+      "FS_RADGRAD_FILL",    // = 1 << 4,
+      "FS_CA_FULL",         // = 1 << 5, /* src.rgba * mask.rgba */
+      "FS_CA_SRCALPHA",     // = 1 << 6, /* src.aaaa * mask.rgba */
+      "FS_YUV",             // = 1 << 7,
+      "FS_SRC_REPEAT_NONE", // = 1 << 8,
+      "FS_MASK_REPEAT_NONE",// = 1 << 9,
+      "FS_SRC_SWIZZLE_RGB", // = 1 << 10,
+      "FS_MASK_SWIZZLE_RGB",// = 1 << 11,
+      "FS_SRC_SET_ALPHA",   // = 1 << 12,
+      "FS_MASK_SET_ALPHA",  // = 1 << 13,
+      "FS_SRC_LUMINANCE",   // = 1 << 14,
+      "FS_MASK_LUMINANCE",  // = 1 << 15,
+   };
+   int i, k;
+   debug_printf("%s: ", __func__);
+
+   for (i = 0, k = 1; k < (1 << 16); i++, k <<= 1) {
+      if (fs_traits & k)
+         debug_printf("%s, ", strings[i]);
+   }
+
+   debug_printf("\n");
+}
+
 struct xorg_shaders {
    struct xorg_renderer *r;
 
@@ -55,7 +87,8 @@ src_in_mask(struct ureg_program *ureg,
             struct ureg_dst dst,
             struct ureg_src src,
             struct ureg_src mask,
-            int component_alpha)
+            unsigned component_alpha,
+            unsigned mask_luminance)
 {
    if (component_alpha == FS_CA_FULL) {
       ureg_MUL(ureg, dst, src, mask);
@@ -64,8 +97,12 @@ src_in_mask(struct ureg_program *ureg,
                ureg_scalar(src, TGSI_SWIZZLE_W), mask);
    }
    else {
-      ureg_MUL(ureg, dst, src,
-               ureg_scalar(mask, TGSI_SWIZZLE_X));
+      if (mask_luminance)
+         ureg_MUL(ureg, dst, src,
+                  ureg_scalar(mask, TGSI_SWIZZLE_X));
+      else
+         ureg_MUL(ureg, dst, src,
+                  ureg_scalar(mask, TGSI_SWIZZLE_W));
    }
 }
 
@@ -435,7 +472,7 @@ create_fs(struct pipe_context *pipe,
    unsigned is_solid   = (fs_traits & FS_SOLID_FILL) != 0;
    unsigned is_lingrad = (fs_traits & FS_LINGRAD_FILL) != 0;
    unsigned is_radgrad = (fs_traits & FS_RADGRAD_FILL) != 0;
-   unsigned comp_alpha = (fs_traits & FS_COMPONENT_ALPHA) != 0;
+   unsigned comp_alpha_mask = fs_traits & FS_COMPONENT_ALPHA;
    unsigned is_yuv = (fs_traits & FS_YUV) != 0;
    unsigned src_repeat_none = (fs_traits & FS_SRC_REPEAT_NONE) != 0;
    unsigned mask_repeat_none = (fs_traits & FS_MASK_REPEAT_NONE) != 0;
@@ -443,6 +480,16 @@ create_fs(struct pipe_context *pipe,
    unsigned mask_swizzle = (fs_traits & FS_MASK_SWIZZLE_RGB) != 0;
    unsigned src_set_alpha = (fs_traits & FS_SRC_SET_ALPHA) != 0;
    unsigned mask_set_alpha = (fs_traits & FS_MASK_SET_ALPHA) != 0;
+   unsigned src_luminance = (fs_traits & FS_SRC_LUMINANCE) != 0;
+   unsigned mask_luminance = (fs_traits & FS_MASK_LUMINANCE) != 0;
+
+   if (src_luminance)
+      assert(!"src_luminance not supported");
+#if 0
+   print_fs_traits(fs_traits);
+#else
+   (void)print_fs_traits;
+#endif
 
    ureg = ureg_create(TGSI_PROCESSOR_FRAGMENT);
    if (ureg == NULL)
@@ -541,7 +588,8 @@ create_fs(struct pipe_context *pipe,
       xrender_tex(ureg, mask, mask_pos, mask_sampler,
                   mask_repeat_none, mask_swizzle, mask_set_alpha);
       /* src IN mask */
-      src_in_mask(ureg, out, ureg_src(src), ureg_src(mask), comp_alpha);
+      src_in_mask(ureg, out, ureg_src(src), ureg_src(mask),
+                  comp_alpha_mask, mask_luminance);
       ureg_release_temporary(ureg, mask);
    }
 
diff --git a/src/gallium/state_trackers/xorg/xorg_exa_tgsi.h b/src/gallium/state_trackers/xorg/xorg_exa_tgsi.h
index 062de947e62..6f2a361d030 100644
--- a/src/gallium/state_trackers/xorg/xorg_exa_tgsi.h
+++ b/src/gallium/state_trackers/xorg/xorg_exa_tgsi.h
@@ -32,6 +32,8 @@ enum xorg_fs_traits {
    FS_MASK_SWIZZLE_RGB = 1 << 11,
    FS_SRC_SET_ALPHA    = 1 << 12,
    FS_MASK_SET_ALPHA   = 1 << 13,
+   FS_SRC_LUMINANCE    = 1 << 14,
+   FS_MASK_LUMINANCE   = 1 << 15,
 
    FS_FILL             = (FS_SOLID_FILL |
                           FS_LINGRAD_FILL |