st/mesa: optionally apply texture swizzle to border color v2

This is the only sane solution for nv50 and nvc0 (really, trust me),
but since on other hardware the border colour is tightly coupled with
texture state they'd have to undo the swizzle, so I've added a cap.

The dependency of update_sampler on the texture updates was
introduced to avoid doing the apply_depthmode to the swizzle twice.

v2: Moved swizzling helper to u_format.c, extended the CAP to
provide more accurate information.

4 files changed, 30 insertions, 4 deletions

diff --git a/src/mesa/state_tracker/st_atom.c b/src/mesa/state_tracker/st_atom.c
index 32bcc266a34..7d383924300 100644
--- a/src/mesa/state_tracker/st_atom.c
+++ b/src/mesa/state_tracker/st_atom.c
@@ -55,10 +55,10 @@ static const struct st_tracked_state *atoms[] =
    &st_update_viewport,
    &st_update_scissor,
    &st_update_blend,
-   &st_update_sampler,
    &st_update_vertex_texture,
    &st_update_fragment_texture,
    &st_update_geometry_texture,
+   &st_update_sampler, /* depends on update_*_texture for swizzle */
    &st_update_framebuffer,
    &st_update_msaa,
    &st_update_vs_constants,
diff --git a/src/mesa/state_tracker/st_atom_sampler.c b/src/mesa/state_tracker/st_atom_sampler.c
index 3eba5b13cae..db51eead16b 100644
--- a/src/mesa/state_tracker/st_atom_sampler.c
+++ b/src/mesa/state_tracker/st_atom_sampler.c
@@ -48,6 +48,8 @@
 
 #include "cso_cache/cso_context.h"
 
+#include "util/u_format.h"
+
 
 /**
  * Convert GLenum texcoord wrap tokens to pipe tokens.
@@ -172,8 +174,10 @@ convert_sampler(struct st_context *st,
        msamp->BorderColor.ui[1] ||
        msamp->BorderColor.ui[2] ||
        msamp->BorderColor.ui[3]) {
+      struct st_texture_object *stobj = st_texture_object(texobj);
       struct gl_texture_image *teximg;
       GLboolean is_integer = GL_FALSE;
+      union pipe_color_union border_color;
 
       teximg = texobj->Image[0][texobj->BaseLevel];
 
@@ -181,9 +185,26 @@ convert_sampler(struct st_context *st,
          is_integer = _mesa_is_enum_format_integer(teximg->InternalFormat);
       }
 
-      st_translate_color(&msamp->BorderColor,
-                         &sampler->border_color,
-                         teximg ? teximg->_BaseFormat : GL_RGBA, is_integer);
+      if (st->apply_texture_swizzle_to_border_color && stobj->sampler_view) {
+         const unsigned char swz[4] =
+         {
+            stobj->sampler_view->swizzle_r,
+            stobj->sampler_view->swizzle_g,
+            stobj->sampler_view->swizzle_b,
+            stobj->sampler_view->swizzle_a,
+         };
+
+         st_translate_color(&msamp->BorderColor,
+                            &border_color,
+                            teximg ? teximg->_BaseFormat : GL_RGBA, is_integer);
+
+         util_format_apply_color_swizzle(&sampler->border_color,
+                                         &border_color, swz, is_integer);
+      } else {
+         st_translate_color(&msamp->BorderColor,
+                            &sampler->border_color,
+                            teximg ? teximg->_BaseFormat : GL_RGBA, is_integer);
+      }
    }
 
    sampler->max_anisotropy = (msamp->MaxAnisotropy == 1.0 ?
diff --git a/src/mesa/state_tracker/st_context.c b/src/mesa/state_tracker/st_context.c
index 2042be36bd3..ed1bf26a260 100644
--- a/src/mesa/state_tracker/st_context.c
+++ b/src/mesa/state_tracker/st_context.c
@@ -188,6 +188,10 @@ st_create_context_priv( struct gl_context *ctx, struct pipe_context *pipe,
 
    st->needs_texcoord_semantic =
       screen->get_param(screen, PIPE_CAP_TGSI_TEXCOORD);
+   st->apply_texture_swizzle_to_border_color =
+      !!(screen->get_param(screen, PIPE_CAP_TEXTURE_BORDER_COLOR_QUIRK) &
+         (PIPE_QUIRK_TEXTURE_BORDER_COLOR_SWIZZLE_NV50 |
+          PIPE_QUIRK_TEXTURE_BORDER_COLOR_SWIZZLE_R600));
 
    /* GL limits and extensions */
    st_init_limits(st);
diff --git a/src/mesa/state_tracker/st_context.h b/src/mesa/state_tracker/st_context.h
index 8786a036ff4..807453e9974 100644
--- a/src/mesa/state_tracker/st_context.h
+++ b/src/mesa/state_tracker/st_context.h
@@ -87,6 +87,7 @@ struct st_context
    boolean prefer_blit_based_texture_transfer;
 
    boolean needs_texcoord_semantic;
+   boolean apply_texture_swizzle_to_border_color;
 
    /* On old libGL's for linux we need to invalidate the drawables
     * on glViewpport calls, this is set via a option.