st/mesa: fix integer texture border color for some formats (v2)

And the clear color too, though that may be an issue only with GL_RGB if it's
actually RGBA in the driver.

NOTE: This is a candidate for the stable branches.

Reviewed-by: Brian Paul <[email protected]>

v2: The types of st_translate_color parameters were changed to gl_color_union
    and pipe_color_union as per Brian's comment.

4 files changed, 112 insertions, 48 deletions

diff --git a/src/mesa/state_tracker/st_atom_sampler.c b/src/mesa/state_tracker/st_atom_sampler.c
index adcc7b50547..3eba5b13cae 100644
--- a/src/mesa/state_tracker/st_atom_sampler.c
+++ b/src/mesa/state_tracker/st_atom_sampler.c
@@ -34,6 +34,7 @@
 
 #include "main/macros.h"
 #include "main/mtypes.h"
+#include "main/glformats.h"
 #include "main/samplerobj.h"
 #include "main/texobj.h"
 
@@ -172,12 +173,17 @@ convert_sampler(struct st_context *st,
        msamp->BorderColor.ui[2] ||
        msamp->BorderColor.ui[3]) {
       struct gl_texture_image *teximg;
+      GLboolean is_integer = GL_FALSE;
 
       teximg = texobj->Image[0][texobj->BaseLevel];
 
-      st_translate_color(msamp->BorderColor.f,
-                         teximg ? teximg->_BaseFormat : GL_RGBA,
-                         sampler->border_color.f);
+      if (teximg) {
+         is_integer = _mesa_is_enum_format_integer(teximg->InternalFormat);
+      }
+
+      st_translate_color(&msamp->BorderColor,
+                         &sampler->border_color,
+                         teximg ? teximg->_BaseFormat : GL_RGBA, is_integer);
    }
 
    sampler->max_anisotropy = (msamp->MaxAnisotropy == 1.0 ?
diff --git a/src/mesa/state_tracker/st_cb_clear.c b/src/mesa/state_tracker/st_cb_clear.c
index e731b6b5e2e..90eb0af4fe0 100644
--- a/src/mesa/state_tracker/st_cb_clear.c
+++ b/src/mesa/state_tracker/st_cb_clear.c
@@ -37,6 +37,7 @@
 #include "main/accum.h"
 #include "main/formats.h"
 #include "main/macros.h"
+#include "main/glformats.h"
 #include "program/prog_instruction.h"
 #include "st_context.h"
 #include "st_atom.h"
@@ -301,9 +302,13 @@ clear_with_quad(struct gl_context *ctx,
    cso_set_geometry_shader_handle(st->cso_context, NULL);
 
    if (ctx->DrawBuffer->_ColorDrawBuffers[0]) {
-      st_translate_color(ctx->Color.ClearColor.f,
-                               ctx->DrawBuffer->_ColorDrawBuffers[0]->_BaseFormat,
-                               clearColor.f);
+      struct gl_renderbuffer *rb = ctx->DrawBuffer->_ColorDrawBuffers[0];
+      GLboolean is_integer = _mesa_is_enum_format_integer(rb->InternalFormat);
+
+      st_translate_color(&ctx->Color.ClearColor,
+                         &clearColor,
+                         ctx->DrawBuffer->_ColorDrawBuffers[0]->_BaseFormat,
+                         is_integer);
    }
 
    /* draw quad matching scissor rect */
@@ -540,9 +545,13 @@ st_Clear(struct gl_context *ctx, GLbitfield mask)
          clear_buffers |= PIPE_CLEAR_DEPTHSTENCIL;
 
       if (ctx->DrawBuffer->_ColorDrawBuffers[0]) {
-         st_translate_color(ctx->Color.ClearColor.f,
+         struct gl_renderbuffer *rb = ctx->DrawBuffer->_ColorDrawBuffers[0];
+         GLboolean is_integer = _mesa_is_enum_format_integer(rb->InternalFormat);
+
+         st_translate_color(&ctx->Color.ClearColor,
+                            &clearColor,
 			    ctx->DrawBuffer->_ColorDrawBuffers[0]->_BaseFormat,
-			    clearColor.f);
+			    is_integer);
       }
 
       st->pipe->clear(st->pipe, clear_buffers, &clearColor,
diff --git a/src/mesa/state_tracker/st_format.c b/src/mesa/state_tracker/st_format.c
index a9ff2cd0dd5..af81f732d21 100644
--- a/src/mesa/state_tracker/st_format.c
+++ b/src/mesa/state_tracker/st_format.c
@@ -1686,44 +1686,92 @@ st_sampler_compat_formats(enum pipe_format format1, enum pipe_format format2)
  * Similarly for texture border colors.
  */
 void
-st_translate_color(const GLfloat colorIn[4], GLenum baseFormat,
-                   GLfloat colorOut[4])
+st_translate_color(union gl_color_union *colorIn,
+                   union pipe_color_union *colorOut,
+                   GLenum baseFormat, GLboolean is_integer)
 {
-   switch (baseFormat) {
-   case GL_RED:
-      colorOut[0] = colorIn[0];
-      colorOut[1] = 0.0F;
-      colorOut[2] = 0.0F;
-      colorOut[3] = 1.0F;
-      break;
-   case GL_RG:
-      colorOut[0] = colorIn[0];
-      colorOut[1] = colorIn[1];
-      colorOut[2] = 0.0F;
-      colorOut[3] = 1.0F;
-      break;
-   case GL_RGB:
-      colorOut[0] = colorIn[0];
-      colorOut[1] = colorIn[1];
-      colorOut[2] = colorIn[2];
-      colorOut[3] = 1.0F;
-      break;
-   case GL_ALPHA:
-      colorOut[0] = colorOut[1] = colorOut[2] = 0.0;
-      colorOut[3] = colorIn[3];
-      break;
-   case GL_LUMINANCE:
-      colorOut[0] = colorOut[1] = colorOut[2] = colorIn[0];
-      colorOut[3] = 1.0;
-      break;
-   case GL_LUMINANCE_ALPHA:
-      colorOut[0] = colorOut[1] = colorOut[2] = colorIn[0];
-      colorOut[3] = colorIn[3];
-      break;
-   case GL_INTENSITY:
-      colorOut[0] = colorOut[1] = colorOut[2] = colorOut[3] = colorIn[0];
-      break;
-   default:
-      COPY_4V(colorOut, colorIn);
+   if (is_integer) {
+      int *in = colorIn->i;
+      int *out = colorOut->i;
+
+      switch (baseFormat) {
+      case GL_RED:
+         out[0] = in[0];
+         out[1] = 0;
+         out[2] = 0;
+         out[3] = 1;
+         break;
+      case GL_RG:
+         out[0] = in[0];
+         out[1] = in[1];
+         out[2] = 0;
+         out[3] = 1;
+         break;
+      case GL_RGB:
+         out[0] = in[0];
+         out[1] = in[1];
+         out[2] = in[2];
+         out[3] = 1;
+         break;
+      case GL_ALPHA:
+         out[0] = out[1] = out[2] = 0;
+         out[3] = in[3];
+         break;
+      case GL_LUMINANCE:
+         out[0] = out[1] = out[2] = in[0];
+         out[3] = 1;
+         break;
+      case GL_LUMINANCE_ALPHA:
+         out[0] = out[1] = out[2] = in[0];
+         out[3] = in[3];
+         break;
+      case GL_INTENSITY:
+         out[0] = out[1] = out[2] = out[3] = in[0];
+         break;
+      default:
+         COPY_4V(out, in);
+      }
+   }
+   else {
+      float *in = colorIn->f;
+      float *out = colorOut->f;
+
+      switch (baseFormat) {
+      case GL_RED:
+         out[0] = in[0];
+         out[1] = 0.0F;
+         out[2] = 0.0F;
+         out[3] = 1.0F;
+         break;
+      case GL_RG:
+         out[0] = in[0];
+         out[1] = in[1];
+         out[2] = 0.0F;
+         out[3] = 1.0F;
+         break;
+      case GL_RGB:
+         out[0] = in[0];
+         out[1] = in[1];
+         out[2] = in[2];
+         out[3] = 1.0F;
+         break;
+      case GL_ALPHA:
+         out[0] = out[1] = out[2] = 0.0F;
+         out[3] = in[3];
+         break;
+      case GL_LUMINANCE:
+         out[0] = out[1] = out[2] = in[0];
+         out[3] = 1.0F;
+         break;
+      case GL_LUMINANCE_ALPHA:
+         out[0] = out[1] = out[2] = in[0];
+         out[3] = in[3];
+         break;
+      case GL_INTENSITY:
+         out[0] = out[1] = out[2] = out[3] = in[0];
+         break;
+      default:
+         COPY_4V(out, in);
+      }
    }
 }
diff --git a/src/mesa/state_tracker/st_format.h b/src/mesa/state_tracker/st_format.h
index 2eef2c0d438..39397b17aa6 100644
--- a/src/mesa/state_tracker/st_format.h
+++ b/src/mesa/state_tracker/st_format.h
@@ -75,7 +75,8 @@ st_sampler_compat_formats(enum pipe_format format1, enum pipe_format format2);
 
 
 extern void
-st_translate_color(const GLfloat colorIn[4], GLenum baseFormat,
-                   GLfloat colorOut[4]);
+st_translate_color(union gl_color_union *colorIn,
+                   union pipe_color_union *colorOut,
+                   GLenum baseFormat, GLboolean is_integer);
 
 #endif /* ST_FORMAT_H */