mesa: Fix GL_LUMINANCE handling for textures in glGetTexImage

We need to rebase colors (ex: set G=B=0) when getting GL_LUMINANCE
textures in following cases:
1. If the luminance texture is actually stored as rgba
2. If getting a luminance texture, but returning rgba
3. If getting an rgba texture, but returning luminance

A similar fix was pushed by Brian Paul for uncompressed textures
in commit: f5d0ced.
Fixes https://bugs.freedesktop.org/show_bug.cgi?id=47220

Observed no regressions in piglit and ogles2conform due to this fix.
This patch will cause failures in intel oglconform pxconv-gettex,
pxstore-gettex and pxtrans-gettex test cases. The cause of failures
is a bug in test cases. Expected luminance value is calculted
incorrectly in test cases: L = R+G+B.

V2: Set G = 0 when getting a RG texture but returning luminance.

Note: This is a candidate for stable branches.

Signed-off-by: Anuj Phogat <[email protected]>
Reviewed-by: Ian Romanick <[email protected]>

1 files changed, 27 insertions, 11 deletions

diff --git a/src/mesa/main/texgetimage.c b/src/mesa/main/texgetimage.c
index 2ccdcebd9f9..69067df9f5d 100644
--- a/src/mesa/main/texgetimage.c
+++ b/src/mesa/main/texgetimage.c
@@ -229,6 +229,8 @@ get_tex_rgba_compressed(struct gl_context *ctx, GLuint dimensions,
    const gl_format texFormat =
       _mesa_get_srgb_format_linear(texImage->TexFormat);
    const GLenum baseFormat = _mesa_get_format_base_format(texFormat);
+   const GLenum destBaseFormat = _mesa_base_tex_format(ctx, format);
+   GLenum rebaseFormat = GL_NONE;
    const GLuint width = texImage->Width;
    const GLuint height = texImage->Height;
    const GLuint depth = texImage->Depth;
@@ -266,9 +268,31 @@ get_tex_rgba_compressed(struct gl_context *ctx, GLuint dimensions,
    }
 
    if (baseFormat == GL_LUMINANCE ||
+       baseFormat == GL_INTENSITY ||
        baseFormat == GL_LUMINANCE_ALPHA) {
+      /* If a luminance (or intensity) texture is read back as RGB(A), the
+       * returned value should be (L,0,0,1), not (L,L,L,1).  Set rebaseFormat
+       * here to get G=B=0.
+       */
+      rebaseFormat = texImage->_BaseFormat;
+   }
+   else if ((baseFormat == GL_RGBA ||
+             baseFormat == GL_RGB  ||
+             baseFormat == GL_RG) &&
+            (destBaseFormat == GL_LUMINANCE ||
+             destBaseFormat == GL_LUMINANCE_ALPHA ||
+             destBaseFormat == GL_LUMINANCE_INTEGER_EXT ||
+             destBaseFormat == GL_LUMINANCE_ALPHA_INTEGER_EXT)) {
+      /* If we're reading back an RGB(A) texture as luminance then we need
+       * to return L=tex(R).  Note, that's different from glReadPixels which
+       * returns L=R+G+B.
+       */
+      rebaseFormat = GL_LUMINANCE_ALPHA; /* this covers GL_LUMINANCE too */
+   }
+
+   if (rebaseFormat) {
       _mesa_rebase_rgba_float(width * height, (GLfloat (*)[4]) tempImage,
-                              baseFormat);
+                              rebaseFormat);
    }
 
    srcRow = tempImage;
@@ -332,7 +356,8 @@ get_tex_rgba_uncompressed(struct gl_context *ctx, GLuint dimensions,
       rebaseFormat = texImage->_BaseFormat;
    }
    else if ((texImage->_BaseFormat == GL_RGBA ||
-             texImage->_BaseFormat == GL_RGB) &&
+             texImage->_BaseFormat == GL_RGB ||
+             texImage->_BaseFormat == GL_RG) &&
             (destBaseFormat == GL_LUMINANCE ||
              destBaseFormat == GL_LUMINANCE_ALPHA ||
              destBaseFormat == GL_LUMINANCE_INTEGER_EXT ||
@@ -419,15 +444,6 @@ get_tex_rgba(struct gl_context *ctx, GLuint dimensions,
          transferOps |= IMAGE_CLAMP_BIT;
       }
    }
-   /* This applies to RGB, RGBA textures. if the format is either LUMINANCE
-    * or LUMINANCE ALPHA, luminance (L) is computed as L=R+G+B .we need to
-    * clamp the sum to [0,1].
-    */
-   else if ((format == GL_LUMINANCE ||
-            format == GL_LUMINANCE_ALPHA) &&
-            dataType == GL_UNSIGNED_NORMALIZED) {
-      transferOps |= IMAGE_CLAMP_BIT;
-   }
 
    if (_mesa_is_format_compressed(texImage->TexFormat)) {
       get_tex_rgba_compressed(ctx, dimensions, format, type,