mesa: fix possible precision issues in pack/unpack/fetch functions

GLfloat doesn't have enough precision to exactly represent 0xffffff
and 0xffffffff. (and a reciprocal of those, if I am not mistaken)

If -ffast-math is enabled, using GLfloat causes assertion failures in:
- fbo-blit-d24s8
- fbo-depth-sample-compare
- fbo-readpixels-depth-formats
- glean/depthStencil

For example:
fbo-depth-sample-compare: main/format_unpack.c:1769:
unpack_float_z_Z24_X8: Assertion `dst[i] <= 1.0F' failed.

Reviewed-by: Brian Paul <[email protected]>

1 files changed, 4 insertions, 4 deletions

diff --git a/src/mesa/main/format_unpack.c b/src/mesa/main/format_unpack.c
index f821c2be4f4..4f23f3dec3d 100644
--- a/src/mesa/main/format_unpack.c
+++ b/src/mesa/main/format_unpack.c
@@ -587,7 +587,7 @@ unpack_Z24_S8(const void *src, GLfloat dst[][4], GLuint n)
 {
    /* only return Z, not stencil data */
    const GLuint *s = ((const GLuint *) src);
-   const GLfloat scale = 1.0F / (GLfloat) 0xffffff;
+   const GLdouble scale = 1.0 / (GLdouble) 0xffffff;
    GLuint i;
    for (i = 0; i < n; i++) {
       dst[i][0] =
@@ -604,7 +604,7 @@ unpack_S8_Z24(const void *src, GLfloat dst[][4], GLuint n)
 {
    /* only return Z, not stencil data */
    const GLuint *s = ((const GLuint *) src);
-   const GLfloat scale = 1.0F / (GLfloat) 0xffffff;
+   const GLdouble scale = 1.0 / (GLdouble) 0xffffff;
    GLuint i;
    for (i = 0; i < n; i++) {
       dst[i][0] =
@@ -1761,7 +1761,7 @@ unpack_float_z_Z24_X8(GLuint n, const void *src, GLfloat *dst)
 {
    /* only return Z, not stencil data */
    const GLuint *s = ((const GLuint *) src);
-   const GLfloat scale = 1.0F / (GLfloat) 0xffffff;
+   const GLdouble scale = 1.0 / (GLdouble) 0xffffff;
    GLuint i;
    for (i = 0; i < n; i++) {
       dst[i] = (s[i] >> 8) * scale;
@@ -1775,7 +1775,7 @@ unpack_float_z_X8_Z24(GLuint n, const void *src, GLfloat *dst)
 {
    /* only return Z, not stencil data */
    const GLuint *s = ((const GLuint *) src);
-   const GLfloat scale = 1.0F / (GLfloat) 0xffffff;
+   const GLdouble scale = 1.0 / (GLdouble) 0xffffff;
    GLuint i;
    for (i = 0; i < n; i++) {
       dst[i] = (s[i] & 0x00ffffff) * scale;