fix bug 426657: need to clamp RGBA before color table lookups

1 files changed, 63 insertions, 27 deletions

diff --git a/src/mesa/main/pixel.c b/src/mesa/main/pixel.c
index dc8f5be3073..55f6b99b0f0 100644
--- a/src/mesa/main/pixel.c
+++ b/src/mesa/main/pixel.c
@@ -1,4 +1,4 @@
-/* $Id: pixel.c,v 1.28 2001/04/10 15:25:45 brianp Exp $ */
+/* $Id: pixel.c,v 1.29 2001/05/23 23:55:01 brianp Exp $ */
 
 /*
  * Mesa 3-D graphics library
@@ -832,10 +832,14 @@ _mesa_map_rgba( const GLcontext *ctx, GLuint n, GLfloat rgba[][4] )
    const GLfloat *aMap = ctx->Pixel.MapAtoA;
    GLuint i;
    for (i=0;i<n;i++) {
-      rgba[i][RCOMP] = rMap[IROUND(rgba[i][RCOMP] * rscale)];
-      rgba[i][GCOMP] = gMap[IROUND(rgba[i][GCOMP] * gscale)];
-      rgba[i][BCOMP] = bMap[IROUND(rgba[i][BCOMP] * bscale)];
-      rgba[i][ACOMP] = aMap[IROUND(rgba[i][ACOMP] * ascale)];
+      GLfloat r = CLAMP(rgba[i][RCOMP], 0.0F, 1.0F);
+      GLfloat g = CLAMP(rgba[i][GCOMP], 0.0F, 1.0F);
+      GLfloat b = CLAMP(rgba[i][BCOMP], 0.0F, 1.0F);
+      GLfloat a = CLAMP(rgba[i][ACOMP], 0.0F, 1.0F);
+      rgba[i][RCOMP] = rMap[IROUND(r * rscale)];
+      rgba[i][GCOMP] = gMap[IROUND(g * gscale)];
+      rgba[i][BCOMP] = bMap[IROUND(b * bscale)];
+      rgba[i][ACOMP] = aMap[IROUND(a * ascale)];
    }
 }
 
@@ -884,24 +888,26 @@ _mesa_lookup_rgba(const struct gl_color_table *table,
       case GL_INTENSITY:
          /* replace RGBA with I */
          if (!table->FloatTable) {
-            const GLfloat scale = (GLfloat) (table->Size - 1);
+            const GLint max = table->Size - 1;
+            const GLfloat scale = (GLfloat) max;
             const GLchan *lut = (const GLchan *) table->Table;
             GLuint i;
             for (i = 0; i < n; i++) {
                GLint j = IROUND(rgba[i][RCOMP] * scale);
-               GLfloat c = CHAN_TO_FLOAT(lut[j]);
+               GLfloat c = CHAN_TO_FLOAT(lut[CLAMP(j, 0, 1)]);
                rgba[i][RCOMP] = rgba[i][GCOMP] =
                   rgba[i][BCOMP] = rgba[i][ACOMP] = c;
             }
 
          }
          else {
-            const GLfloat scale = (GLfloat) (table->Size - 1);
+            const GLint max = table->Size - 1;
+            const GLfloat scale = (GLfloat) max;
             const GLfloat *lut = (const GLfloat *) table->Table;
             GLuint i;
             for (i = 0; i < n; i++) {
                GLint j = IROUND(rgba[i][RCOMP] * scale);
-               GLfloat c = lut[j];
+               GLfloat c = lut[CLAMP(j, 0, max)];
                rgba[i][RCOMP] = rgba[i][GCOMP] =
                   rgba[i][BCOMP] = rgba[i][ACOMP] = c;
             }
@@ -910,22 +916,24 @@ _mesa_lookup_rgba(const struct gl_color_table *table,
       case GL_LUMINANCE:
          /* replace RGB with L */
          if (!table->FloatTable) {
-            const GLfloat scale = (GLfloat) (table->Size - 1);
+            const GLint max = table->Size - 1;
+            const GLfloat scale = (GLfloat) max;
             const GLchan *lut = (const GLchan *) table->Table;
             GLuint i;
             for (i = 0; i < n; i++) {
                GLint j = IROUND(rgba[i][RCOMP] * scale);
-               GLfloat c = CHAN_TO_FLOAT(lut[j]);
+               GLfloat c = CHAN_TO_FLOAT(lut[CLAMP(j, 0, max)]);
                rgba[i][RCOMP] = rgba[i][GCOMP] = rgba[i][BCOMP] = c;
             }
          }
          else {
-            const GLfloat scale = (GLfloat) (table->Size - 1);
+            const GLint max = table->Size - 1;
+            const GLfloat scale = (GLfloat) max;
             const GLfloat *lut = (const GLfloat *) table->Table;
             GLuint i;
             for (i = 0; i < n; i++) {
                GLint j = IROUND(rgba[i][RCOMP] * scale);
-               GLfloat c = lut[j];
+               GLfloat c = lut[CLAMP(j, 0, max)];
                rgba[i][RCOMP] = rgba[i][GCOMP] = rgba[i][BCOMP] = c;
             }
          }
@@ -933,48 +941,58 @@ _mesa_lookup_rgba(const struct gl_color_table *table,
       case GL_ALPHA:
          /* replace A with A */
          if (!table->FloatTable) {
-            const GLfloat scale = (GLfloat) (table->Size - 1);
+            const GLint max = table->Size - 1;
+            const GLfloat scale = (GLfloat) max;
             const GLchan *lut = (const GLchan *) table->Table;
             GLuint i;
             for (i = 0; i < n; i++) {
                GLint j = IROUND(rgba[i][ACOMP] * scale);
-               rgba[i][ACOMP] = CHAN_TO_FLOAT(lut[j]);
+               rgba[i][ACOMP] = CHAN_TO_FLOAT(lut[CLAMP(j, 0, max)]);
             }
          }
          else  {
-            const GLfloat scale = (GLfloat) (table->Size - 1);
+            const GLint max = table->Size - 1;
+            const GLfloat scale = (GLfloat) max;
             const GLfloat *lut = (const GLfloat *) table->Table;
             GLuint i;
             for (i = 0; i < n; i++) {
                GLint j = IROUND(rgba[i][ACOMP] * scale);
-               rgba[i][ACOMP] = lut[j];
+               rgba[i][ACOMP] = lut[CLAMP(j, 0, max)];
             }
          }
          break;
       case GL_LUMINANCE_ALPHA:
          /* replace RGBA with LLLA */
          if (!table->FloatTable) {
-            const GLfloat scale = (GLfloat) (table->Size - 1);
+            const GLint max = table->Size - 1;
+            const GLfloat scale = (GLfloat) max;
             const GLchan *lut = (const GLchan *) table->Table;
             GLuint i;
             for (i = 0; i < n; i++) {
                GLint jL = IROUND(rgba[i][RCOMP] * scale);
                GLint jA = IROUND(rgba[i][ACOMP] * scale);
-               GLfloat luminance = CHAN_TO_FLOAT(lut[jL * 2 + 0]);
-               GLfloat alpha     = CHAN_TO_FLOAT(lut[jA * 2 + 1]);
+               GLfloat luminance, alpha;
+               jL = CLAMP(jL, 0, max);
+               jA = CLAMP(jA, 0, max);
+               luminance = CHAN_TO_FLOAT(lut[jL * 2 + 0]);
+               alpha     = CHAN_TO_FLOAT(lut[jA * 2 + 1]);
                rgba[i][RCOMP] = rgba[i][GCOMP] = rgba[i][BCOMP] = luminance;
                rgba[i][ACOMP] = alpha;;
             }
          }
          else {
-            const GLfloat scale = (GLfloat) (table->Size - 1);
+            const GLint max = table->Size - 1;
+            const GLfloat scale = (GLfloat) max;
             const GLfloat *lut = (const GLfloat *) table->Table;
             GLuint i;
             for (i = 0; i < n; i++) {
                GLint jL = IROUND(rgba[i][RCOMP] * scale);
                GLint jA = IROUND(rgba[i][ACOMP] * scale);
-               GLfloat luminance = lut[jL * 2 + 0];
-               GLfloat alpha     = lut[jA * 2 + 1];
+               GLfloat luminance, alpha;
+               jL = CLAMP(jL, 0, max);
+               jA = CLAMP(jA, 0, max);
+               luminance = lut[jL * 2 + 0];
+               alpha     = lut[jA * 2 + 1];
                rgba[i][RCOMP] = rgba[i][GCOMP] = rgba[i][BCOMP] = luminance;
                rgba[i][ACOMP] = alpha;;
             }
@@ -983,26 +1001,34 @@ _mesa_lookup_rgba(const struct gl_color_table *table,
       case GL_RGB:
          /* replace RGB with RGB */
          if (!table->FloatTable) {
-            const GLfloat scale = (GLfloat) (table->Size - 1);
+            const GLint max = table->Size - 1;
+            const GLfloat scale = (GLfloat) max;
             const GLchan *lut = (const GLchan *) table->Table;
             GLuint i;
             for (i = 0; i < n; i++) {
                GLint jR = IROUND(rgba[i][RCOMP] * scale);
                GLint jG = IROUND(rgba[i][GCOMP] * scale);
                GLint jB = IROUND(rgba[i][BCOMP] * scale);
+               jR = CLAMP(jR, 0, max);
+               jG = CLAMP(jG, 0, max);
+               jB = CLAMP(jB, 0, max);
                rgba[i][RCOMP] = CHAN_TO_FLOAT(lut[jR * 3 + 0]);
                rgba[i][GCOMP] = CHAN_TO_FLOAT(lut[jG * 3 + 1]);
                rgba[i][BCOMP] = CHAN_TO_FLOAT(lut[jB * 3 + 2]);
             }
          }
          else {
-            const GLfloat scale = (GLfloat) (table->Size - 1);
+            const GLint max = table->Size - 1;
+            const GLfloat scale = (GLfloat) max;
             const GLfloat *lut = (const GLfloat *) table->Table;
             GLuint i;
             for (i = 0; i < n; i++) {
                GLint jR = IROUND(rgba[i][RCOMP] * scale);
                GLint jG = IROUND(rgba[i][GCOMP] * scale);
                GLint jB = IROUND(rgba[i][BCOMP] * scale);
+               jR = CLAMP(jR, 0, max);
+               jG = CLAMP(jG, 0, max);
+               jB = CLAMP(jB, 0, max);
                rgba[i][RCOMP] = lut[jR * 3 + 0];
                rgba[i][GCOMP] = lut[jG * 3 + 1];
                rgba[i][BCOMP] = lut[jB * 3 + 2];
@@ -1012,7 +1038,8 @@ _mesa_lookup_rgba(const struct gl_color_table *table,
       case GL_RGBA:
          /* replace RGBA with RGBA */
          if (!table->FloatTable) {
-            const GLfloat scale = (GLfloat) (table->Size - 1);
+            const GLint max = table->Size - 1;
+            const GLfloat scale = (GLfloat) max;
             const GLchan *lut = (const GLchan *) table->Table;
             GLuint i;
             for (i = 0; i < n; i++) {
@@ -1020,6 +1047,10 @@ _mesa_lookup_rgba(const struct gl_color_table *table,
                GLint jG = IROUND(rgba[i][GCOMP] * scale);
                GLint jB = IROUND(rgba[i][BCOMP] * scale);
                GLint jA = IROUND(rgba[i][ACOMP] * scale);
+               jR = CLAMP(jR, 0, max);
+               jG = CLAMP(jG, 0, max);
+               jB = CLAMP(jB, 0, max);
+               jA = CLAMP(jA, 0, max);
                rgba[i][RCOMP] = CHAN_TO_FLOAT(lut[jR * 4 + 0]);
                rgba[i][GCOMP] = CHAN_TO_FLOAT(lut[jG * 4 + 1]);
                rgba[i][BCOMP] = CHAN_TO_FLOAT(lut[jB * 4 + 2]);
@@ -1027,7 +1058,8 @@ _mesa_lookup_rgba(const struct gl_color_table *table,
             }
          }
          else {
-            const GLfloat scale = (GLfloat) (table->Size - 1);
+            const GLint max = table->Size - 1;
+            const GLfloat scale = (GLfloat) max;
             const GLfloat *lut = (const GLfloat *) table->Table;
             GLuint i;
             for (i = 0; i < n; i++) {
@@ -1035,6 +1067,10 @@ _mesa_lookup_rgba(const struct gl_color_table *table,
                GLint jG = IROUND(rgba[i][GCOMP] * scale);
                GLint jB = IROUND(rgba[i][BCOMP] * scale);
                GLint jA = IROUND(rgba[i][ACOMP] * scale);
+               jR = CLAMP(jR, 0, max);
+               jG = CLAMP(jG, 0, max);
+               jB = CLAMP(jB, 0, max);
+               jA = CLAMP(jA, 0, max);
                rgba[i][RCOMP] = lut[jR * 4 + 0];
                rgba[i][GCOMP] = lut[jG * 4 + 1];
                rgba[i][BCOMP] = lut[jB * 4 + 2];