mesa: replace more MAX_WIDTH stack allocations with heap allocations

2 files changed, 111 insertions, 38 deletions

diff --git a/src/mesa/main/pack.c b/src/mesa/main/pack.c
index 2d66f8fc5e8..6d524e64908 100644
--- a/src/mesa/main/pack.c
+++ b/src/mesa/main/pack.c
@@ -484,11 +484,25 @@ _mesa_pack_rgba_span_float(struct gl_context *ctx, GLuint n, GLfloat rgba[][4],
                            const struct gl_pixelstore_attrib *dstPacking,
                            GLbitfield transferOps)
 {
-   GLfloat luminance[MAX_WIDTH];
+   GLfloat *luminance;
    const GLint comps = _mesa_components_in_format(dstFormat);
    const GLboolean intDstFormat = _mesa_is_integer_format(dstFormat);
    GLuint i;
 
+   if (dstFormat == GL_LUMINANCE ||
+       dstFormat == GL_LUMINANCE_ALPHA ||
+       dstFormat == GL_LUMINANCE_INTEGER_EXT ||
+       dstFormat == GL_LUMINANCE_ALPHA_INTEGER_EXT) {
+      luminance = (GLfloat *) malloc(n * sizeof(GLfloat));
+      if (!luminance) {
+         _mesa_error(ctx, GL_OUT_OF_MEMORY, "pixel packing");
+         return;
+      }
+   }
+   else {
+      luminance = NULL;
+   }
+
    /* XXX
     * This test should probably go away.  Have the caller set/clear the
     * IMAGE_CLAMP_BIT as needed.
@@ -1907,6 +1921,8 @@ _mesa_pack_rgba_span_float(struct gl_context *ctx, GLuint n, GLfloat rgba[][4],
          }
       }
    }
+
+   free(luminance);
 }
 
 
@@ -3476,9 +3492,14 @@ _mesa_unpack_color_span_chan( struct gl_context *ctx,
       /*
        * Extract image data and convert to RGBA floats
        */
-      assert(n <= MAX_WIDTH);
       if (srcFormat == GL_COLOR_INDEX) {
-         GLuint indexes[MAX_WIDTH];
+         GLuint *indexes = (GLuint *) malloc(n * sizeof(GLuint));
+
+         if (!indexes) {
+            _mesa_error(ctx, GL_OUT_OF_MEMORY, "pixel unpacking");
+            return;
+         }
+
          extract_uint_indexes(n, indexes, srcFormat, srcType, source,
                               srcPacking);
 
@@ -3489,6 +3510,7 @@ _mesa_unpack_color_span_chan( struct gl_context *ctx,
             for (i = 0; i < n; i++) {
                dest[i] = (GLchan) (indexes[i] & 0xff);
             }
+            free(indexes);
             free(rgba);
             return;
          }
@@ -3504,6 +3526,8 @@ _mesa_unpack_color_span_chan( struct gl_context *ctx,
           * with color indexes.
           */
          transferOps &= ~(IMAGE_SCALE_BIAS_BIT | IMAGE_MAP_COLOR_BIT);
+
+         free(indexes);
       }
       else {
          /* non-color index data */
@@ -3674,9 +3698,15 @@ _mesa_unpack_color_span_float( struct gl_context *ctx,
       /*
        * Extract image data and convert to RGBA floats
        */
-      assert(n <= MAX_WIDTH);
       if (srcFormat == GL_COLOR_INDEX) {
-         GLuint indexes[MAX_WIDTH];
+         GLuint *indexes = (GLuint *) malloc(n * sizeof(GLuint));
+
+         if (!indexes) {
+            _mesa_error(ctx, GL_OUT_OF_MEMORY, "pixel unpacking");
+            free(rgba);
+            return;
+         }
+
          extract_uint_indexes(n, indexes, srcFormat, srcType, source,
                               srcPacking);
 
@@ -3687,6 +3717,7 @@ _mesa_unpack_color_span_float( struct gl_context *ctx,
             for (i = 0; i < n; i++) {
                dest[i] = (GLchan) (indexes[i] & 0xff);
             }
+            free(indexes);
             free(rgba);
             return;
          }
@@ -3703,7 +3734,7 @@ _mesa_unpack_color_span_float( struct gl_context *ctx,
           */
          transferOps &= ~(IMAGE_SCALE_BIAS_BIT | IMAGE_MAP_COLOR_BIT);
 
-         free(rgba);
+         free(indexes);
       }
       else {
          /* non-color index data */
@@ -3776,6 +3807,8 @@ _mesa_unpack_color_span_float( struct gl_context *ctx,
             dst += dstComponents;
          }
       }
+
+      free(rgba);
    }
 }
 
@@ -3799,8 +3832,6 @@ _mesa_unpack_color_span_uint(struct gl_context *ctx,
       return;
    }
 
-   ASSERT(n <= MAX_WIDTH);
-
    ASSERT(dstFormat == GL_ALPHA ||
           dstFormat == GL_LUMINANCE ||
           dstFormat == GL_LUMINANCE_ALPHA ||
@@ -3982,7 +4013,6 @@ _mesa_unpack_dudv_span_byte( struct gl_context *ctx,
       /*
        * Extract image data and convert to RGBA floats
        */
-      assert(n <= MAX_WIDTH);
       extract_float_rgba(n, rgba, srcFormat, srcType, source,
                          srcPacking->SwapBytes);
 
@@ -4018,7 +4048,7 @@ _mesa_unpack_dudv_span_byte( struct gl_context *ctx,
  *        transferOps - the pixel transfer operations to apply
  */
 void
-_mesa_unpack_index_span( const struct gl_context *ctx, GLuint n,
+_mesa_unpack_index_span( struct gl_context *ctx, GLuint n,
                          GLenum dstType, GLvoid *dest,
                          GLenum srcType, const GLvoid *source,
                          const struct gl_pixelstore_attrib *srcPacking,
@@ -4056,8 +4086,12 @@ _mesa_unpack_index_span( const struct gl_context *ctx, GLuint n,
       /*
        * general solution
        */
-      GLuint indexes[MAX_WIDTH];
-      assert(n <= MAX_WIDTH);
+      GLuint *indexes = (GLuint *) malloc(n * sizeof(GLuint));
+
+      if (!indexes) {
+         _mesa_error(ctx, GL_OUT_OF_MEMORY, "pixel unpacking");
+         return;
+      }
 
       extract_uint_indexes(n, indexes, GL_COLOR_INDEX, srcType, source,
                            srcPacking);
@@ -4091,19 +4125,24 @@ _mesa_unpack_index_span( const struct gl_context *ctx, GLuint n,
          default:
             _mesa_problem(ctx, "bad dstType in _mesa_unpack_index_span");
       }
+
+      free(indexes);
    }
 }
 
 
 void
-_mesa_pack_index_span( const struct gl_context *ctx, GLuint n,
+_mesa_pack_index_span( struct gl_context *ctx, GLuint n,
                        GLenum dstType, GLvoid *dest, const GLuint *source,
                        const struct gl_pixelstore_attrib *dstPacking,
                        GLbitfield transferOps )
 {
-   GLuint indexes[MAX_WIDTH];
+   GLuint *indexes = (GLuint *) malloc(n * sizeof(GLuint));
 
-   ASSERT(n <= MAX_WIDTH);
+   if (!indexes) {
+      _mesa_error(ctx, GL_OUT_OF_MEMORY, "pixel packing");
+      return;
+   }
 
    transferOps &= (IMAGE_MAP_COLOR_BIT | IMAGE_SHIFT_OFFSET_BIT);
 
@@ -4208,6 +4247,8 @@ _mesa_pack_index_span( const struct gl_context *ctx, GLuint n,
    default:
       _mesa_problem(ctx, "bad type in _mesa_pack_index_span");
    }
+
+   free(indexes);
 }
 
 
@@ -4226,7 +4267,7 @@ _mesa_pack_index_span( const struct gl_context *ctx, GLuint n,
  *        transferOps - apply offset/bias/lookup ops?
  */
 void
-_mesa_unpack_stencil_span( const struct gl_context *ctx, GLuint n,
+_mesa_unpack_stencil_span( struct gl_context *ctx, GLuint n,
                            GLenum dstType, GLvoid *dest,
                            GLenum srcType, const GLvoid *source,
                            const struct gl_pixelstore_attrib *srcPacking,
@@ -4270,8 +4311,12 @@ _mesa_unpack_stencil_span( const struct gl_context *ctx, GLuint n,
       /*
        * general solution
        */
-      GLuint indexes[MAX_WIDTH];
-      assert(n <= MAX_WIDTH);
+      GLuint *indexes = (GLuint *) malloc(n * sizeof(GLuint));
+
+      if (!indexes) {
+         _mesa_error(ctx, GL_OUT_OF_MEMORY, "stencil unpacking");
+         return;
+      }
 
       extract_uint_indexes(n, indexes, GL_STENCIL_INDEX, srcType, source,
                            srcPacking);
@@ -4316,18 +4361,23 @@ _mesa_unpack_stencil_span( const struct gl_context *ctx, GLuint n,
          default:
             _mesa_problem(ctx, "bad dstType in _mesa_unpack_stencil_span");
       }
+
+      free(indexes);
    }
 }
 
 
 void
-_mesa_pack_stencil_span( const struct gl_context *ctx, GLuint n,
+_mesa_pack_stencil_span( struct gl_context *ctx, GLuint n,
                          GLenum dstType, GLvoid *dest, const GLstencil *source,
                          const struct gl_pixelstore_attrib *dstPacking )
 {
-   GLstencil stencil[MAX_WIDTH];
+   GLstencil *stencil = (GLstencil *) malloc(n * sizeof(GLstencil));
 
-   ASSERT(n <= MAX_WIDTH);
+   if (!stencil) {
+      _mesa_error(ctx, GL_OUT_OF_MEMORY, "stencil packing");
+      return;
+   }
 
    if (ctx->Pixel.IndexShift || ctx->Pixel.IndexOffset ||
        ctx->Pixel.MapStencilFlag) {
@@ -4466,6 +4516,8 @@ _mesa_pack_stencil_span( const struct gl_context *ctx, GLuint n,
    default:
       _mesa_problem(ctx, "bad type in _mesa_pack_index_span");
    }
+
+   free(stencil);
 }
 
 #define DEPTH_VALUES(GLTYPE, GLTYPE2FLOAT)                              \
@@ -4496,12 +4548,12 @@ _mesa_pack_stencil_span( const struct gl_context *ctx, GLuint n,
  *                  (ignored for GLfloat).
  */
 void
-_mesa_unpack_depth_span( const struct gl_context *ctx, GLuint n,
+_mesa_unpack_depth_span( struct gl_context *ctx, GLuint n,
                          GLenum dstType, GLvoid *dest, GLuint depthMax,
                          GLenum srcType, const GLvoid *source,
                          const struct gl_pixelstore_attrib *srcPacking )
 {
-   GLfloat depthTemp[MAX_WIDTH], *depthValues;
+   GLfloat *depthTemp, *depthValues;
    GLboolean needClamp = GL_FALSE;
 
    /* Look for special cases first.
@@ -4547,6 +4599,12 @@ _mesa_unpack_depth_span( const struct gl_context *ctx, GLuint n,
 
    /* general case path follows */
 
+   depthTemp = (GLfloat *) malloc(n * sizeof(GLfloat));
+   if (!depthTemp) {
+      _mesa_error(ctx, GL_OUT_OF_MEMORY, "pixel unpacking");
+      return;
+   }
+
    if (dstType == GL_FLOAT) {
       depthValues = (GLfloat *) dest;
    }
@@ -4629,6 +4687,7 @@ _mesa_unpack_depth_span( const struct gl_context *ctx, GLuint n,
          break;
       default:
          _mesa_problem(NULL, "bad type in _mesa_unpack_depth_span()");
+         free(depthTemp);
          return;
    }
 
@@ -4688,6 +4747,8 @@ _mesa_unpack_depth_span( const struct gl_context *ctx, GLuint n,
       ASSERT(dstType == GL_FLOAT);
       /*ASSERT(depthMax == 1.0F);*/
    }
+
+   free(depthTemp);
 }
 
 
@@ -4695,13 +4756,15 @@ _mesa_unpack_depth_span( const struct gl_context *ctx, GLuint n,
  * Pack an array of depth values.  The values are floats in [0,1].
  */
 void
-_mesa_pack_depth_span( const struct gl_context *ctx, GLuint n, GLvoid *dest,
+_mesa_pack_depth_span( struct gl_context *ctx, GLuint n, GLvoid *dest,
                        GLenum dstType, const GLfloat *depthSpan,
                        const struct gl_pixelstore_attrib *dstPacking )
 {
-   GLfloat depthCopy[MAX_WIDTH];
-
-   ASSERT(n <= MAX_WIDTH);
+   GLfloat *depthCopy = (GLfloat *) malloc(n * sizeof(GLfloat));
+   if (!depthCopy) {
+      _mesa_error(ctx, GL_OUT_OF_MEMORY, "pixel packing");
+      return;
+   }
 
    if (ctx->Pixel.DepthScale != 1.0 || ctx->Pixel.DepthBias != 0.0) {
       memcpy(depthCopy, depthSpan, n * sizeof(GLfloat));
@@ -4803,6 +4866,8 @@ _mesa_pack_depth_span( const struct gl_context *ctx, GLuint n, GLvoid *dest,
    default:
       _mesa_problem(ctx, "bad type in _mesa_pack_depth_span");
    }
+
+   free(depthCopy);
 }
 
 
@@ -4811,16 +4876,21 @@ _mesa_pack_depth_span( const struct gl_context *ctx, GLuint n, GLvoid *dest,
  * Pack depth and stencil values as GL_DEPTH_STENCIL/GL_UNSIGNED_INT_24_8.
  */
 void
-_mesa_pack_depth_stencil_span(const struct gl_context *ctx, GLuint n, GLuint *dest,
+_mesa_pack_depth_stencil_span(struct gl_context *ctx, GLuint n, GLuint *dest,
                               const GLfloat *depthVals,
                               const GLstencil *stencilVals,
                               const struct gl_pixelstore_attrib *dstPacking)
 {
-   GLfloat depthCopy[MAX_WIDTH];
-   GLstencil stencilCopy[MAX_WIDTH];
+   GLfloat *depthCopy = (GLfloat *) malloc(n * sizeof(GLfloat));
+   GLstencil *stencilCopy = (GLstencil *) malloc(n * sizeof(GLstencil));
    GLuint i;
 
-   ASSERT(n <= MAX_WIDTH);
+   if (!depthCopy || !stencilCopy) {
+      _mesa_error(ctx, GL_OUT_OF_MEMORY, "pixel packing");
+      free(depthCopy);
+      free(stencilCopy);
+      return;
+   }
 
    if (ctx->Pixel.DepthScale != 1.0 || ctx->Pixel.DepthBias != 0.0) {
       memcpy(depthCopy, depthVals, n * sizeof(GLfloat));
@@ -4844,6 +4914,9 @@ _mesa_pack_depth_stencil_span(const struct gl_context *ctx, GLuint n, GLuint *de
    if (dstPacking->SwapBytes) {
       _mesa_swap4(dest, n);
    }
+
+   free(depthCopy);
+   free(stencilCopy);
 }
 
 
diff --git a/src/mesa/main/pack.h b/src/mesa/main/pack.h
index 65d3f7a0fb2..78238ea5839 100644
--- a/src/mesa/main/pack.h
+++ b/src/mesa/main/pack.h
@@ -90,7 +90,7 @@ _mesa_unpack_dudv_span_byte(struct gl_context *ctx,
                             GLbitfield transferOps);
 
 extern void
-_mesa_unpack_index_span(const struct gl_context *ctx, GLuint n,
+_mesa_unpack_index_span(struct gl_context *ctx, GLuint n,
                         GLenum dstType, GLvoid *dest,
                         GLenum srcType, const GLvoid *source,
                         const struct gl_pixelstore_attrib *srcPacking,
@@ -98,39 +98,39 @@ _mesa_unpack_index_span(const struct gl_context *ctx, GLuint n,
 
 
 extern void
-_mesa_pack_index_span(const struct gl_context *ctx, GLuint n,
+_mesa_pack_index_span(struct gl_context *ctx, GLuint n,
                       GLenum dstType, GLvoid *dest, const GLuint *source,
                       const struct gl_pixelstore_attrib *dstPacking,
                       GLbitfield transferOps);
 
 
 extern void
-_mesa_unpack_stencil_span(const struct gl_context *ctx, GLuint n,
+_mesa_unpack_stencil_span(struct gl_context *ctx, GLuint n,
                           GLenum dstType, GLvoid *dest,
                           GLenum srcType, const GLvoid *source,
                           const struct gl_pixelstore_attrib *srcPacking,
                           GLbitfield transferOps);
 
 extern void
-_mesa_pack_stencil_span(const struct gl_context *ctx, GLuint n,
+_mesa_pack_stencil_span(struct gl_context *ctx, GLuint n,
                         GLenum dstType, GLvoid *dest, const GLstencil *source,
                         const struct gl_pixelstore_attrib *dstPacking);
 
 
 extern void
-_mesa_unpack_depth_span(const struct gl_context *ctx, GLuint n,
+_mesa_unpack_depth_span(struct gl_context *ctx, GLuint n,
                         GLenum dstType, GLvoid *dest, GLuint depthMax,
                         GLenum srcType, const GLvoid *source,
                         const struct gl_pixelstore_attrib *srcPacking);
 
 extern void
-_mesa_pack_depth_span(const struct gl_context *ctx, GLuint n, GLvoid *dest,
+_mesa_pack_depth_span(struct gl_context *ctx, GLuint n, GLvoid *dest,
                       GLenum dstType, const GLfloat *depthSpan,
                       const struct gl_pixelstore_attrib *dstPacking);
 
 
 extern void
-_mesa_pack_depth_stencil_span(const struct gl_context *ctx,
+_mesa_pack_depth_stencil_span(struct gl_context *ctx,
                               GLuint n, GLuint *dest,
                               const GLfloat *depthVals,
                               const GLstencil *stencilVals,