mesa: handle MapTextureImage() failures in glTexImage code

Reviewed-by: Eric Anholt <[email protected]>

1 files changed, 123 insertions, 76 deletions

diff --git a/src/mesa/main/texstore.c b/src/mesa/main/texstore.c
index 892f07da8bd..6319e70e6c5 100644
--- a/src/mesa/main/texstore.c
+++ b/src/mesa/main/texstore.c
@@ -4596,16 +4596,20 @@ _mesa_store_teximage1d(struct gl_context *ctx, GLenum target, GLint level,
                                0, 0, width, 1,
                                rwMode,
                                &dstMap, &dstRowStride);
+   if (dstMap) {
+      success = _mesa_texstore(ctx, 1, texImage->_BaseFormat,
+                               texImage->TexFormat,
+                               0, 0, 0,  /* dstX/Y/Zoffset */
+                               0, /* dstRowStride */
+                               &dstMap,
+                               width, 1, 1,
+                               format, type, pixels, packing);
 
-   success = _mesa_texstore(ctx, 1, texImage->_BaseFormat,
-                            texImage->TexFormat,
-                            0, 0, 0,  /* dstX/Y/Zoffset */
-                            0, /* dstRowStride */
-                            &dstMap,
-                            width, 1, 1,
-                            format, type, pixels, packing);
-
-   ctx->Driver.UnmapTextureImage(ctx, texImage, 0);
+      ctx->Driver.UnmapTextureImage(ctx, texImage, 0);
+   }
+   else {
+      success = GL_FALSE;
+   }
 
    if (!success)
       _mesa_error(ctx, GL_OUT_OF_MEMORY, "glTexImage1D");
@@ -4668,15 +4672,19 @@ _mesa_store_teximage2d(struct gl_context *ctx, GLenum target, GLint level,
                                      0, 0, width, 1,
                                      rwMode,
                                      &dstMap, &dstRowStride);
-         assert(dstMap);
-         success = _mesa_texstore(ctx, 2, texImage->_BaseFormat,
-                                  texImage->TexFormat,
-                                  0, 0, 0,  /* dstX/Y/Zoffset */
-                                  dstRowStride,
-                                  &dstMap,
-                                  width, 1, 1,
-                                  format, type, pixels, packing);
-         ctx->Driver.UnmapTextureImage(ctx, texImage, y);
+         if (dstMap) {
+            success = _mesa_texstore(ctx, 2, texImage->_BaseFormat,
+                                     texImage->TexFormat,
+                                     0, 0, 0,  /* dstX/Y/Zoffset */
+                                     dstRowStride,
+                                     &dstMap,
+                                     width, 1, 1,
+                                     format, type, pixels, packing);
+            ctx->Driver.UnmapTextureImage(ctx, texImage, y);
+         }
+         else {
+            success = GL_FALSE;
+         }
 
          if (!success)
             break;
@@ -4689,16 +4697,20 @@ _mesa_store_teximage2d(struct gl_context *ctx, GLenum target, GLint level,
                                   0, 0, width, height,
                                   rwMode,
                                   &dstMap, &dstRowStride);
-      assert(dstMap);
-      success = _mesa_texstore(ctx, 2, texImage->_BaseFormat,
-                               texImage->TexFormat,
-                               0, 0, 0,  /* dstX/Y/Zoffset */
-                               dstRowStride,
-                               &dstMap,
-                               width, height, 1,
-                               format, type, pixels, packing);
+      if (dstMap) {
+         success = _mesa_texstore(ctx, 2, texImage->_BaseFormat,
+                                  texImage->TexFormat,
+                                  0, 0, 0,  /* dstX/Y/Zoffset */
+                                  dstRowStride,
+                                  &dstMap,
+                                  width, height, 1,
+                                  format, type, pixels, packing);
 
-      ctx->Driver.UnmapTextureImage(ctx, texImage, 0);
+         ctx->Driver.UnmapTextureImage(ctx, texImage, 0);
+      }
+      else {
+         success = GL_FALSE;
+      }
    }
 
    if (!success)
@@ -4723,7 +4735,7 @@ _mesa_store_teximage3d(struct gl_context *ctx, GLenum target, GLint level,
                        struct gl_texture_image *texImage)
 {
    const GLbitfield rwMode = get_read_write_mode(format, texImage->TexFormat);
-   GLboolean success;
+   GLboolean success = GL_TRUE;
    GLint slice;
    GLubyte **sliceMaps;
    GLint dstRowStride;
@@ -4755,7 +4767,7 @@ _mesa_store_teximage3d(struct gl_context *ctx, GLenum target, GLint level,
       height = 1;
    }
 
-   sliceMaps = (GLubyte **) malloc(depth * sizeof(GLubyte *));
+   sliceMaps = (GLubyte **) calloc(depth, sizeof(GLubyte *));
 
    /* Map dest texture buffer slices */
    for (slice = 0; slice < depth; slice++) {
@@ -4763,19 +4775,27 @@ _mesa_store_teximage3d(struct gl_context *ctx, GLenum target, GLint level,
                                   0, 0, width, height,
                                   rwMode,
                                   &sliceMaps[slice], &dstRowStride);
+      if (!sliceMaps[slice]) {
+         success = GL_FALSE;
+         break;
+      }
    }
 
-   success = _mesa_texstore(ctx, 3, texImage->_BaseFormat,
-                            texImage->TexFormat,
-                            0, 0, 0,  /* dstX/Y/Zoffset */
-                            dstRowStride,
-                            sliceMaps,
-                            width, height, depth,
-                            format, type, pixels, packing);
+   if (success) {
+      success = _mesa_texstore(ctx, 3, texImage->_BaseFormat,
+                               texImage->TexFormat,
+                               0, 0, 0,  /* dstX/Y/Zoffset */
+                               dstRowStride,
+                               sliceMaps,
+                               width, height, depth,
+                               format, type, pixels, packing);
+   }
 
    /* Unmap dest texture buffer slices */
    for (slice = 0; slice < depth; slice++) {
-      ctx->Driver.UnmapTextureImage(ctx, texImage, slice);
+      if (sliceMaps[slice]) {
+         ctx->Driver.UnmapTextureImage(ctx, texImage, slice);
+      }
    }
 
    if (!success)
@@ -4818,15 +4838,20 @@ _mesa_store_texsubimage1d(struct gl_context *ctx, GLenum target, GLint level,
                                rwMode,
                                &dstMap, &dstRowStride);
 
-   success = _mesa_texstore(ctx, 1, texImage->_BaseFormat,
-                            texImage->TexFormat,
-                            0, 0, 0,  /* dstX/Y/Zoffset */
-                            dstRowStride,
-                            &dstMap,
-                            width, 1, 1,
-                            format, type, pixels, packing);
+   if (dstMap) {
+      success = _mesa_texstore(ctx, 1, texImage->_BaseFormat,
+                               texImage->TexFormat,
+                               0, 0, 0,  /* dstX/Y/Zoffset */
+                               dstRowStride,
+                               &dstMap,
+                               width, 1, 1,
+                               format, type, pixels, packing);
 
-   ctx->Driver.UnmapTextureImage(ctx, texImage, 0);
+      ctx->Driver.UnmapTextureImage(ctx, texImage, 0);
+   }
+   else {
+      success = GL_FALSE;
+   }
 
    if (!success)
       _mesa_error(ctx, GL_OUT_OF_MEMORY, "glTexSubImage1D");
@@ -4866,15 +4891,20 @@ _mesa_store_texsubimage2d(struct gl_context *ctx, GLenum target, GLint level,
                                rwMode,
                                &dstMap, &dstRowStride);
 
-   success = _mesa_texstore(ctx, 2, texImage->_BaseFormat,
-                            texImage->TexFormat,
-                            0, 0, 0,  /* dstX/Y/Zoffset */
-                            dstRowStride,
-                            &dstMap,
-                            width, height, 1,
-                            format, type, pixels, packing);
+   if (dstMap) {
+      success = _mesa_texstore(ctx, 2, texImage->_BaseFormat,
+                               texImage->TexFormat,
+                               0, 0, 0,  /* dstX/Y/Zoffset */
+                               dstRowStride,
+                               &dstMap,
+                               width, height, 1,
+                               format, type, pixels, packing);
 
-   ctx->Driver.UnmapTextureImage(ctx, texImage, 0);
+      ctx->Driver.UnmapTextureImage(ctx, texImage, 0);
+   }
+   else {
+      success = GL_FALSE;
+   }
 
    if (!success)
       _mesa_error(ctx, GL_OUT_OF_MEMORY, "glTexSubImage2D");
@@ -4897,7 +4927,7 @@ _mesa_store_texsubimage3d(struct gl_context *ctx, GLenum target, GLint level,
                           struct gl_texture_image *texImage)
 {
    const GLbitfield rwMode = get_read_write_mode(format, texImage->TexFormat);
-   GLboolean success;
+   GLboolean success = GL_TRUE;
    GLint slice;
    GLubyte **sliceMaps;
    GLint dstRowStride;
@@ -4909,7 +4939,7 @@ _mesa_store_texsubimage3d(struct gl_context *ctx, GLenum target, GLint level,
    if (!pixels)
       return;
 
-   sliceMaps = (GLubyte **) malloc(depth * sizeof(GLubyte *));
+   sliceMaps = (GLubyte **) calloc(depth, sizeof(GLubyte *));
 
    /* Map dest texture buffer slices */
    for (slice = 0; slice < depth; slice++) {
@@ -4917,19 +4947,27 @@ _mesa_store_texsubimage3d(struct gl_context *ctx, GLenum target, GLint level,
                                   xoffset, yoffset, width, height,
                                   rwMode,
                                   &sliceMaps[slice], &dstRowStride);
+      if (!sliceMaps[slice]) {
+         success = GL_FALSE;
+         break;
+      }
    }
 
-   success = _mesa_texstore(ctx, 3, texImage->_BaseFormat,
-                            texImage->TexFormat,
-                            0, 0, 0,
-                            dstRowStride,
-                            sliceMaps,
-                            width, height, depth,
-                            format, type, pixels, packing);
+   if (success) {
+      success = _mesa_texstore(ctx, 3, texImage->_BaseFormat,
+                               texImage->TexFormat,
+                               0, 0, 0,
+                               dstRowStride,
+                               sliceMaps,
+                               width, height, depth,
+                               format, type, pixels, packing);
+   }
 
    /* Unmap dest texture buffer slices */
    for (slice = 0; slice < depth; slice++) {
-      ctx->Driver.UnmapTextureImage(ctx, texImage, zoffset + slice);
+      if (sliceMaps[slice]) {
+         ctx->Driver.UnmapTextureImage(ctx, texImage, zoffset + slice);
+      }
    }
 
    if (!success)
@@ -5008,11 +5046,15 @@ _mesa_store_compressed_teximage2d(struct gl_context *ctx,
                                0, 0, width, height,
                                GL_MAP_WRITE_BIT,
                                &dstMap, &dstRowStride);
+   if (dstMap) {
+      /* copy the data */
+      memcpy(dstMap, data, imageSize);
 
-   /* copy the data */
-   memcpy(dstMap, data, imageSize);
-
-   ctx->Driver.UnmapTextureImage(ctx, texImage, 0);
+      ctx->Driver.UnmapTextureImage(ctx, texImage, 0);
+   }
+   else {
+      _mesa_error(ctx, GL_OUT_OF_MEMORY, "glCompressedTexImage2D");
+   }
 
    _mesa_unmap_teximage_pbo(ctx, &ctx->Unpack);
 }
@@ -5112,17 +5154,22 @@ _mesa_store_compressed_texsubimage2d(struct gl_context *ctx, GLenum target,
                                GL_MAP_WRITE_BIT,
                                &dstMap, &dstRowStride);
 
-   bytesPerRow = srcRowStride;  /* bytes per row of blocks */
-   rows = height / bh;  /* rows in blocks */
+   if (dstMap) {
+      bytesPerRow = srcRowStride;  /* bytes per row of blocks */
+      rows = height / bh;  /* rows in blocks */
 
-   /* copy rows of blocks */
-   for (i = 0; i < rows; i++) {
-      memcpy(dstMap, src, bytesPerRow);
-      dstMap += dstRowStride;
-      src += srcRowStride;
-   }
+      /* copy rows of blocks */
+      for (i = 0; i < rows; i++) {
+         memcpy(dstMap, src, bytesPerRow);
+         dstMap += dstRowStride;
+         src += srcRowStride;
+      }
 
-   ctx->Driver.UnmapTextureImage(ctx, texImage, 0);
+      ctx->Driver.UnmapTextureImage(ctx, texImage, 0);
+   }
+   else {
+      _mesa_error(ctx, GL_OUT_OF_MEMORY, "glCompressedTexSubImage2D");
+   }
 
    _mesa_unmap_teximage_pbo(ctx, &ctx->Unpack);
 }