From 0bcad0295543425f5ec4b44bfe6e600794e9c166 Mon Sep 17 00:00:00 2001 From: Brian Paul Date: Thu, 20 Sep 2012 09:13:37 -0600 Subject: llvmpipe: fix overflow bug in total texture size computation v2: use uint64_t for the total_size variable, per Jose. Also add two earlier checks for exceeding the max texture size. For example a 1K^3 RGBA volume would overflow the lpr->image_stride variable. Use simple algebra to avoid overflow in intermediate values. So instead of "x * y > z" use "x > z / y". This should work if we happen to be on a platform that doesn't have 64-bit types. Reviewed-by: Jose Fonseca --- src/gallium/drivers/llvmpipe/lp_texture.c | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) (limited to 'src/gallium/drivers/llvmpipe') diff --git a/src/gallium/drivers/llvmpipe/lp_texture.c b/src/gallium/drivers/llvmpipe/lp_texture.c index c0a612cd067..841df004d14 100644 --- a/src/gallium/drivers/llvmpipe/lp_texture.c +++ b/src/gallium/drivers/llvmpipe/lp_texture.c @@ -113,7 +113,7 @@ llvmpipe_texture_layout(struct llvmpipe_screen *screen, unsigned width = pt->width0; unsigned height = pt->height0; unsigned depth = pt->depth0; - size_t total_size = 0; + uint64_t total_size = 0; assert(LP_MAX_TEXTURE_2D_LEVELS <= LP_MAX_TEXTURE_LEVELS); assert(LP_MAX_TEXTURE_3D_LEVELS <= LP_MAX_TEXTURE_LEVELS); @@ -140,6 +140,12 @@ llvmpipe_texture_layout(struct llvmpipe_screen *screen, lpr->row_stride[level] = align(nblocksx * block_size, 16); + /* if row_stride * height > LP_MAX_TEXTURE_SIZE */ + if (lpr->row_stride[level] > LP_MAX_TEXTURE_SIZE / nblocksy) { + /* image too large */ + goto fail; + } + lpr->img_stride[level] = lpr->row_stride[level] * nblocksy; } @@ -172,7 +178,15 @@ llvmpipe_texture_layout(struct llvmpipe_screen *screen, } } - total_size += lpr->num_slices_faces[level] * lpr->img_stride[level]; + /* if img_stride * num_slices_faces > LP_MAX_TEXTURE_SIZE */ + if (lpr->img_stride[level] > + LP_MAX_TEXTURE_SIZE / lpr->num_slices_faces[level]) { + /* volume too large */ + goto fail; + } + + total_size += (uint64_t) lpr->num_slices_faces[level] + * (uint64_t) lpr->img_stride[level]; if (total_size > LP_MAX_TEXTURE_SIZE) { goto fail; } -- cgit v1.2.3