swr: Correct texture allocation and limit max size to 2GB

This patch fixes piglit tex3d-maxsize by correcting 4 things:

The total_size calculation was using 32-bit math, therefore a >4GB
allocation request overflowed and was not returning false (unsupported).

Changed AlignedMalloc arguments from "unsigned int" to size_t, to handle
>4GB allocations.

Added error checking on texture allocations to fail gracefully.

Finally, temporarily decreased supported max texture size from 4GB to 2GB.
The gallivm texture-sampler needs some additional work to correctly handle
larger than 2GB textures (offsets to LLVMBuildGEP are signed).

I'm working on a follow-on patch to allow up to 4GB textures, as this is
useful in HPC visualization applications.

Fixes piglit tex3d-maxsize.

v2: Updated patch description to clarify ">4GB".

Reviewed-By: George Kyriazis <[email protected]>

2 files changed, 10 insertions, 4 deletions

diff --git a/src/gallium/drivers/swr/rasterizer/common/os.h b/src/gallium/drivers/swr/rasterizer/common/os.h
index 4ed6b88e454..358cb33b6e0 100644
--- a/src/gallium/drivers/swr/rasterizer/common/os.h
+++ b/src/gallium/drivers/swr/rasterizer/common/os.h
@@ -210,7 +210,7 @@ unsigned char _BitScanReverse(unsigned int *Index, unsigned int Mask)
 }
 
 inline
-void *AlignedMalloc(unsigned int size, unsigned int alignment)
+void *AlignedMalloc(size_t size, size_t alignment)
 {
     void *ret;
     if (posix_memalign(&ret, alignment, size))
diff --git a/src/gallium/drivers/swr/swr_screen.cpp b/src/gallium/drivers/swr/swr_screen.cpp
index 3f2433e65a9..71a07ebe8d2 100644
--- a/src/gallium/drivers/swr/swr_screen.cpp
+++ b/src/gallium/drivers/swr/swr_screen.cpp
@@ -50,7 +50,7 @@
  * Max texture sizes
  * XXX Check max texture size values against core and sampler.
  */
-#define SWR_MAX_TEXTURE_SIZE (4 * 1024 * 1024 * 1024ULL) /* 4GB */
+#define SWR_MAX_TEXTURE_SIZE (2 * 1024 * 1024 * 1024ULL) /* 2GB */
 #define SWR_MAX_TEXTURE_2D_LEVELS 14  /* 8K x 8K for now */
 #define SWR_MAX_TEXTURE_3D_LEVELS 12  /* 2K x 2K x 2K for now */
 #define SWR_MAX_TEXTURE_CUBE_LEVELS 14  /* 8K x 8K for now */
@@ -821,13 +821,15 @@ swr_texture_layout(struct swr_screen *screen,
          ComputeSurfaceOffset<false>(0, 0, 0, 0, 0, level, &res->swr);
    }
 
-   size_t total_size = res->swr.depth * res->swr.qpitch * res->swr.pitch *
-                       res->swr.numSamples;
+   size_t total_size = (uint64_t)res->swr.depth * res->swr.qpitch *
+                                 res->swr.pitch * res->swr.numSamples;
    if (total_size > SWR_MAX_TEXTURE_SIZE)
       return false;
 
    if (allocate) {
       res->swr.xpBaseAddress = (gfxptr_t)AlignedMalloc(total_size, 64);
+      if (!res->swr.xpBaseAddress)
+         return false;
 
       if (res->has_depth && res->has_stencil) {
          res->secondary = res->swr;
@@ -843,6 +845,10 @@ swr_texture_layout(struct swr_screen *screen,
                       res->secondary.pitch * res->secondary.numSamples;
 
          res->secondary.xpBaseAddress = (gfxptr_t) AlignedMalloc(total_size, 64);
+         if (!res->secondary.xpBaseAddress) {
+            AlignedFree((void *)res->swr.xpBaseAddress);
+            return false;
+         }
       }
    }