gallivm: Support multiple pixels in lp_build_fetch_rgba_aos().

This allows to do the unpacking of formats that fit in 4 x unorm8 in
parallel, 4 pixels at a time.

6 files changed, 202 insertions, 112 deletions

diff --git a/src/gallium/auxiliary/draw/draw_llvm_translate.c b/src/gallium/auxiliary/draw/draw_llvm_translate.c
index ec7d0a455c7..6ebe1f7de47 100644
--- a/src/gallium/auxiliary/draw/draw_llvm_translate.c
+++ b/src/gallium/auxiliary/draw/draw_llvm_translate.c
@@ -495,5 +495,5 @@ draw_llvm_translate_from(LLVMBuilderRef builder,
 
    format_desc = util_format_description(from_format);
    zero = LLVMConstNull(LLVMInt32Type());
-   return lp_build_fetch_rgba_aos(builder, format_desc, type, vbuffer, zero, zero);
+   return lp_build_fetch_rgba_aos(builder, format_desc, type, vbuffer, zero, zero, zero);
 }
diff --git a/src/gallium/auxiliary/gallivm/lp_bld_format.h b/src/gallium/auxiliary/gallivm/lp_bld_format.h
index 9056eae9403..60e22d727ad 100644
--- a/src/gallium/auxiliary/gallivm/lp_bld_format.h
+++ b/src/gallium/auxiliary/gallivm/lp_bld_format.h
@@ -61,7 +61,8 @@ LLVMValueRef
 lp_build_fetch_rgba_aos(LLVMBuilderRef builder,
                         const struct util_format_description *format_desc,
                         struct lp_type type,
-                        LLVMValueRef ptr,
+                        LLVMValueRef base_ptr,
+                        LLVMValueRef offset,
                         LLVMValueRef i,
                         LLVMValueRef j);
 
@@ -105,11 +106,12 @@ lp_build_fetch_rgba_soa(LLVMBuilderRef builder,
 
 
 LLVMValueRef
-lp_build_unpack_subsampled_to_rgba_aos(LLVMBuilderRef builder,
-                                       const struct util_format_description *format_desc,
-                                       unsigned n,
-                                       LLVMValueRef packed,
-                                       LLVMValueRef i,
-                                       LLVMValueRef j);
+lp_build_fetch_subsampled_rgba_aos(LLVMBuilderRef builder,
+                                   const struct util_format_description *format_desc,
+                                   unsigned n,
+                                   LLVMValueRef base_ptr,
+                                   LLVMValueRef offset,
+                                   LLVMValueRef i,
+                                   LLVMValueRef j);
 
 #endif /* !LP_BLD_FORMAT_H */
diff --git a/src/gallium/auxiliary/gallivm/lp_bld_format_aos.c b/src/gallium/auxiliary/gallivm/lp_bld_format_aos.c
index c3d1fc7c3dc..88a5093979d 100644
--- a/src/gallium/auxiliary/gallivm/lp_bld_format_aos.c
+++ b/src/gallium/auxiliary/gallivm/lp_bld_format_aos.c
@@ -102,7 +102,9 @@ format_matches_type(const struct util_format_description *desc,
    assert(type.length % 4 == 0);
 
    if (desc->layout != UTIL_FORMAT_LAYOUT_PLAIN ||
-       desc->colorspace != UTIL_FORMAT_COLORSPACE_RGB) {
+       desc->colorspace != UTIL_FORMAT_COLORSPACE_RGB ||
+       desc->block.width != 1 ||
+       desc->block.height != 1) {
       return FALSE;
    }
 
@@ -137,18 +139,15 @@ format_matches_type(const struct util_format_description *desc,
  * Unpack a single pixel into its RGBA components.
  *
  * @param desc  the pixel format for the packed pixel value
- * @param type  the desired return type (float[4] vs. ubyte[4])
  * @param packed integer pixel in a format such as PIPE_FORMAT_B8G8R8A8_UNORM
  *
  * @return RGBA in a float[4] or ubyte[4] or ushort[4] vector.
  */
 static INLINE LLVMValueRef
-lp_build_unpack_rgba_aos(const struct util_format_description *desc,
-                         struct lp_build_context *bld,
-                         LLVMValueRef packed)
+lp_build_unpack_arith_rgba_aos(LLVMBuilderRef builder,
+                               const struct util_format_description *desc,
+                               LLVMValueRef packed)
 {
-   LLVMBuilderRef builder = bld->builder;
-   struct lp_type type = bld->type;
    LLVMValueRef shifted, casted, scaled, masked;
    LLVMValueRef shifts[4];
    LLVMValueRef masks[4];
@@ -167,8 +166,7 @@ lp_build_unpack_rgba_aos(const struct util_format_description *desc,
 
    /* Do the intermediate integer computations with 32bit integers since it
     * matches floating point size */
-   if (desc->block.bits < 32)
-      packed = LLVMBuildZExt(builder, packed, LLVMInt32Type(), "");
+   assert (LLVMTypeOf(packed) == LLVMInt32Type());
 
    /* Broadcast the packed value to all four channels
     * before: packed = BGRA
@@ -246,20 +244,6 @@ lp_build_unpack_rgba_aos(const struct util_format_description *desc,
    else
       scaled = casted;
 
-   /*
-    * Type conversion.
-    *
-    * TODO: We could avoid floating conversion for integer to
-    * integer conversions.
-    */
-
-   lp_build_conv(builder,
-                 lp_float32_vec4_type(),
-                 type,
-                 &scaled, 1, &scaled, 1);
-
-   scaled = lp_build_format_swizzle_aos(desc, bld, scaled);
-
    return scaled;
 }
 
@@ -382,17 +366,51 @@ LLVMValueRef
 lp_build_fetch_rgba_aos(LLVMBuilderRef builder,
                         const struct util_format_description *format_desc,
                         struct lp_type type,
-                        LLVMValueRef ptr,
+                        LLVMValueRef base_ptr,
+                        LLVMValueRef offset,
                         LLVMValueRef i,
                         LLVMValueRef j)
 {
+   unsigned num_pixels = type.length / 4;
    struct lp_build_context bld;
 
-   /* XXX: For now we only support one pixel at a time */
-   assert(type.length == 4);
+   assert(type.length <= LP_MAX_VECTOR_LENGTH);
+   assert(type.length % 4 == 0);
 
    lp_build_context_init(&bld, builder, type);
 
+   /*
+    * Trivial case
+    *
+    * The format matches the type (apart of a swizzle) so no need for
+    * scaling or converting.
+    */
+
+   if (format_matches_type(format_desc, type) &&
+       format_desc->block.bits <= type.width * 4 &&
+       util_is_pot(format_desc->block.bits)) {
+      LLVMValueRef packed;
+
+      /*
+       * The format matches the type (apart of a swizzle) so no need for
+       * scaling or converting.
+       */
+
+      packed = lp_build_gather(builder, type.length/4,
+                               format_desc->block.bits, type.width*4,
+                               base_ptr, offset);
+
+      assert(format_desc->block.bits <= type.width * type.length);
+
+      packed = LLVMBuildBitCast(builder, packed, lp_build_vec_type(type), "");
+
+      return lp_build_format_swizzle_aos(format_desc, &bld, packed);
+   }
+
+   /*
+    * Bit arithmetic
+    */
+
    if (format_desc->layout == UTIL_FORMAT_LAYOUT_PLAIN &&
        (format_desc->colorspace == UTIL_FORMAT_COLORSPACE_RGB ||
         format_desc->colorspace == UTIL_FORMAT_COLORSPACE_ZS) &&
@@ -403,56 +421,74 @@ lp_build_fetch_rgba_aos(LLVMBuilderRef builder,
        format_desc->is_bitmask &&
        !format_desc->is_mixed &&
        (format_desc->channel[0].type == UTIL_FORMAT_TYPE_UNSIGNED ||
-        format_desc->channel[1].type == UTIL_FORMAT_TYPE_UNSIGNED))
-   {
-      LLVMValueRef packed;
+        format_desc->channel[1].type == UTIL_FORMAT_TYPE_UNSIGNED)) {
 
-      ptr = LLVMBuildBitCast(builder, ptr,
-                             LLVMPointerType(LLVMIntType(format_desc->block.bits), 0) ,
-                             "");
+      LLVMValueRef tmps[LP_MAX_VECTOR_LENGTH/4];
+      LLVMValueRef res;
+      unsigned k;
 
-      packed = LLVMBuildLoad(builder, ptr, "packed");
-
-      if (format_matches_type(format_desc, type)) {
-         /*
-          * The format matches the type (apart of a swizzle) so no need for
-          * scaling or converting.
-          */
+      /*
+       * Unpack a pixel at a time into a <4 x float> RGBA vector
+       */
 
-         assert(format_desc->block.bits <= type.width * type.length);
-         if (format_desc->block.bits < type.width * type.length) {
-            packed = LLVMBuildZExt(builder, packed,
-                                   LLVMIntType(type.width * type.length), "");
-         }
+      for (k = 0; k < num_pixels; ++k) {
+         LLVMValueRef packed;
 
-         packed = LLVMBuildBitCast(builder, packed, lp_build_vec_type(type), "");
+         packed = lp_build_gather_elem(builder, num_pixels,
+                                       format_desc->block.bits, 32,
+                                       base_ptr, offset, k);
 
-         return lp_build_format_swizzle_aos(format_desc, &bld, packed);
-      } else {
-         return lp_build_unpack_rgba_aos(format_desc, &bld, packed);
+         tmps[k] = lp_build_unpack_arith_rgba_aos(builder, format_desc,
+                                                  packed);
       }
+
+      /*
+       * Type conversion.
+       *
+       * TODO: We could avoid floating conversion for integer to
+       * integer conversions.
+       */
+
+      lp_build_conv(builder,
+                    lp_float32_vec4_type(),
+                    type,
+                    tmps, num_pixels, &res, 1);
+
+      return lp_build_format_swizzle_aos(format_desc, &bld, res);
    }
-   else if (format_desc->layout == UTIL_FORMAT_LAYOUT_SUBSAMPLED) {
-      LLVMValueRef packed;
-      LLVMValueRef rgba;
 
-      ptr = LLVMBuildBitCast(builder, ptr,
-                             LLVMPointerType(LLVMInt32Type(), 0),
-                             "packed_ptr");
+   /*
+    * YUV / subsampled formats
+    */
+
+   if (format_desc->layout == UTIL_FORMAT_LAYOUT_SUBSAMPLED) {
+      struct lp_type tmp_type;
+      LLVMValueRef tmp;
 
-      packed = LLVMBuildLoad(builder, ptr, "packed");
+      memset(&tmp_type, 0, sizeof tmp_type);
+      tmp_type.width = 8;
+      tmp_type.length = num_pixels * 4;
+      tmp_type.norm = TRUE;
 
-      rgba = lp_build_unpack_subsampled_to_rgba_aos(builder, format_desc,
-                                                    1, packed, i, j);
+      tmp = lp_build_fetch_subsampled_rgba_aos(builder,
+                                               format_desc,
+                                               num_pixels,
+                                               base_ptr,
+                                               offset,
+                                               i, j);
 
       lp_build_conv(builder,
-                    lp_unorm8_vec4_type(),
-                    type,
-                    &rgba, 1, &rgba, 1);
+                    tmp_type, type,
+                    &tmp, 1, &tmp, 1);
 
-      return rgba;
+      return tmp;
    }
-   else if (format_desc->fetch_rgba_float) {
+
+   /*
+    * Fallback to util_format_description::fetch_rgba_float().
+    */
+
+   if (format_desc->fetch_rgba_float) {
       /*
        * Fallback to calling util_format_description::fetch_rgba_float.
        *
@@ -469,8 +505,9 @@ lp_build_fetch_rgba_aos(LLVMBuilderRef builder,
       LLVMTypeRef pf32t = LLVMPointerType(f32t, 0);
       LLVMValueRef function;
       LLVMValueRef tmp_ptr;
-      LLVMValueRef tmp_val;
-      LLVMValueRef args[4];
+      LLVMValueRef tmps[LP_MAX_VECTOR_LENGTH/4];
+      LLVMValueRef res;
+      unsigned k;
 
       util_snprintf(name, sizeof name, "util_format_%s_fetch_rgba_float",
                     format_desc->short_name);
@@ -508,28 +545,43 @@ lp_build_fetch_rgba_aos(LLVMBuilderRef builder,
        * in the SoA vectors.
        */
 
-      args[0] = LLVMBuildBitCast(builder, tmp_ptr, pf32t, "");
-      args[1] = ptr;
-      args[2] = i;
-      args[3] = j;
+      for (k = 0; k < num_pixels; ++k) {
+         LLVMValueRef args[4];
 
-      LLVMBuildCall(builder, function, args, Elements(args), "");
+         args[0] = LLVMBuildBitCast(builder, tmp_ptr, pf32t, "");
+         args[1] = lp_build_gather_elem_ptr(builder, num_pixels,
+                                            base_ptr, offset, k);
+
+         if (num_pixels == 1) {
+            args[2] = i;
+            args[3] = j;
+         }
+         else {
+            LLVMValueRef index = LLVMConstInt(LLVMInt32Type(), k, 0);
+            args[2] = LLVMBuildExtractElement(builder, i, index, "");
+            args[3] = LLVMBuildExtractElement(builder, j, index, "");
+         }
 
-      tmp_val = LLVMBuildLoad(builder, tmp_ptr, "");
+         LLVMBuildCall(builder, function, args, Elements(args), "");
 
-      if (type.floating) {
-         /* No further conversion necessary */
-      } else {
-         lp_build_conv(builder,
-                       lp_float32_vec4_type(),
-                       type,
-                       &tmp_val, 1, &tmp_val, 1);
+         tmps[k] = LLVMBuildLoad(builder, tmp_ptr, "");
       }
 
-      return tmp_val;
-   }
-   else {
-      assert(0);
-      return lp_build_undef(type);
+      /*
+       * Type conversion.
+       *
+       * TODO: We could avoid floating conversion for integer to
+       * integer conversions.
+       */
+
+      lp_build_conv(builder,
+                    lp_float32_vec4_type(),
+                    type,
+                    tmps, num_pixels, &res, 1);
+
+      return res;
    }
+
+   assert(0);
+   return lp_build_undef(type);
 }
diff --git a/src/gallium/auxiliary/gallivm/lp_bld_format_soa.c b/src/gallium/auxiliary/gallivm/lp_bld_format_soa.c
index e4004fbb7b5..9f405921b0a 100644
--- a/src/gallium/auxiliary/gallivm/lp_bld_format_soa.c
+++ b/src/gallium/auxiliary/gallivm/lp_bld_format_soa.c
@@ -346,18 +346,49 @@ lp_build_fetch_rgba_soa(LLVMBuilderRef builder,
                                format_desc,
                                type,
                                packed, rgba_out);
+      return;
    }
-   else {
-      /*
-       * Fallback to calling lp_build_fetch_rgba_aos for each pixel.
-       *
-       * This is not the most efficient way of fetching pixels, as we
-       * miss some opportunities to do vectorization, but this is
-       * convenient for formats or scenarios for which there was no
-       * opportunity or incentive to optimize.
-       */
 
+   /*
+    * Try calling lp_build_fetch_rgba_aos for all pixels.
+    */
+
+   if (util_format_fits_8unorm(format_desc) &&
+       type.floating && type.width == 32 && type.length == 4) {
+      struct lp_type tmp_type;
+      LLVMValueRef tmp;
+
+      memset(&tmp_type, 0, sizeof tmp_type);
+      tmp_type.width = 8;
+      tmp_type.length = type.length * 4;
+      tmp_type.norm = TRUE;
+
+      tmp = lp_build_fetch_rgba_aos(builder, format_desc, tmp_type,
+                                    base_ptr, offset, i, j);
+
+      lp_build_rgba8_to_f32_soa(builder,
+                                type,
+                                tmp,
+                                rgba_out);
+
+      return;
+   }
+
+   /*
+    * Fallback to calling lp_build_fetch_rgba_aos for each pixel.
+    *
+    * This is not the most efficient way of fetching pixels, as we
+    * miss some opportunities to do vectorization, but this is
+    * convenient for formats or scenarios for which there was no
+    * opportunity or incentive to optimize.
+    */
+
+   {
       unsigned k, chan;
+      struct lp_type tmp_type;
+
+      tmp_type = type;
+      tmp_type.length = 4;
 
       for (chan = 0; chan < 4; ++chan) {
          rgba_out[chan] = lp_build_undef(type);
@@ -367,18 +398,17 @@ lp_build_fetch_rgba_soa(LLVMBuilderRef builder,
       for(k = 0; k < type.length; ++k) {
          LLVMValueRef index = LLVMConstInt(LLVMInt32Type(), k, 0);
          LLVMValueRef offset_elem;
-         LLVMValueRef ptr;
          LLVMValueRef i_elem, j_elem;
          LLVMValueRef tmp;
 
          offset_elem = LLVMBuildExtractElement(builder, offset, index, "");
-         ptr = LLVMBuildGEP(builder, base_ptr, &offset_elem, 1, "");
 
          i_elem = LLVMBuildExtractElement(builder, i, index, "");
          j_elem = LLVMBuildExtractElement(builder, j, index, "");
 
          /* Get a single float[4]={R,G,B,A} pixel */
-         tmp = lp_build_fetch_rgba_aos(builder, format_desc, type, ptr,
+         tmp = lp_build_fetch_rgba_aos(builder, format_desc, tmp_type,
+                                       base_ptr, offset_elem,
                                        i_elem, j_elem);
 
          /*
diff --git a/src/gallium/auxiliary/gallivm/lp_bld_format_yuv.c b/src/gallium/auxiliary/gallivm/lp_bld_format_yuv.c
index d3eba50b77e..e8f4ab69ff7 100644
--- a/src/gallium/auxiliary/gallivm/lp_bld_format_yuv.c
+++ b/src/gallium/auxiliary/gallivm/lp_bld_format_yuv.c
@@ -35,9 +35,6 @@
 
 
 #include "util/u_format.h"
-#include "util/u_memory.h"
-#include "util/u_math.h"
-#include "util/u_string.h"
 
 #include "lp_bld_arit.h"
 #include "lp_bld_init.h"
@@ -359,16 +356,23 @@ grgb_to_rgba_aos(LLVMBuilderRef builder,
  * @return  a <4*n x i8> vector with the pixel RGBA values in AoS
  */
 LLVMValueRef
-lp_build_unpack_subsampled_to_rgba_aos(LLVMBuilderRef builder,
-                                       const struct util_format_description *format_desc,
-                                       unsigned n,
-                                       LLVMValueRef packed,
-                                       LLVMValueRef i,
-                                       LLVMValueRef j)
+lp_build_fetch_subsampled_rgba_aos(LLVMBuilderRef builder,
+                                   const struct util_format_description *format_desc,
+                                   unsigned n,
+                                   LLVMValueRef base_ptr,
+                                   LLVMValueRef offset,
+                                   LLVMValueRef i,
+                                   LLVMValueRef j)
 {
+   LLVMValueRef packed;
    LLVMValueRef rgba;
 
    assert(format_desc->layout == UTIL_FORMAT_LAYOUT_SUBSAMPLED);
+   assert(format_desc->block.bits == 32);
+   assert(format_desc->block.width == 2);
+   assert(format_desc->block.height == 1);
+
+   packed = lp_build_gather(builder, n, 32, 32, base_ptr, offset);
 
    (void)j;
 
diff --git a/src/gallium/drivers/llvmpipe/lp_test_format.c b/src/gallium/drivers/llvmpipe/lp_test_format.c
index e1277d800e6..2855d7cea4f 100644
--- a/src/gallium/drivers/llvmpipe/lp_test_format.c
+++ b/src/gallium/drivers/llvmpipe/lp_test_format.c
@@ -86,6 +86,7 @@ add_fetch_rgba_test(unsigned verbose,
    LLVMTypeRef args[4];
    LLVMValueRef func;
    LLVMValueRef packed_ptr;
+   LLVMValueRef offset = LLVMConstNull(LLVMInt32Type());
    LLVMValueRef rgba_ptr;
    LLVMValueRef i;
    LLVMValueRef j;
@@ -112,7 +113,8 @@ add_fetch_rgba_test(unsigned verbose,
    builder = LLVMCreateBuilder();
    LLVMPositionBuilderAtEnd(builder, block);
 
-   rgba = lp_build_fetch_rgba_aos(builder, desc, type, packed_ptr, i, j);
+   rgba = lp_build_fetch_rgba_aos(builder, desc, type,
+                                  packed_ptr, offset, i, j);
 
    LLVMBuildStore(builder, rgba, rgba_ptr);