llvmpipe: optimize tile writing code

The code which converts/copies color tiles to the linear layout has
been rewritten.  There's less arithmetic and better loop unrolling,
and possibly a better memory access pattern.
Some demos, like gears, are about 20% faster now.

1 files changed, 109 insertions, 18 deletions

diff --git a/src/gallium/drivers/llvmpipe/lp_tile_soa.py b/src/gallium/drivers/llvmpipe/lp_tile_soa.py
index a603b7f9f42..5d53689a3db 100644
--- a/src/gallium/drivers/llvmpipe/lp_tile_soa.py
+++ b/src/gallium/drivers/llvmpipe/lp_tile_soa.py
@@ -129,22 +129,8 @@ def generate_format_read(format, dst_type, dst_native_type, dst_suffix):
     print
     
 
-def generate_format_write(format, src_type, src_native_type, src_suffix):
-    '''Generate the function to write pixels to a particular format'''
-
-    name = short_name(format)
-
-    dst_native_type = native_type(format)
-
-    print 'static void'
-    print 'lp_tile_%s_write_%s(const %s *src, uint8_t *dst, unsigned dst_stride, unsigned x0, unsigned y0, unsigned w, unsigned h)' % (name, src_suffix, src_native_type)
-    print '{'
-    print '   unsigned x, y;'
-    print '   uint8_t *dst_row = dst + y0*dst_stride;'
-    print '   for (y = 0; y < h; ++y) {'
-    print '      %s *dst_pixel = (%s *)(dst_row + x0*%u);' % (dst_native_type, dst_native_type, format.stride())
-    print '      for (x = 0; x < w; ++x) {'
-
+def compute_inverse_swizzle(format):
+    '''Return an array[4] of inverse swizzle terms'''
     inv_swizzle = [None]*4
     if format.colorspace == 'rgb':
         for i in range(4):
@@ -155,8 +141,86 @@ def generate_format_write(format, src_type, src_native_type, src_suffix):
         swizzle = format.out_swizzle[0]
         if swizzle < 4:
             inv_swizzle[swizzle] = 0
-    else:
-        assert False
+    return inv_swizzle
+
+
+def pack_rgba(format, src_type, r, g, b, a):
+    """Return an expression for packing r, g, b, a into a pixel of the
+    given format.  Ex: '(b << 24) | (g << 16) | (r << 8) | (a << 0)'
+    """
+    assert format.colorspace == 'rgb'
+    inv_swizzle = compute_inverse_swizzle(format)
+    shift = 0
+    expr = None
+    for i in range(4):
+		# choose r, g, b, or a depending on the inverse swizzle term
+        if inv_swizzle[i] == 0:
+            value = r
+        elif inv_swizzle[i] == 1:
+            value = g
+        elif inv_swizzle[i] == 2:
+            value = b
+        elif inv_swizzle[i] == 3:
+            value = a
+        else:
+            value = None
+
+        if value:
+            dst_type = format.in_types[i]
+            dst_native_type = native_type(format)
+            value = conversion_expr(src_type, dst_type, dst_native_type, value)
+            term = "((%s) << %d)" % (value, shift)
+            if expr:
+                expr = expr + " | " + term
+            else:
+                expr = term
+
+        width = format.in_types[i].size
+        shift = shift + width
+    return expr
+
+
+def emit_unrolled_write_code(format, src_type):
+    '''Emit code for writing a block based on unrolled loops.
+    This is considerably faster than the TILE_PIXEL-based code below.
+    '''
+    dst_native_type = native_type(format)
+    print '   const unsigned dstpix_stride = dst_stride / %d;' % format.stride()
+    print '   %s *dstpix = (%s *) dst;' % (dst_native_type, dst_native_type)
+    print '   unsigned int qx, qy, i;'
+    print
+    print '   for (qy = 0; qy < h; qy += TILE_VECTOR_HEIGHT) {'
+    print '      const unsigned py = y0 + qy;'
+    print '      for (qx = 0; qx < w; qx += TILE_VECTOR_WIDTH) {'
+    print '         const unsigned px = x0 + qx;'
+    print '         const uint8_t *r = src + 0 * TILE_C_STRIDE;'
+    print '         const uint8_t *g = src + 1 * TILE_C_STRIDE;'
+    print '         const uint8_t *b = src + 2 * TILE_C_STRIDE;'
+    print '         const uint8_t *a = src + 3 * TILE_C_STRIDE;'
+    print '         (void) r; (void) g; (void) b; (void) a; /* silence warnings */'
+    print '         for (i = 0; i < TILE_C_STRIDE; i += 2) {'
+    print '            const uint32_t pixel0 = %s;' % pack_rgba(format, src_type, "r[i+0]", "g[i+0]", "b[i+0]", "a[i+0]")
+    print '            const uint32_t pixel1 = %s;' % pack_rgba(format, src_type, "r[i+1]", "g[i+1]", "b[i+1]", "a[i+1]")
+    print '            const unsigned offset = (py + tile_y_offset[i]) * dstpix_stride + (px + tile_x_offset[i]);'
+    print '            dstpix[offset + 0] = pixel0;'
+    print '            dstpix[offset + 1] = pixel1;'
+    print '         }'
+    print '         src += TILE_X_STRIDE;'
+    print '      }'
+    print '   }'
+
+
+def emit_tile_pixel_write_code(format, src_type):
+    '''Emit code for writing a block based on the TILE_PIXEL macro.'''
+    dst_native_type = native_type(format)
+
+    inv_swizzle = compute_inverse_swizzle(format)
+
+    print '   unsigned x, y;'
+    print '   uint8_t *dst_row = dst + y0*dst_stride;'
+    print '   for (y = 0; y < h; ++y) {'
+    print '      %s *dst_pixel = (%s *)(dst_row + x0*%u);' % (dst_native_type, dst_native_type, format.stride())
+    print '      for (x = 0; x < w; ++x) {'
 
     if format.layout == ARITH:
         print '         %s pixel = 0;' % dst_native_type
@@ -185,6 +249,20 @@ def generate_format_write(format, src_type, src_native_type, src_suffix):
     print '      }'
     print '      dst_row += dst_stride;'
     print '   }'
+
+
+def generate_format_write(format, src_type, src_native_type, src_suffix):
+    '''Generate the function to write pixels to a particular format'''
+
+    name = short_name(format)
+
+    print 'static void'
+    print 'lp_tile_%s_write_%s(const %s *src, uint8_t *dst, unsigned dst_stride, unsigned x0, unsigned y0, unsigned w, unsigned h)' % (name, src_suffix, src_native_type)
+    print '{'
+    if format.layout == ARITH and format.colorspace == 'rgb':
+        emit_unrolled_write_code(format, src_type)
+    else:
+        emit_tile_pixel_write_code(format, src_type)
     print '}'
     print
     
@@ -265,6 +343,19 @@ def main():
     print '   { 10, 11, 14, 15}'
     print '};'
     print
+    print '/* Note: these lookup tables could be replaced with some'
+    print ' * bit-twiddling code, but this is a little faster.'
+    print ' */'
+    print 'static unsigned tile_x_offset[TILE_VECTOR_WIDTH * TILE_VECTOR_HEIGHT] = {'
+    print '   0, 1, 0, 1, 2, 3, 2, 3,'
+    print '   0, 1, 0, 1, 2, 3, 2, 3'
+    print '};'
+    print
+    print 'static unsigned tile_y_offset[TILE_VECTOR_WIDTH * TILE_VECTOR_HEIGHT] = {'
+    print '   0, 0, 1, 1, 0, 0, 1, 1,'
+    print '   2, 2, 3, 3, 2, 2, 3, 3'
+    print '};'
+    print
 
     generate_clamp()