gallivm: Fix format manipulation for big-endian

This patch fixes various format manipulation for big-endian
architectures.

Reviewed-by: Roland Scheidegger <[email protected]>
Reviewed-by: Jose Fonseca <[email protected]>

1 files changed, 39 insertions, 0 deletions

diff --git a/src/gallium/auxiliary/gallivm/lp_bld_format_yuv.c b/src/gallium/auxiliary/gallivm/lp_bld_format_yuv.c
index f77eb1212b1..e542abce0cc 100644
--- a/src/gallium/auxiliary/gallivm/lp_bld_format_yuv.c
+++ b/src/gallium/auxiliary/gallivm/lp_bld_format_yuv.c
@@ -72,9 +72,15 @@ uyvy_to_yuv_soa(struct gallivm_state *gallivm,
    assert(lp_check_value(type, i));
 
    /*
+    * Little endian:
     * y = (uyvy >> (16*i + 8)) & 0xff
     * u = (uyvy        ) & 0xff
     * v = (uyvy >> 16  ) & 0xff
+    *
+    * Big endian:
+    * y = (uyvy >> (-16*i + 16)) & 0xff
+    * u = (uyvy >> 24) & 0xff
+    * v = (uyvy >>  8) & 0xff
     */
 
 #if defined(PIPE_ARCH_X86) || defined(PIPE_ARCH_X86_64)
@@ -98,13 +104,23 @@ uyvy_to_yuv_soa(struct gallivm_state *gallivm,
 #endif
    {
       LLVMValueRef shift;
+#ifdef PIPE_ARCH_LITTLE_ENDIAN
       shift = LLVMBuildMul(builder, i, lp_build_const_int_vec(gallivm, type, 16), "");
       shift = LLVMBuildAdd(builder, shift, lp_build_const_int_vec(gallivm, type, 8), "");
+#else
+      shift = LLVMBuildMul(builder, i, lp_build_const_int_vec(gallivm, type, -16), "");
+      shift = LLVMBuildAdd(builder, shift, lp_build_const_int_vec(gallivm, type, 16), "");
+#endif
       *y = LLVMBuildLShr(builder, packed, shift, "");
    }
 
+#ifdef PIPE_ARCH_LITTLE_ENDIAN
    *u = packed;
    *v = LLVMBuildLShr(builder, packed, lp_build_const_int_vec(gallivm, type, 16), "");
+#else
+   *u = LLVMBuildLShr(builder, packed, lp_build_const_int_vec(gallivm, type, 24), "");
+   *v = LLVMBuildLShr(builder, packed, lp_build_const_int_vec(gallivm, type, 8), "");
+#endif
 
    mask = lp_build_const_int_vec(gallivm, type, 0xff);
 
@@ -140,9 +156,15 @@ yuyv_to_yuv_soa(struct gallivm_state *gallivm,
    assert(lp_check_value(type, i));
 
    /*
+   * Little endian:
     * y = (yuyv >> 16*i) & 0xff
     * u = (yuyv >> 8   ) & 0xff
     * v = (yuyv >> 24  ) & 0xff
+    *
+    * Big endian:
+    * y = (yuyv >> (-16*i + 24) & 0xff
+    * u = (yuyv >> 16)          & 0xff
+    * v = (yuyv)                & 0xff
     */
 
 #if defined(PIPE_ARCH_X86) || defined(PIPE_ARCH_X86_64)
@@ -165,12 +187,22 @@ yuyv_to_yuv_soa(struct gallivm_state *gallivm,
 #endif
    {
       LLVMValueRef shift;
+#ifdef PIPE_ARCH_LITTLE_ENDIAN
       shift = LLVMBuildMul(builder, i, lp_build_const_int_vec(gallivm, type, 16), "");
+#else
+      shift = LLVMBuildMul(builder, i, lp_build_const_int_vec(gallivm, type, -16), "");
+      shift = LLVMBuildAdd(builder, shift, lp_build_const_int_vec(gallivm, type, 24), "");
+#endif
       *y = LLVMBuildLShr(builder, packed, shift, "");
    }
 
+#ifdef PIPE_ARCH_LITTLE_ENDIAN
    *u = LLVMBuildLShr(builder, packed, lp_build_const_int_vec(gallivm, type, 8), "");
    *v = LLVMBuildLShr(builder, packed, lp_build_const_int_vec(gallivm, type, 24), "");
+#else
+   *u = LLVMBuildLShr(builder, packed, lp_build_const_int_vec(gallivm, type, 16), "");
+   *v = packed;
+#endif
 
    mask = lp_build_const_int_vec(gallivm, type, 0xff);
 
@@ -302,10 +334,17 @@ rgb_to_rgba_aos(struct gallivm_state *gallivm,
     * Make a 4 x unorm8 vector
     */
 
+#ifdef PIPE_ARCH_LITTLE_ENDIAN
    r = r;
    g = LLVMBuildShl(builder, g, lp_build_const_int_vec(gallivm, type, 8), "");
    b = LLVMBuildShl(builder, b, lp_build_const_int_vec(gallivm, type, 16), "");
    a = lp_build_const_int_vec(gallivm, type, 0xff000000);
+#else
+   r = LLVMBuildShl(builder, r, lp_build_const_int_vec(gallivm, type, 24), "");
+   g = LLVMBuildShl(builder, g, lp_build_const_int_vec(gallivm, type, 16), "");
+   b = LLVMBuildShl(builder, b, lp_build_const_int_vec(gallivm, type, 8), "");
+   a = lp_build_const_int_vec(gallivm, type, 0x000000ff);
+#endif
 
    rgba = r;
    rgba = LLVMBuildOr(builder, rgba, g, "");