Import Thomas Helstrom's SSE memcpy code from the via X.org driver.

Add a TextureMemCpy callback, called from texstore.c when copying
texture data via the memcpy_texture() path.
Enable this code in the via driver - 100% speedup in texdown.c results.

4 files changed, 147 insertions, 1 deletions

diff --git a/src/mesa/drivers/dri/unichrome/Makefile b/src/mesa/drivers/dri/unichrome/Makefile
index 7875ba8f726..5fe00c1bd1f 100644
--- a/src/mesa/drivers/dri/unichrome/Makefile
+++ b/src/mesa/drivers/dri/unichrome/Makefile
@@ -12,6 +12,7 @@ DRIVER_SOURCES = \
 	via_fb.c \
 	via_tex.c \
 	via_ioctl.c \
+	via_memcpy.c \
 	via_render.c \
 	via_screen.c \
 	via_span.c \
diff --git a/src/mesa/drivers/dri/unichrome/via_memcpy.c b/src/mesa/drivers/dri/unichrome/via_memcpy.c
new file mode 100644
index 00000000000..351ba26ff2a
--- /dev/null
+++ b/src/mesa/drivers/dri/unichrome/via_memcpy.c
@@ -0,0 +1,138 @@
+/*
+ * Copyright (C) 2004 Thomas Hellstrom, All Rights Reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sub license,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the
+ * next paragraph) shall be included in all copies or substantial portions
+ * of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
+ * THE CODE SUPPLIER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+/* Thomas' orginal gutted for mesa by Keith Whitwell
+ */
+
+#include "via_tex.h"
+
+
+#define SSE_PREFETCH "  prefetchnta "
+#define FENCE __asm__ __volatile__ ("sfence":::"memory");
+
+
+#define PREFETCH1(arch_prefetch,from)			\
+    __asm__ __volatile__ (				\
+			  "1:  " arch_prefetch "(%0)\n"	\
+			  arch_prefetch "32(%0)\n"	\
+			  arch_prefetch "64(%0)\n"	\
+			  arch_prefetch "96(%0)\n"	\
+			  arch_prefetch "128(%0)\n"	\
+			  arch_prefetch "160(%0)\n"	\
+			  arch_prefetch "192(%0)\n"	\
+			  arch_prefetch "256(%0)\n"	\
+			  arch_prefetch "288(%0)\n"	\
+			  "2:\n"			\
+			  : : "r" (from) );
+
+
+
+#define small_memcpy(to,from,n)						\
+    {									\
+	__asm__ __volatile__(						\
+			     "movl %2,%%ecx\n\t"			\
+                             "sarl $2,%%ecx\n\t"			\
+			     "rep ; movsl\n\t"				\
+			     "testb $2,%b2\n\t"				\
+			     "je 1f\n\t"				\
+			     "movsw\n"					\
+			     "1:\ttestb $1,%b2\n\t"			\
+			     "je 2f\n\t"				\
+			     "movsb\n"					\
+			     "2:"					\
+			     :"=&D" (to), "=&S" (from)			\
+			     :"q" (n),"0" ((long) to),"1" ((long) from) \
+			     : "%ecx","memory");			\
+    }
+
+
+#define SSE_CPY(prefetch,from,to,dummy,lcnt)				\
+    if ((unsigned long) from & 15)			 {		\
+	__asm__ __volatile__ (						\
+			      "1:\n"					\
+                              prefetch "320(%1)\n"			\
+			      "  movups (%1), %%xmm0\n"			\
+			      "  movups 16(%1), %%xmm1\n"		\
+			      "  movntps %%xmm0, (%0)\n"		\
+			      "  movntps %%xmm1, 16(%0)\n"		\
+                              prefetch "352(%1)\n"			\
+			      "  movups 32(%1), %%xmm2\n"		\
+			      "  movups 48(%1), %%xmm3\n"		\
+			      "  movntps %%xmm2, 32(%0)\n"		\
+			      "  movntps %%xmm3, 48(%0)\n"		\
+			      "  addl $64,%0\n"				\
+			      "  addl $64,%1\n"				\
+			      "  decl %2\n"				\
+			      "  jne 1b\n"				\
+			      :"=&D"(to), "=&S"(from), "=&r"(dummy)	\
+			      :"0" (to), "1" (from), "2" (lcnt): "memory"); \
+    } else {								\
+	__asm__ __volatile__ (						\
+			      "2:\n"					\
+			      prefetch "320(%1)\n"			\
+			      "  movaps (%1), %%xmm0\n"			\
+			      "  movaps 16(%1), %%xmm1\n"		\
+			      "  movntps %%xmm0, (%0)\n"		\
+			      "  movntps %%xmm1, 16(%0)\n"		\
+                              prefetch "352(%1)\n"			\
+			      "  movaps 32(%1), %%xmm2\n"		\
+			      "  movaps 48(%1), %%xmm3\n"		\
+			      "  movntps %%xmm2, 32(%0)\n"		\
+			      "  movntps %%xmm3, 48(%0)\n"		\
+			      "  addl $64,%0\n"				\
+			      "  addl $64,%1\n"				\
+			      "  decl %2\n"				\
+			      "  jne 2b\n"				\
+			      :"=&D"(to), "=&S"(from), "=&r"(dummy)	\
+			      :"0" (to), "1" (from), "2" (lcnt): "memory"); \
+    }
+
+
+
+/*
+ */
+void via_sse_memcpy(void *to,
+		    const void *from,
+		    size_t sz)
+
+{
+   int dummy;
+   int lcnt = sz >> 6;
+   int rest = sz & 63;
+
+   PREFETCH1(SSE_PREFETCH,from);
+
+   if (lcnt > 5) {
+      lcnt -= 5;
+      SSE_CPY(SSE_PREFETCH,from,to,dummy,lcnt);
+      lcnt = 5;
+   }
+   if (lcnt) {
+      SSE_CPY("#",from,to,dummy,lcnt);
+   }
+   if (rest) small_memcpy(to, from, rest);
+   FENCE;
+}
+
+
+
diff --git a/src/mesa/drivers/dri/unichrome/via_tex.c b/src/mesa/drivers/dri/unichrome/via_tex.c
index 94cdf61accd..9672888f8ea 100644
--- a/src/mesa/drivers/dri/unichrome/via_tex.c
+++ b/src/mesa/drivers/dri/unichrome/via_tex.c
@@ -947,7 +947,12 @@ void viaInitTextureFuncs(struct dd_function_table * functions)
    functions->NewTextureImage = viaNewTextureImage;
    functions->DeleteTexture = _mesa_delete_texture_object;
    functions->FreeTexImageData = viaFreeTextureImageData;
-					    
+
+   if (getenv("VIA_NO_SSE"))
+      functions->TextureMemCpy = _mesa_memcpy;
+   else
+      functions->TextureMemCpy = via_sse_memcpy;
+
    functions->UpdateTexturePalette = 0;
    functions->IsTextureResident = viaIsTextureResident;
 }
diff --git a/src/mesa/drivers/dri/unichrome/via_tex.h b/src/mesa/drivers/dri/unichrome/via_tex.h
index f6c024e4381..8277aada407 100644
--- a/src/mesa/drivers/dri/unichrome/via_tex.h
+++ b/src/mesa/drivers/dri/unichrome/via_tex.h
@@ -34,4 +34,6 @@ GLboolean viaUpdateTextureState(GLcontext *ctx);
 void viaInitTextureFuncs(struct dd_function_table * functions);
 GLboolean viaSwapOutWork( struct via_context *vmesa );
 
+void via_sse_memcpy( void *to, const void *from, size_t sz );
+
 #endif