Split out runtime assembly functions.

1 files changed, 126 insertions, 723 deletions

diff --git a/src/mesa/tnl/t_vertex_sse.c b/src/mesa/tnl/t_vertex_sse.c
index d4eefdb6fc9..733d462901b 100644
--- a/src/mesa/tnl/t_vertex_sse.c
+++ b/src/mesa/tnl/t_vertex_sse.c
@@ -33,619 +33,41 @@
 #include "simple_list.h"
 #include "enums.h"
 
-#if defined(USE_X86_ASM)
+#if defined(USE_SSE_ASM)
+
+#include "x86/rtasm/x86sse.h"
+#include "x86/common_x86_asm.h"
+
 
 #define X    0
 #define Y    1
 #define Z    2
 #define W    3
 
-#define DISASSEM 0
-
-struct x86_reg {
-   GLuint file:3;
-   GLuint idx:3;
-   GLuint mod:2;		/* mod_REG if this is just a register */
-   GLint  disp:24;		/* only +/- 23bits of offset - should be enough... */
-};
 
 struct x86_program {
-   GLcontext *ctx;
-   
-   GLubyte *store;
-   GLubyte *csr;
-
-   GLuint stack_offset;
+   struct x86_function func;
 
+   GLcontext *ctx;
    GLboolean inputs_safe;
    GLboolean outputs_safe;
    GLboolean have_sse2;
-   GLboolean need_emms;
    
    struct x86_reg identity;
    struct x86_reg chan0;
-
-};
-
-
-#define X86_TWOB 0x0f
-
-/* There are more but these are all we'll use:
- */
-enum x86_reg_file {
-   file_REG32,
-   file_MMX,
-   file_XMM
 };
 
-/* Values for mod field of modr/m byte
- */
-enum x86_reg_mod {
-   mod_INDIRECT,
-   mod_DISP8,
-   mod_DISP32,
-   mod_REG
-};
-
-enum x86_reg_name {
-   reg_AX,
-   reg_CX,
-   reg_DX,
-   reg_BX,
-   reg_SP,
-   reg_BP,
-   reg_SI,
-   reg_DI
-};
-
-
-enum x86_cc {
-   cc_O,			/* overflow */
-   cc_NO,			/* not overflow */
-   cc_NAE,			/* not above or equal / carry */
-   cc_AE,			/* above or equal / not carry */
-   cc_E,			/* equal / zero */
-   cc_NE			/* not equal / not zero */
-};
-
-#define cc_Z  cc_E
-#define cc_NZ cc_NE
-
-
-/* Create and manipulate registers and regmem values:
- */
-static struct x86_reg make_reg( GLuint file,
-				GLuint idx )
-{
-   struct x86_reg reg;
-
-   reg.file = file;
-   reg.idx = idx;
-   reg.mod = mod_REG;
-   reg.disp = 0;
-
-   return reg;
-}
-
-static struct x86_reg make_disp( struct x86_reg reg,
-				 GLint disp )
-{
-   assert(reg.file == file_REG32);
-
-   if (reg.mod == mod_REG)
-      reg.disp = disp;
-   else
-      reg.disp += disp;
-
-   if (reg.disp == 0)
-      reg.mod = mod_INDIRECT;
-   else if (reg.disp <= 127 && reg.disp >= -128)
-      reg.mod = mod_DISP8;
-   else
-      reg.mod = mod_DISP32;
-
-   return reg;
-}
-
-static struct x86_reg deref( struct x86_reg reg )
-{
-   return make_disp(reg, 0);
-}
-
-static struct x86_reg get_base_reg( struct x86_reg reg )
-{
-   return make_reg( reg.file, reg.idx );
-}
-
-
-/* Retreive a reference to one of the function arguments, taking into
- * account any push/pop activity:
- */
-static struct x86_reg make_fn_arg( struct x86_program *p,
-				   GLuint arg )
-{
-   return make_disp(make_reg(file_REG32, reg_SP), 
-		    p->stack_offset + arg * 4);	/* ??? */
-}
 
 static struct x86_reg get_identity( struct x86_program *p )
 {
    return p->identity;
 }
 
-
-/* Emit bytes to the instruction stream:
- */
-static void emit_1b( struct x86_program *p, GLbyte b0 )
-{
-   *(GLbyte *)(p->csr++) = b0;
-}
-
-static void emit_1i( struct x86_program *p, GLint i0 )
-{
-   *(GLint *)(p->csr) = i0;
-   p->csr += 4;
-}
-
-static void disassem( struct x86_program *p, const char *fn )
-{
-#if DISASSEM
-   static const char *last_fn;
-   if (fn && fn != last_fn) {
-      _mesa_printf("0x%x: %s\n", p->csr, fn);
-      last_fn = fn;
-   }
-#endif
-}
-
-static void emit_1ub_fn( struct x86_program *p, GLubyte b0, const char *fn )
-{
-   disassem(p, fn);
-   *(p->csr++) = b0;
-}
-
-static void emit_2ub_fn( struct x86_program *p, GLubyte b0, GLubyte b1, const char *fn )
-{
-   disassem(p, fn);
-   *(p->csr++) = b0;
-   *(p->csr++) = b1;
-}
-
-static void emit_3ub_fn( struct x86_program *p, GLubyte b0, GLubyte b1, GLubyte b2, const char *fn )
-{
-   disassem(p, fn);
-   *(p->csr++) = b0;
-   *(p->csr++) = b1;
-   *(p->csr++) = b2;
-}
-
-#define emit_1ub(p, b0)         emit_1ub_fn(p, b0, __FUNCTION__)
-#define emit_2ub(p, b0, b1)     emit_2ub_fn(p, b0, b1, __FUNCTION__)
-#define emit_3ub(p, b0, b1, b2) emit_3ub_fn(p, b0, b1, b2, __FUNCTION__)
-
-
-/* Labels, jumps and fixup:
- */
-static GLubyte *get_label( struct x86_program *p )
-{
-   return p->csr;
-}
-
-static void x86_jcc( struct x86_program *p,
-		      GLuint cc,
-		      GLubyte *label )
-{
-   GLint offset = label - (get_label(p) + 2);
-   
-   if (offset <= 127 && offset >= -128) {
-      emit_1ub(p, 0x70 + cc);
-      emit_1b(p, (GLbyte) offset);
-   }
-   else {
-      offset = label - (get_label(p) + 6);
-      emit_2ub(p, 0x0f, 0x80 + cc);
-      emit_1i(p, offset);
-   }
-}
-
-/* Always use a 32bit offset for forward jumps:
- */
-static GLubyte *x86_jcc_forward( struct x86_program *p,
-				 GLuint cc )
-{
-   emit_2ub(p, 0x0f, 0x80 + cc);
-   emit_1i(p, 0);
-   return get_label(p);
-}
-
-/* Fixup offset from forward jump:
- */
-static void do_fixup( struct x86_program *p,
-		      GLubyte *fixup )
-{
-   *(int *)(fixup - 4) = get_label(p) - fixup;
-}
-
-static void x86_push( struct x86_program *p,
-		       struct x86_reg reg )
-{
-   assert(reg.mod == mod_REG);
-   emit_1ub(p, 0x50 + reg.idx);
-   p->stack_offset += 4;
-}
-
-static void x86_pop( struct x86_program *p,
-		       struct x86_reg reg )
-{
-   assert(reg.mod == mod_REG);
-   emit_1ub(p, 0x58 + reg.idx);
-   p->stack_offset -= 4;
-}
-
-static void x86_inc( struct x86_program *p,
-		       struct x86_reg reg )
-{
-   assert(reg.mod == mod_REG);
-   emit_1ub(p, 0x40 + reg.idx);
-}
-
-static void x86_dec( struct x86_program *p,
-		       struct x86_reg reg )
-{
-   assert(reg.mod == mod_REG);
-   emit_1ub(p, 0x48 + reg.idx);
-}
-
-static void x86_ret( struct x86_program *p )
-{
-   emit_1ub(p, 0xc3);
-}
-
-static void mmx_emms( struct x86_program *p )
-{
-   assert(p->need_emms);
-   emit_2ub(p, 0x0f, 0x77);
-   p->need_emms = 0;
-}
-
-
-
-
-/* Build a modRM byte + possible displacement.  No treatment of SIB
- * indexing.  BZZT - no way to encode an absolute address.
- */
-static void emit_modrm( struct x86_program *p, 
-			struct x86_reg reg, 
-			struct x86_reg regmem )
-{
-   GLubyte val = 0;
-   
-   assert(reg.mod == mod_REG);
-   
-   val |= regmem.mod << 6;     	/* mod field */
-   val |= reg.idx << 3;		/* reg field */
-   val |= regmem.idx;		/* r/m field */
-   
-   emit_1ub_fn(p, val, 0);
-
-   /* Oh-oh we've stumbled into the SIB thing.
-    */
-   if (regmem.idx == reg_SP) {
-      emit_1ub_fn(p, 0x24, 0);		/* simplistic! */
-   }
-
-   switch (regmem.mod) {
-   case mod_REG:
-   case mod_INDIRECT:
-      break;
-   case mod_DISP8:
-      emit_1b(p, regmem.disp);
-      break;
-   case mod_DISP32:
-      emit_1i(p, regmem.disp);
-      break;
-   default:
-      _mesa_printf("unknown regmem.mod %d\n", regmem.mod);
-      abort();
-      break;
-   }
-}
-
-/* Many x86 instructions have two opcodes to cope with the situations
- * where the destination is a register or memory reference
- * respectively.  This function selects the correct opcode based on
- * the arguments presented.
- */
-static void emit_op_modrm( struct x86_program *p,
-			   GLubyte op_dst_is_reg, 
-			   GLubyte op_dst_is_mem,
-			   struct x86_reg dst,
-			   struct x86_reg src )
-{
-   switch (dst.mod) {
-   case mod_REG:
-      emit_1ub_fn(p, op_dst_is_reg, 0);
-      emit_modrm(p, dst, src);
-      break;
-   case mod_INDIRECT:
-   case mod_DISP32:
-   case mod_DISP8:
-      assert(src.mod == mod_REG);
-      emit_1ub_fn(p, op_dst_is_mem, 0);
-      emit_modrm(p, src, dst);
-      break;
-   default:
-      _mesa_printf("unknown dst.mod %d\n", dst.mod);
-      abort();
-      break;
-   }
-}
-
-static void x86_mov( struct x86_program *p,
-		      struct x86_reg dst,
-		      struct x86_reg src )
-{
-   emit_op_modrm( p, 0x8b, 0x89, dst, src );
-}
-
-static void x86_xor( struct x86_program *p,
-		      struct x86_reg dst,
-		      struct x86_reg src )
-{
-   emit_op_modrm( p, 0x33, 0x31, dst, src );
-}
-
-static void x86_cmp( struct x86_program *p,
-		      struct x86_reg dst,
-		      struct x86_reg src )
-{
-   emit_op_modrm( p, 0x3b, 0x39, dst, src );
-}
-
-static void sse2_movd( struct x86_program *p,
-		       struct x86_reg dst,
-		       struct x86_reg src )
-{
-   assert(p->have_sse2);
-   emit_2ub(p, 0x66, X86_TWOB);
-   emit_op_modrm( p, 0x6e, 0x7e, dst, src );
-}
-
-static void mmx_movd( struct x86_program *p,
-		       struct x86_reg dst,
-		       struct x86_reg src )
-{
-   p->need_emms = 1;
-   emit_1ub(p, X86_TWOB);
-   emit_op_modrm( p, 0x6e, 0x7e, dst, src );
-}
-
-static void mmx_movq( struct x86_program *p,
-		       struct x86_reg dst,
-		       struct x86_reg src )
-{
-   p->need_emms = 1;
-   emit_1ub(p, X86_TWOB);
-   emit_op_modrm( p, 0x6f, 0x7f, dst, src );
-}
-
-
-static void sse_movss( struct x86_program *p,
-		       struct x86_reg dst,
-		       struct x86_reg src )
-{
-   emit_2ub(p, 0xF3, X86_TWOB);
-   emit_op_modrm( p, 0x10, 0x11, dst, src );
-}
-
-static void sse_movaps( struct x86_program *p,
-			 struct x86_reg dst,
-			 struct x86_reg src )
-{
-   emit_1ub(p, X86_TWOB);
-   emit_op_modrm( p, 0x28, 0x29, dst, src );
-}
-
-static void sse_movups( struct x86_program *p,
-			 struct x86_reg dst,
-			 struct x86_reg src )
-{
-   emit_1ub(p, X86_TWOB);
-   emit_op_modrm( p, 0x10, 0x11, dst, src );
-}
-
-static void sse_movhps( struct x86_program *p,
-			struct x86_reg dst,
-			struct x86_reg src )
-{
-   assert(dst.mod != mod_REG || src.mod != mod_REG);
-   emit_1ub(p, X86_TWOB);
-   emit_op_modrm( p, 0x16, 0x17, dst, src ); /* cf movlhps */
-}
-
-static void sse_movlps( struct x86_program *p,
-			struct x86_reg dst,
-			struct x86_reg src )
-{
-   assert(dst.mod != mod_REG || src.mod != mod_REG);
-   emit_1ub(p, X86_TWOB);
-   emit_op_modrm( p, 0x12, 0x13, dst, src ); /* cf movhlps */
-}
-
-/* SSE operations often only have one format, with dest constrained to
- * be a register:
- */
-static void sse_mulps( struct x86_program *p,
-			struct x86_reg dst,
-			struct x86_reg src )
-{
-   emit_2ub(p, X86_TWOB, 0x59);
-   emit_modrm( p, dst, src );
-}
-
-static void sse_addps( struct x86_program *p,
-			struct x86_reg dst,
-			struct x86_reg src )
-{
-   emit_2ub(p, X86_TWOB, 0x58);
-   emit_modrm( p, dst, src );
-}
-
-static void sse_movhlps( struct x86_program *p,
-			 struct x86_reg dst,
-			 struct x86_reg src )
-{
-   assert(dst.mod == mod_REG && src.mod == mod_REG);
-   emit_2ub(p, X86_TWOB, 0x12);
-   emit_modrm( p, dst, src );
-}
-
-static void sse_movlhps( struct x86_program *p,
-			 struct x86_reg dst,
-			 struct x86_reg src )
-{
-   assert(dst.mod == mod_REG && src.mod == mod_REG);
-   emit_2ub(p, X86_TWOB, 0x16);
-   emit_modrm( p, dst, src );
-}
-
-static void sse2_cvtps2dq( struct x86_program *p,
-			struct x86_reg dst,
-			struct x86_reg src )
-{
-   assert(p->have_sse2);
-   emit_3ub(p, 0x66, X86_TWOB, 0x5B);
-   emit_modrm( p, dst, src );
-}
-
-static void sse2_packssdw( struct x86_program *p,
-			struct x86_reg dst,
-			struct x86_reg src )
-{
-   assert(p->have_sse2);
-   emit_3ub(p, 0x66, X86_TWOB, 0x6B);
-   emit_modrm( p, dst, src );
-}
-
-static void sse2_packsswb( struct x86_program *p,
-			struct x86_reg dst,
-			struct x86_reg src )
-{
-   assert(p->have_sse2);
-   emit_3ub(p, 0x66, X86_TWOB, 0x63);
-   emit_modrm( p, dst, src );
-}
-
-static void sse2_packuswb( struct x86_program *p,
-			   struct x86_reg dst,
-			   struct x86_reg src )
-{
-   assert(p->have_sse2);
-   emit_3ub(p, 0x66, X86_TWOB, 0x67);
-   emit_modrm( p, dst, src );
-}
-
-static void sse_cvtps2pi( struct x86_program *p,
-			  struct x86_reg dst,
-			  struct x86_reg src )
-{
-   assert(dst.file == file_MMX && 
-	  (src.file == file_XMM || src.mod != mod_REG));
-
-   p->need_emms = 1;
-
-   emit_2ub(p, X86_TWOB, 0x2d);
-   emit_modrm( p, dst, src );
-}
-
-static void mmx_packssdw( struct x86_program *p,
-			  struct x86_reg dst,
-			  struct x86_reg src )
-{
-   assert(dst.file == file_MMX && 
-	  (src.file == file_MMX || src.mod != mod_REG));
-
-   p->need_emms = 1;
-
-   emit_2ub(p, X86_TWOB, 0x6b);
-   emit_modrm( p, dst, src );
-}
-
-static void mmx_packuswb( struct x86_program *p,
-			  struct x86_reg dst,
-			  struct x86_reg src )
-{
-   assert(dst.file == file_MMX && 
-	  (src.file == file_MMX || src.mod != mod_REG));
-
-   p->need_emms = 1;
-
-   emit_2ub(p, X86_TWOB, 0x67);
-   emit_modrm( p, dst, src );
-}
-
-
-/* Load effective address:
- */
-static void x86_lea( struct x86_program *p,
-		     struct x86_reg dst,
-		     struct x86_reg src )
-{
-   emit_1ub(p, 0x8d);
-   emit_modrm( p, dst, src );
-}
-
-static void x86_test( struct x86_program *p,
-		      struct x86_reg dst,
-		      struct x86_reg src )
-{
-   emit_1ub(p, 0x85);
-   emit_modrm( p, dst, src );
-}
-
-
-
-
-/**
- * Perform a reduced swizzle:
- */
-static void sse2_pshufd( struct x86_program *p,
-			 struct x86_reg dest,
-			 struct x86_reg arg0,
-			 GLubyte x,
-			 GLubyte y,
-			 GLubyte z,
-			 GLubyte w) 
-{
-   assert(p->have_sse2);
-   emit_3ub(p, 0x66, X86_TWOB, 0x70);
-   emit_modrm(p, dest, arg0);
-   emit_1ub(p, (x|(y<<2)|(z<<4)|w<<6)); 
-}
-
-
-/* Shufps can also be used to implement a reduced swizzle when dest ==
- * arg0.
- */
-static void sse_shufps( struct x86_program *p,
-			 struct x86_reg dest,
-			 struct x86_reg arg0,
-			 GLubyte x,
-			 GLubyte y,
-			 GLubyte z,
-			 GLubyte w) 
-{
-   emit_2ub(p, X86_TWOB, 0xC6);
-   emit_modrm(p, dest, arg0);
-   emit_1ub(p, (x|(y<<2)|(z<<4)|w<<6)); 
-}
-
-
 static void emit_load4f_4( struct x86_program *p, 			   
 			   struct x86_reg dest,
 			   struct x86_reg arg0 )
 {
-   sse_movups(p, dest, arg0);
+   sse_movups(&p->func, dest, arg0);
 }
 
 static void emit_load4f_3( struct x86_program *p, 
@@ -659,10 +81,10 @@ static void emit_load4f_3( struct x86_program *p,
     * 0 0 c 1
     * a b c 1
     */
-   sse_movss(p, dest, make_disp(arg0, 8));
-   sse_shufps(p, dest, get_identity(p), X,Y,Z,W );
-   sse_shufps(p, dest, dest, Y,Z,X,W );
-   sse_movlps(p, dest, arg0);
+   sse_movss(&p->func, dest, x86_make_disp(arg0, 8));
+   sse_shufps(&p->func, dest, get_identity(p), X,Y,Z,W );
+   sse_shufps(&p->func, dest, dest, Y,Z,X,W );
+   sse_movlps(&p->func, dest, arg0);
 }
 
 static void emit_load4f_2( struct x86_program *p, 
@@ -671,8 +93,8 @@ static void emit_load4f_2( struct x86_program *p,
 {
    /* Initialize from identity, then pull in low two words:
     */
-   sse_movups(p, dest, get_identity(p));
-   sse_movlps(p, dest, arg0);
+   sse_movups(&p->func, dest, get_identity(p));
+   sse_movlps(&p->func, dest, arg0);
 }
 
 static void emit_load4f_1( struct x86_program *p, 
@@ -680,8 +102,8 @@ static void emit_load4f_1( struct x86_program *p,
 			   struct x86_reg arg0 )
 {
    /* Pull in low word, then swizzle in identity */
-   sse_movss(p, dest, arg0);
-   sse_shufps(p, dest, get_identity(p), X,Y,Z,W );
+   sse_movss(&p->func, dest, arg0);
+   sse_shufps(&p->func, dest, get_identity(p), X,Y,Z,W );
 }
 
 
@@ -694,16 +116,16 @@ static void emit_load3f_3( struct x86_program *p,
     * array.
     */
    if (p->inputs_safe) {
-      sse_movups(p, dest, arg0);
+      sse_movups(&p->func, dest, arg0);
    } 
    else {
       /* c 0 0 0
        * c c c c
        * a b c c 
        */
-      sse_movss(p, dest, make_disp(arg0, 8));
-      sse_shufps(p, dest, dest, X,X,X,X);
-      sse_movlps(p, dest, arg0);
+      sse_movss(&p->func, dest, x86_make_disp(arg0, 8));
+      sse_shufps(&p->func, dest, dest, X,X,X,X);
+      sse_movlps(&p->func, dest, arg0);
    }
 }
 
@@ -725,7 +147,7 @@ static void emit_load2f_2( struct x86_program *p,
 			   struct x86_reg dest,
 			   struct x86_reg arg0 )
 {
-   sse_movlps(p, dest, arg0);
+   sse_movlps(&p->func, dest, arg0);
 }
 
 static void emit_load2f_1( struct x86_program *p, 
@@ -739,7 +161,7 @@ static void emit_load1f_1( struct x86_program *p,
 			   struct x86_reg dest,
 			   struct x86_reg arg0 )
 {
-   sse_movss(p, dest, arg0);
+   sse_movss(&p->func, dest, arg0);
 }
 
 static void (*load[4][4])( struct x86_program *p, 
@@ -772,9 +194,6 @@ static void emit_load( struct x86_program *p,
 		       struct x86_reg src,
 		       GLuint src_sz)
 {
-   if (DISASSEM)
-      _mesa_printf("load %d/%d\n", sz, src_sz);
-
    load[sz-1][src_sz-1](p, dest, src);
 }
 
@@ -782,7 +201,7 @@ static void emit_store4f( struct x86_program *p,
 			  struct x86_reg dest,
 			  struct x86_reg arg0 )
 {
-   sse_movups(p, dest, arg0);
+   sse_movups(&p->func, dest, arg0);
 }
 
 static void emit_store3f( struct x86_program *p, 
@@ -793,14 +212,14 @@ static void emit_store3f( struct x86_program *p,
       /* Emit the extra dword anyway.  This may hurt writecombining,
        * may cause other problems.
        */
-      sse_movups(p, dest, arg0);
+      sse_movups(&p->func, dest, arg0);
    }
    else {
       /* Alternate strategy - emit two, shuffle, emit one.
        */
-      sse_movlps(p, dest, arg0);
-      sse_shufps(p, arg0, arg0, Z, Z, Z, Z ); /* NOTE! destructive */
-      sse_movss(p, make_disp(dest,8), arg0);
+      sse_movlps(&p->func, dest, arg0);
+      sse_shufps(&p->func, arg0, arg0, Z, Z, Z, Z ); /* NOTE! destructive */
+      sse_movss(&p->func, x86_make_disp(dest,8), arg0);
    }
 }
 
@@ -808,14 +227,14 @@ static void emit_store2f( struct x86_program *p,
 			   struct x86_reg dest,
 			   struct x86_reg arg0 )
 {
-   sse_movlps(p, dest, arg0);
+   sse_movlps(&p->func, dest, arg0);
 }
 
 static void emit_store1f( struct x86_program *p, 
 			  struct x86_reg dest,
 			  struct x86_reg arg0 )
 {
-   sse_movss(p, dest, arg0);
+   sse_movss(&p->func, dest, arg0);
 }
 
 
@@ -835,8 +254,6 @@ static void emit_store( struct x86_program *p,
 			struct x86_reg temp )
 
 {
-   if (DISASSEM)
-      _mesa_printf("store %d\n", sz);
    store[sz-1](p, dest, temp);
 }
 
@@ -846,23 +263,23 @@ static void emit_pack_store_4ub( struct x86_program *p,
 {
    /* Scale by 255.0
     */
-   sse_mulps(p, temp, p->chan0);
+   sse_mulps(&p->func, temp, p->chan0);
 
    if (p->have_sse2) {
-      sse2_cvtps2dq(p, temp, temp);
-      sse2_packssdw(p, temp, temp);
-      sse2_packuswb(p, temp, temp);
-      sse_movss(p, dest, temp);
+      sse2_cvtps2dq(&p->func, temp, temp);
+      sse2_packssdw(&p->func, temp, temp);
+      sse2_packuswb(&p->func, temp, temp);
+      sse_movss(&p->func, dest, temp);
    }
    else {
-      struct x86_reg mmx0 = make_reg(file_MMX, 0);
-      struct x86_reg mmx1 = make_reg(file_MMX, 1);
-      sse_cvtps2pi(p, mmx0, temp);
-      sse_movhlps(p, temp, temp);
-      sse_cvtps2pi(p, mmx1, temp);
-      mmx_packssdw(p, mmx0, mmx1);
-      mmx_packuswb(p, mmx0, mmx0);
-      mmx_movd(p, dest, mmx0);
+      struct x86_reg mmx0 = x86_make_reg(file_MMX, 0);
+      struct x86_reg mmx1 = x86_make_reg(file_MMX, 1);
+      sse_cvtps2pi(&p->func, mmx0, temp);
+      sse_movhlps(&p->func, temp, temp);
+      sse_cvtps2pi(&p->func, mmx1, temp);
+      mmx_packssdw(&p->func, mmx0, mmx1);
+      mmx_packuswb(&p->func, mmx0, mmx0);
+      mmx_movd(&p->func, dest, mmx0);
    }
 }
 
@@ -881,11 +298,11 @@ static void get_src_ptr( struct x86_program *p,
 			 struct tnl_clipspace_attr *a )
 {
    struct tnl_clipspace *vtx = GET_VERTEX_STATE(p->ctx);
-   struct x86_reg ptr_to_src = make_disp(vtxREG, get_offset(vtx, &a->inputptr));
+   struct x86_reg ptr_to_src = x86_make_disp(vtxREG, get_offset(vtx, &a->inputptr));
 
    /* Load current a[j].inputptr
     */
-   x86_mov(p, srcREG, ptr_to_src);
+   x86_mov(&p->func, srcREG, ptr_to_src);
 }
 
 static void update_src_ptr( struct x86_program *p,
@@ -895,16 +312,16 @@ static void update_src_ptr( struct x86_program *p,
 {
    if (a->inputstride) {
       struct tnl_clipspace *vtx = GET_VERTEX_STATE(p->ctx);
-      struct x86_reg ptr_to_src = make_disp(vtxREG, get_offset(vtx, &a->inputptr));
+      struct x86_reg ptr_to_src = x86_make_disp(vtxREG, get_offset(vtx, &a->inputptr));
 
       /* add a[j].inputstride (hardcoded value - could just as easily
        * pull the stride value from memory each time).
        */
-      x86_lea(p, srcREG, make_disp(srcREG, a->inputstride));
+      x86_lea(&p->func, srcREG, x86_make_disp(srcREG, a->inputstride));
       
       /* save new value of a[j].inputptr 
        */
-      x86_mov(p, ptr_to_src, srcREG);
+      x86_mov(&p->func, ptr_to_src, srcREG);
    }
 }
 
@@ -922,56 +339,55 @@ static GLboolean build_vertex_emit( struct x86_program *p )
    struct tnl_clipspace *vtx = GET_VERTEX_STATE(ctx);
    GLuint j = 0;
 
-   struct x86_reg vertexEAX = make_reg(file_REG32, reg_AX);
-   struct x86_reg srcECX = make_reg(file_REG32, reg_CX);
-   struct x86_reg countEBP = make_reg(file_REG32, reg_BP);
-   struct x86_reg vtxESI = make_reg(file_REG32, reg_SI);
-   struct x86_reg temp = make_reg(file_XMM, 0);
-   struct x86_reg vp0 = make_reg(file_XMM, 1);
-   struct x86_reg vp1 = make_reg(file_XMM, 2);
+   struct x86_reg vertexEAX = x86_make_reg(file_REG32, reg_AX);
+   struct x86_reg srcECX = x86_make_reg(file_REG32, reg_CX);
+   struct x86_reg countEBP = x86_make_reg(file_REG32, reg_BP);
+   struct x86_reg vtxESI = x86_make_reg(file_REG32, reg_SI);
+   struct x86_reg temp = x86_make_reg(file_XMM, 0);
+   struct x86_reg vp0 = x86_make_reg(file_XMM, 1);
+   struct x86_reg vp1 = x86_make_reg(file_XMM, 2);
    GLubyte *fixup, *label;
 
-   p->csr = p->store;
+   x86_init_func(&p->func);
    
    /* Push a few regs?
     */
-/*    x86_push(p, srcECX); */
-   x86_push(p, countEBP);
-   x86_push(p, vtxESI);
+   x86_push(&p->func, countEBP);
+   x86_push(&p->func, vtxESI);
 
 
    /* Get vertex count, compare to zero
     */
-   x86_xor(p, srcECX, srcECX);
-   x86_mov(p, countEBP, make_fn_arg(p, 2));
-   x86_cmp(p, countEBP, srcECX);
-   fixup = x86_jcc_forward(p, cc_E);
+   x86_xor(&p->func, srcECX, srcECX);
+   x86_mov(&p->func, countEBP, x86_fn_arg(&p->func, 2));
+   x86_cmp(&p->func, countEBP, srcECX);
+   fixup = x86_jcc_forward(&p->func, cc_E);
 
    /* Initialize destination register. 
     */
-   x86_mov(p, vertexEAX, make_fn_arg(p, 3));
+   x86_mov(&p->func, vertexEAX, x86_fn_arg(&p->func, 3));
 
    /* Dereference ctx to get tnl, then vtx:
     */
-   x86_mov(p, vtxESI, make_fn_arg(p, 1));
-   x86_mov(p, vtxESI, make_disp(vtxESI, get_offset(ctx, &ctx->swtnl_context)));
-   vtxESI = make_disp(vtxESI, get_offset(tnl, &tnl->clipspace));
+   x86_mov(&p->func, vtxESI, x86_fn_arg(&p->func, 1));
+   x86_mov(&p->func, vtxESI, x86_make_disp(vtxESI, get_offset(ctx, &ctx->swtnl_context)));
+   vtxESI = x86_make_disp(vtxESI, get_offset(tnl, &tnl->clipspace));
 
    
    /* Possibly load vp0, vp1 for viewport calcs:
     */
    if (vtx->need_viewport) {
-      sse_movups(p, vp0, make_disp(vtxESI, get_offset(vtx, &vtx->vp_scale[0])));
-      sse_movups(p, vp1, make_disp(vtxESI, get_offset(vtx, &vtx->vp_xlate[0])));
+      sse_movups(&p->func, vp0, x86_make_disp(vtxESI, get_offset(vtx, &vtx->vp_scale[0])));
+      sse_movups(&p->func, vp1, x86_make_disp(vtxESI, get_offset(vtx, &vtx->vp_xlate[0])));
    }
 
    /* always load, needed or not:
     */
-   sse_movups(p, p->chan0, make_disp(vtxESI, get_offset(vtx, &vtx->chan_scale[0])));
-   sse_movups(p, p->identity, make_disp(vtxESI, get_offset(vtx, &vtx->identity[0])));
+   sse_movups(&p->func, p->chan0, x86_make_disp(vtxESI, get_offset(vtx, &vtx->chan_scale[0])));
+   sse_movups(&p->func, p->identity, x86_make_disp(vtxESI, get_offset(vtx, &vtx->identity[0])));
 
    /* Note address for loop jump */
-   label = get_label(p);
+   label = x86_get_label(&p->func);
 
    /* Emit code for each of the attributes.  Currently routes
     * everything through SSE registers, even when it might be more
@@ -981,7 +397,7 @@ static GLboolean build_vertex_emit( struct x86_program *p )
     */
    while (j < vtx->attr_count) {
       struct tnl_clipspace_attr *a = &vtx->attr[j];
-      struct x86_reg dest = make_disp(vertexEAX, a->vertoffset);
+      struct x86_reg dest = x86_make_disp(vertexEAX, a->vertoffset);
 
       /* Now, load an XMM reg from src, perhaps transform, then save.
        * Could be shortcircuited in specific cases:
@@ -989,13 +405,13 @@ static GLboolean build_vertex_emit( struct x86_program *p )
       switch (a->format) {
       case EMIT_1F:
 	 get_src_ptr(p, srcECX, vtxESI, a);
-	 emit_load(p, temp, 1, deref(srcECX), a->inputsize);
+	 emit_load(p, temp, 1, x86_deref(srcECX), a->inputsize);
 	 emit_store(p, dest, 1, temp);
 	 update_src_ptr(p, srcECX, vtxESI, a);
 	 break;
       case EMIT_2F:
 	 get_src_ptr(p, srcECX, vtxESI, a);
-	 emit_load(p, temp, 2, deref(srcECX), a->inputsize);
+	 emit_load(p, temp, 2, x86_deref(srcECX), a->inputsize);
 	 emit_store(p, dest, 2, temp);
 	 update_src_ptr(p, srcECX, vtxESI, a);
 	 break;
@@ -1004,58 +420,58 @@ static GLboolean build_vertex_emit( struct x86_program *p )
 	  */
 	 if (0) {
 	    get_src_ptr(p, srcECX, vtxESI, a);
-	    emit_load(p, temp, 3, deref(srcECX), a->inputsize);
+	    emit_load(p, temp, 3, x86_deref(srcECX), a->inputsize);
 	    emit_store(p, dest, 3, temp);
 	    update_src_ptr(p, srcECX, vtxESI, a);
 	 }
 	 else {
 	    get_src_ptr(p, srcECX, vtxESI, a);
-	    emit_load(p, temp, 2, deref(srcECX), a->inputsize);
+	    emit_load(p, temp, 2, x86_deref(srcECX), a->inputsize);
 	    emit_store(p, dest, 2, temp);
 	    if (a->inputsize > 2) {
-	       emit_load(p, temp, 1, make_disp(srcECX, 8), 1);
-	       emit_store(p, make_disp(dest,8), 1, temp);
+	       emit_load(p, temp, 1, x86_make_disp(srcECX, 8), 1);
+	       emit_store(p, x86_make_disp(dest,8), 1, temp);
 	    }
 	    else {
-	       sse_movss(p, make_disp(dest,8), get_identity(p));
+	       sse_movss(&p->func, x86_make_disp(dest,8), get_identity(p));
 	    }
 	    update_src_ptr(p, srcECX, vtxESI, a);
 	 }
 	 break;
       case EMIT_4F:
 	 get_src_ptr(p, srcECX, vtxESI, a);
-	 emit_load(p, temp, 4, deref(srcECX), a->inputsize);
+	 emit_load(p, temp, 4, x86_deref(srcECX), a->inputsize);
 	 emit_store(p, dest, 4, temp);
 	 update_src_ptr(p, srcECX, vtxESI, a);
 	 break;
       case EMIT_2F_VIEWPORT: 
 	 get_src_ptr(p, srcECX, vtxESI, a);
-	 emit_load(p, temp, 2, deref(srcECX), a->inputsize);
-	 sse_mulps(p, temp, vp0);
-	 sse_addps(p, temp, vp1);
+	 emit_load(p, temp, 2, x86_deref(srcECX), a->inputsize);
+	 sse_mulps(&p->func, temp, vp0);
+	 sse_addps(&p->func, temp, vp1);
 	 emit_store(p, dest, 2, temp);
 	 update_src_ptr(p, srcECX, vtxESI, a);
 	 break;
       case EMIT_3F_VIEWPORT: 
 	 get_src_ptr(p, srcECX, vtxESI, a);
-	 emit_load(p, temp, 3, deref(srcECX), a->inputsize);
-	 sse_mulps(p, temp, vp0);
-	 sse_addps(p, temp, vp1);
+	 emit_load(p, temp, 3, x86_deref(srcECX), a->inputsize);
+	 sse_mulps(&p->func, temp, vp0);
+	 sse_addps(&p->func, temp, vp1);
 	 emit_store(p, dest, 3, temp);
 	 update_src_ptr(p, srcECX, vtxESI, a);
 	 break;
       case EMIT_4F_VIEWPORT: 
 	 get_src_ptr(p, srcECX, vtxESI, a);
-	 emit_load(p, temp, 4, deref(srcECX), a->inputsize);
-	 sse_mulps(p, temp, vp0);
-	 sse_addps(p, temp, vp1);
+	 emit_load(p, temp, 4, x86_deref(srcECX), a->inputsize);
+	 sse_mulps(&p->func, temp, vp0);
+	 sse_addps(&p->func, temp, vp1);
 	 emit_store(p, dest, 4, temp);
 	 update_src_ptr(p, srcECX, vtxESI, a);
 	 break;
       case EMIT_3F_XYW:
 	 get_src_ptr(p, srcECX, vtxESI, a);
-	 emit_load(p, temp, 4, deref(srcECX), a->inputsize);
-	 sse_shufps(p, temp, temp, X, Y, W, Z);
+	 emit_load(p, temp, 4, x86_deref(srcECX), a->inputsize);
+	 sse_shufps(&p->func, temp, temp, X, Y, W, Z);
 	 emit_store(p, dest, 3, temp);
 	 update_src_ptr(p, srcECX, vtxESI, a);
 	 break;
@@ -1067,9 +483,9 @@ static GLboolean build_vertex_emit( struct x86_program *p )
 	     a[-1].vertoffset + a[-1].vertattrsize <= a->vertoffset - 3)
 	 {
 	    get_src_ptr(p, srcECX, vtxESI, a);
-	    emit_load(p, temp, 1, deref(srcECX), a->inputsize);
-	    sse_shufps(p, temp, temp, X, X, X, X);
-	    emit_pack_store_4ub(p, make_disp(dest, -3), temp); /* overkill! */
+	    emit_load(p, temp, 1, x86_deref(srcECX), a->inputsize);
+	    sse_shufps(&p->func, temp, temp, X, X, X, X);
+	    emit_pack_store_4ub(p, x86_make_disp(dest, -3), temp); /* overkill! */
 	    update_src_ptr(p, srcECX, vtxESI, a);
 	 }
 	 else {
@@ -1084,9 +500,9 @@ static GLboolean build_vertex_emit( struct x86_program *p )
 	 if (j == vtx->attr_count - 1 ||
 	     a[1].vertoffset >= a->vertoffset + 4) {
 	    get_src_ptr(p, srcECX, vtxESI, a);
-	    emit_load(p, temp, 3, deref(srcECX), a->inputsize);
+	    emit_load(p, temp, 3, x86_deref(srcECX), a->inputsize);
 	    if (a->format == EMIT_3UB_3F_BGR)
-	       sse_shufps(p, temp, temp, Z, Y, X, W);
+	       sse_shufps(&p->func, temp, temp, Z, Y, X, W);
 	    emit_pack_store_4ub(p, dest, temp);
 	    update_src_ptr(p, srcECX, vtxESI, a);
 	 }
@@ -1096,23 +512,23 @@ static GLboolean build_vertex_emit( struct x86_program *p )
 		  a[1].format == EMIT_1UB_1F &&
 		  a[1].vertoffset == a->vertoffset + 3) {
 	    get_src_ptr(p, srcECX, vtxESI, a);
-	    emit_load(p, temp, 3, deref(srcECX), a->inputsize);
+	    emit_load(p, temp, 3, x86_deref(srcECX), a->inputsize);
 	    update_src_ptr(p, srcECX, vtxESI, a);
 
 	    /* Make room for incoming value:
 	     */
-	    sse_shufps(p, temp, temp, W, X, Y, Z);
+	    sse_shufps(&p->func, temp, temp, W, X, Y, Z);
 
 	    get_src_ptr(p, srcECX, vtxESI, &a[1]);
-	    emit_load(p, temp, 1, deref(srcECX), a[1].inputsize);
+	    emit_load(p, temp, 1, x86_deref(srcECX), a[1].inputsize);
 	    update_src_ptr(p, srcECX, vtxESI, &a[1]);
 
 	    /* Rearrange and possibly do BGR conversion:
 	     */
 	    if (a->format == EMIT_3UB_3F_BGR)
-	       sse_shufps(p, temp, temp, W, Z, Y, X);
+	       sse_shufps(&p->func, temp, temp, W, Z, Y, X);
 	    else
-	       sse_shufps(p, temp, temp, Y, Z, W, X);
+	       sse_shufps(&p->func, temp, temp, Y, Z, W, X);
 
 	    emit_pack_store_4ub(p, dest, temp);
 	    j++;		/* NOTE: two attrs consumed */
@@ -1125,28 +541,28 @@ static GLboolean build_vertex_emit( struct x86_program *p )
 
       case EMIT_4UB_4F_RGBA:
 	 get_src_ptr(p, srcECX, vtxESI, a);
-	 emit_load(p, temp, 4, deref(srcECX), a->inputsize);
+	 emit_load(p, temp, 4, x86_deref(srcECX), a->inputsize);
 	 emit_pack_store_4ub(p, dest, temp);
 	 update_src_ptr(p, srcECX, vtxESI, a);
 	 break;
       case EMIT_4UB_4F_BGRA:
 	 get_src_ptr(p, srcECX, vtxESI, a);
-	 emit_load(p, temp, 4, deref(srcECX), a->inputsize);
-	 sse_shufps(p, temp, temp, Z, Y, X, W);
+	 emit_load(p, temp, 4, x86_deref(srcECX), a->inputsize);
+	 sse_shufps(&p->func, temp, temp, Z, Y, X, W);
 	 emit_pack_store_4ub(p, dest, temp);
 	 update_src_ptr(p, srcECX, vtxESI, a);
 	 break;
       case EMIT_4UB_4F_ARGB:
 	 get_src_ptr(p, srcECX, vtxESI, a);
-	 emit_load(p, temp, 4, deref(srcECX), a->inputsize);
-	 sse_shufps(p, temp, temp, W, X, Y, Z);
+	 emit_load(p, temp, 4, x86_deref(srcECX), a->inputsize);
+	 sse_shufps(&p->func, temp, temp, W, X, Y, Z);
 	 emit_pack_store_4ub(p, dest, temp);
 	 update_src_ptr(p, srcECX, vtxESI, a);
 	 break;
       case EMIT_4UB_4F_ABGR:
 	 get_src_ptr(p, srcECX, vtxESI, a);
-	 emit_load(p, temp, 4, deref(srcECX), a->inputsize);
-	 sse_shufps(p, temp, temp, W, Z, Y, X);
+	 emit_load(p, temp, 4, x86_deref(srcECX), a->inputsize);
+	 sse_shufps(&p->func, temp, temp, W, Z, Y, X);
 	 emit_pack_store_4ub(p, dest, temp);
 	 update_src_ptr(p, srcECX, vtxESI, a);
 	 break;
@@ -1154,13 +570,13 @@ static GLboolean build_vertex_emit( struct x86_program *p )
 	 switch (CHAN_TYPE) {
 	 case GL_UNSIGNED_BYTE:
 	    get_src_ptr(p, srcECX, vtxESI, a);
-	    emit_load(p, temp, 4, deref(srcECX), a->inputsize);
+	    emit_load(p, temp, 4, x86_deref(srcECX), a->inputsize);
 	    emit_pack_store_4ub(p, dest, temp);
 	    update_src_ptr(p, srcECX, vtxESI, a);
 	    break;
 	 case GL_FLOAT:
 	    get_src_ptr(p, srcECX, vtxESI, a);
-	    emit_load(p, temp, 4, deref(srcECX), a->inputsize);
+	    emit_load(p, temp, 4, x86_deref(srcECX), a->inputsize);
 	    emit_store(p, dest, 4, temp);
 	    update_src_ptr(p, srcECX, vtxESI, a);
 	    break;
@@ -1182,35 +598,33 @@ static GLboolean build_vertex_emit( struct x86_program *p )
 
    /* Next vertex:
     */
-   x86_lea(p, vertexEAX, make_disp(vertexEAX, vtx->vertex_size));
+   x86_lea(&p->func, vertexEAX, x86_make_disp(vertexEAX, vtx->vertex_size));
 
    /* decr count, loop if not zero
     */
-   x86_dec(p, countEBP);
-   x86_test(p, countEBP, countEBP); 
-   x86_jcc(p, cc_NZ, label);
+   x86_dec(&p->func, countEBP);
+   x86_test(&p->func, countEBP, countEBP); 
+   x86_jcc(&p->func, cc_NZ, label);
 
    /* Exit mmx state?
     */
-   if (p->need_emms)
-      mmx_emms(p);
+   if (p->func.need_emms)
+      mmx_emms(&p->func);
 
    /* Land forward jump here:
     */
-   do_fixup(p, fixup);
+   x86_fixup_fwd_jump(&p->func, fixup);
 
    /* Pop regs and return
     */
-   x86_pop(p, get_base_reg(vtxESI));
-   x86_pop(p, countEBP);
-/*    x86_pop(p, srcECX); */
-   x86_ret(p);
+   x86_pop(&p->func, x86_get_base_reg(vtxESI));
+   x86_pop(&p->func, countEBP);
+   x86_ret(&p->func);
 
-   vtx->emit = (tnl_emit_func)p->store;
+   vtx->emit = (tnl_emit_func)x86_get_func(&p->func);
    return GL_TRUE;
 }
 
-#include "x86/common_x86_asm.h"
 
 
 void _tnl_generate_sse_emit( GLcontext *ctx )
@@ -1224,37 +638,26 @@ void _tnl_generate_sse_emit( GLcontext *ctx )
    }
 
    memset(&p, 0, sizeof(p));
-   p.ctx = ctx;
-   p.store = MALLOC(1024);
 
+   p.ctx = ctx;
    p.inputs_safe = 0;		/* for now */
    p.outputs_safe = 1;		/* for now */
    p.have_sse2 = cpu_has_xmm2;
-   p.identity = make_reg(file_XMM, 6);
-   p.chan0 = make_reg(file_XMM, 7);
+   p.identity = x86_make_reg(file_XMM, 6);
+   p.chan0 = x86_make_reg(file_XMM, 7);
+
+   x86_init_func(&p.func);
 
    if (build_vertex_emit(&p)) {
       _tnl_register_fastpath( vtx, GL_TRUE );
-      if (DISASSEM)
-	 _mesa_printf("disassemble 0x%x 0x%x\n", p.store, p.csr);
    }
    else {
       /* Note the failure so that we don't keep trying to codegen an
        * impossible state:
        */
       _tnl_register_fastpath( vtx, GL_FALSE );
-      FREE(p.store);
+      x86_release_func(&p.func);
    }
-
-   (void)sse2_movd;
-   (void)x86_inc;
-   (void)x86_xor;
-   (void)mmx_movq;
-   (void)sse_movlhps;
-   (void)sse_movhps;
-   (void)sse_movaps;
-   (void)sse2_packsswb;
-   (void)sse2_pshufd;
 }
 
 #else