Merge branch 'master' into glsl2

49 files changed, 1439 insertions, 1001 deletions

diff --git a/src/gallium/auxiliary/Makefile b/src/gallium/auxiliary/Makefile
index dcebab7c0f2..843b72bc38b 100644
--- a/src/gallium/auxiliary/Makefile
+++ b/src/gallium/auxiliary/Makefile
@@ -149,6 +149,7 @@ C_SOURCES = \
 
 GALLIVM_SOURCES = \
         gallivm/lp_bld_arit.c \
+        gallivm/lp_bld_assert.c \
         gallivm/lp_bld_const.c \
         gallivm/lp_bld_conv.c \
         gallivm/lp_bld_debug.c \
diff --git a/src/gallium/auxiliary/SConscript b/src/gallium/auxiliary/SConscript
index 72a16617db8..1f091987218 100644
--- a/src/gallium/auxiliary/SConscript
+++ b/src/gallium/auxiliary/SConscript
@@ -34,14 +34,14 @@ env.CodeGenerate(
     target = 'util/u_format_table.c',
     script = '#src/gallium/auxiliary/util/u_format_table.py',
     source = ['#src/gallium/auxiliary/util/u_format.csv'],
-    command = 'python $SCRIPT $SOURCE > $TARGET'
+    command = python_cmd + ' $SCRIPT $SOURCE > $TARGET'
 )
 
 env.CodeGenerate(
     target = 'util/u_half.c',
     script = 'util/u_half.py',
     source = [],
-    command = 'python $SCRIPT > $TARGET'
+    command = python_cmd + ' $SCRIPT > $TARGET'
 )
 
 env.Depends('util/u_format_table.c', [
@@ -198,6 +198,7 @@ source = [
 if env['llvm']:
     source += [
     'gallivm/lp_bld_arit.c',
+    'gallivm/lp_bld_assert.c',
     'gallivm/lp_bld_const.c',
     'gallivm/lp_bld_conv.c',
     'gallivm/lp_bld_debug.c',
diff --git a/src/gallium/auxiliary/draw/draw_context.c b/src/gallium/auxiliary/draw/draw_context.c
index c127f741881..995b675b9a1 100644
--- a/src/gallium/auxiliary/draw/draw_context.c
+++ b/src/gallium/auxiliary/draw/draw_context.c
@@ -288,12 +288,19 @@ draw_set_mapped_constant_buffer(struct draw_context *draw,
                 shader_type == PIPE_SHADER_GEOMETRY);
    debug_assert(slot < PIPE_MAX_CONSTANT_BUFFERS);
 
-   if (shader_type == PIPE_SHADER_VERTEX) {
+   switch (shader_type) {
+   case PIPE_SHADER_VERTEX:
       draw->pt.user.vs_constants[slot] = buffer;
+      draw->pt.user.vs_constants_size[slot] = size;
       draw_vs_set_constants(draw, slot, buffer, size);
-   } else if (shader_type == PIPE_SHADER_GEOMETRY) {
+      break;
+   case PIPE_SHADER_GEOMETRY:
       draw->pt.user.gs_constants[slot] = buffer;
+      draw->pt.user.gs_constants_size[slot] = size;
       draw_gs_set_constants(draw, slot, buffer, size);
+      break;
+   default:
+      assert(0 && "invalid shader type in draw_set_mapped_constant_buffer");
    }
 }
 
diff --git a/src/gallium/auxiliary/draw/draw_decompose_tmp.h b/src/gallium/auxiliary/draw/draw_decompose_tmp.h
new file mode 100644
index 00000000000..a52d2b50588
--- /dev/null
+++ b/src/gallium/auxiliary/draw/draw_decompose_tmp.h
@@ -0,0 +1,425 @@
+/*
+ * Mesa 3-D graphics library
+ * Version:  7.9
+ *
+ * Copyright 2008 Tungsten Graphics, Inc., Cedar Park, Texas.
+ * Copyright (C) 2010 LunarG Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ *
+ * Authors:
+  *   Keith Whitwell <[email protected]>
+ *    Chia-I Wu <[email protected]>
+ */
+
+/* these macros are optional */
+#ifndef LOCAL_VARS
+#define LOCAL_VARS
+#endif
+#ifndef FUNC_ENTER
+#define FUNC_ENTER do {} while (0)
+#endif
+#ifndef FUNC_EXIT
+#define FUNC_EXIT do {} while (0)
+#endif
+#ifndef LINE_ADJ
+#define LINE_ADJ(flags, a0, i0, i1, a1) LINE(flags, i0, i1)
+#endif
+#ifndef TRIANGLE_ADJ
+#define TRIANGLE_ADJ(flags, i0, a0, i1, a1, i2, a2) TRIANGLE(flags, i0, i1, i2)
+#endif
+
+static void
+FUNC(FUNC_VARS)
+{
+   unsigned idx[6], i;
+   ushort flags;
+   LOCAL_VARS
+
+   FUNC_ENTER;
+
+   /* prim, count, and last_vertex_last should have been defined */
+   if (0) {
+      debug_printf("%s: prim 0x%x, count %d, last_vertex_last %d\n",
+            __FUNCTION__, prim, count, last_vertex_last);
+   }
+
+   switch (prim) {
+   case PIPE_PRIM_POINTS:
+      for (i = 0; i < count; i++) {
+         idx[0] = GET_ELT(i);
+         POINT(idx[0]);
+      }
+      break;
+
+   case PIPE_PRIM_LINES:
+      flags = DRAW_PIPE_RESET_STIPPLE;
+      for (i = 0; i + 1 < count; i += 2) {
+         idx[0] = GET_ELT(i);
+         idx[1] = GET_ELT(i + 1);
+         LINE(flags, idx[0], idx[1]);
+      }
+      break;
+
+   case PIPE_PRIM_LINE_LOOP:
+   case PIPE_PRIM_LINE_STRIP:
+      if (count >= 2) {
+         flags = DRAW_PIPE_RESET_STIPPLE;
+         idx[1] = GET_ELT(0);
+         idx[2] = idx[1];
+
+         for (i = 1; i < count; i++, flags = 0) {
+            idx[0] = idx[1];
+            idx[1] = GET_ELT(i);
+            LINE(flags, idx[0], idx[1]);
+         }
+         /* close the loop */
+         if (prim == PIPE_PRIM_LINE_LOOP)
+            LINE(flags, idx[1], idx[2]);
+      }
+      break;
+
+   case PIPE_PRIM_TRIANGLES:
+      flags = DRAW_PIPE_RESET_STIPPLE | DRAW_PIPE_EDGE_FLAG_ALL;
+      for (i = 0; i + 2 < count; i += 3) {
+         idx[0] = GET_ELT(i);
+         idx[1] = GET_ELT(i + 1);
+         idx[2] = GET_ELT(i + 2);
+         TRIANGLE(flags, idx[0], idx[1], idx[2]);
+      }
+      break;
+
+   case PIPE_PRIM_TRIANGLE_STRIP:
+      if (count >= 3) {
+         flags = DRAW_PIPE_RESET_STIPPLE | DRAW_PIPE_EDGE_FLAG_ALL;
+         idx[1] = GET_ELT(0);
+         idx[2] = GET_ELT(1);
+
+         if (last_vertex_last) {
+            for (i = 0; i + 2 < count; i++) {
+               idx[0] = idx[1];
+               idx[1] = idx[2];
+               idx[2] = GET_ELT(i + 2);
+               /* always emit idx[2] last */
+               if (i & 1)
+                  TRIANGLE(flags, idx[1], idx[0], idx[2]);
+               else
+                  TRIANGLE(flags, idx[0], idx[1], idx[2]);
+            }
+         }
+         else {
+            for (i = 0; i + 2 < count; i++) {
+               idx[0] = idx[1];
+               idx[1] = idx[2];
+               idx[2] = GET_ELT(i + 2);
+               /* always emit idx[0] first */
+               if (i & 1)
+                  TRIANGLE(flags, idx[0], idx[2], idx[1]);
+               else
+                  TRIANGLE(flags, idx[0], idx[1], idx[2]);
+            }
+         }
+      }
+      break;
+
+   case PIPE_PRIM_TRIANGLE_FAN:
+      if (count >= 3) {
+         flags = DRAW_PIPE_RESET_STIPPLE | DRAW_PIPE_EDGE_FLAG_ALL;
+         idx[0] = GET_ELT(0);
+         idx[2] = GET_ELT(1);
+
+         /* idx[0] is neither the first nor the last vertex */
+         if (last_vertex_last) {
+            for (i = 0; i + 2 < count; i++) {
+               idx[1] = idx[2];
+               idx[2] = GET_ELT(i + 2);
+               /* always emit idx[2] last */
+               TRIANGLE(flags, idx[0], idx[1], idx[2]);
+            }
+         }
+         else {
+            for (i = 0; i + 2 < count; i++) {
+               idx[1] = idx[2];
+               idx[2] = GET_ELT(i + 2);
+               /* always emit idx[1] first */
+               TRIANGLE(flags, idx[1], idx[2], idx[0]);
+            }
+         }
+      }
+      break;
+
+   case PIPE_PRIM_QUADS:
+      if (last_vertex_last) {
+         for (i = 0; i + 3 < count; i += 4) {
+            idx[0] = GET_ELT(i);
+            idx[1] = GET_ELT(i + 1);
+            idx[2] = GET_ELT(i + 2);
+            idx[3] = GET_ELT(i + 3);
+
+            flags = DRAW_PIPE_RESET_STIPPLE |
+                    DRAW_PIPE_EDGE_FLAG_0 |
+                    DRAW_PIPE_EDGE_FLAG_2;
+            /* always emit idx[3] last */
+            TRIANGLE(flags, idx[0], idx[1], idx[3]);
+
+            flags = DRAW_PIPE_EDGE_FLAG_0 |
+                    DRAW_PIPE_EDGE_FLAG_1;
+            TRIANGLE(flags, idx[1], idx[2], idx[3]);
+         }
+      }
+      else {
+         for (i = 0; i + 3 < count; i += 4) {
+            idx[0] = GET_ELT(i);
+            idx[1] = GET_ELT(i + 1);
+            idx[2] = GET_ELT(i + 2);
+            idx[3] = GET_ELT(i + 3);
+
+            flags = DRAW_PIPE_RESET_STIPPLE |
+                    DRAW_PIPE_EDGE_FLAG_0 |
+                    DRAW_PIPE_EDGE_FLAG_1;
+            /* XXX should always emit idx[0] first */
+            /* always emit idx[3] first */
+            TRIANGLE(flags, idx[3], idx[0], idx[1]);
+
+            flags = DRAW_PIPE_EDGE_FLAG_1 |
+                    DRAW_PIPE_EDGE_FLAG_2;
+            TRIANGLE(flags, idx[3], idx[1], idx[2]);
+         }
+      }
+      break;
+
+   case PIPE_PRIM_QUAD_STRIP:
+      if (count >= 4) {
+         idx[2] = GET_ELT(0);
+         idx[3] = GET_ELT(1);
+
+         if (last_vertex_last) {
+            for (i = 0; i + 3 < count; i += 2) {
+               idx[0] = idx[2];
+               idx[1] = idx[3];
+               idx[2] = GET_ELT(i + 2);
+               idx[3] = GET_ELT(i + 3);
+
+               /* always emit idx[3] last */
+               flags = DRAW_PIPE_RESET_STIPPLE |
+                       DRAW_PIPE_EDGE_FLAG_0 |
+                       DRAW_PIPE_EDGE_FLAG_2;
+               TRIANGLE(flags, idx[2], idx[0], idx[3]);
+
+               flags = DRAW_PIPE_EDGE_FLAG_0 |
+                       DRAW_PIPE_EDGE_FLAG_1;
+               TRIANGLE(flags, idx[0], idx[1], idx[3]);
+            }
+         }
+         else {
+            for (i = 0; i + 3 < count; i += 2) {
+               idx[0] = idx[2];
+               idx[1] = idx[3];
+               idx[2] = GET_ELT(i + 2);
+               idx[3] = GET_ELT(i + 3);
+
+               flags = DRAW_PIPE_RESET_STIPPLE |
+                       DRAW_PIPE_EDGE_FLAG_0 |
+                       DRAW_PIPE_EDGE_FLAG_1;
+               /* XXX should always emit idx[0] first */
+               /* always emit idx[3] first */
+               TRIANGLE(flags, idx[3], idx[2], idx[0]);
+
+               flags = DRAW_PIPE_EDGE_FLAG_1 |
+                       DRAW_PIPE_EDGE_FLAG_2;
+               TRIANGLE(flags, idx[3], idx[0], idx[1]);
+            }
+         }
+      }
+      break;
+
+   case PIPE_PRIM_POLYGON:
+      if (count >= 3) {
+         ushort edge_next, edge_finish;
+
+         if (last_vertex_last) {
+            flags = (DRAW_PIPE_RESET_STIPPLE |
+                     DRAW_PIPE_EDGE_FLAG_2 |
+                     DRAW_PIPE_EDGE_FLAG_0);
+            edge_next = DRAW_PIPE_EDGE_FLAG_0;
+            edge_finish = DRAW_PIPE_EDGE_FLAG_1;
+         }
+         else {
+            flags = (DRAW_PIPE_RESET_STIPPLE |
+                     DRAW_PIPE_EDGE_FLAG_0 |
+                     DRAW_PIPE_EDGE_FLAG_1);
+            edge_next = DRAW_PIPE_EDGE_FLAG_1;
+            edge_finish = DRAW_PIPE_EDGE_FLAG_2;
+         }
+
+         idx[0] = GET_ELT(0);
+         idx[2] = GET_ELT(1);
+
+         for (i = 0; i + 2 < count; i++, flags = edge_next) {
+            idx[1] = idx[2];
+            idx[2] = GET_ELT(i + 2);
+
+            if (i + 3 == count)
+               flags |= edge_finish;
+
+            /* idx[0] is both the first and the last vertex */
+            if (last_vertex_last)
+               TRIANGLE(flags, idx[1], idx[2], idx[0]);
+            else
+               TRIANGLE(flags, idx[0], idx[1], idx[2]);
+         }
+      }
+      break;
+
+   case PIPE_PRIM_LINES_ADJACENCY:
+      flags = DRAW_PIPE_RESET_STIPPLE;
+      for (i = 0; i + 3 < count; i += 4) {
+         idx[0] = GET_ELT(i);
+         idx[1] = GET_ELT(i + 1);
+         idx[2] = GET_ELT(i + 2);
+         idx[3] = GET_ELT(i + 3);
+         LINE_ADJ(flags, idx[0], idx[1], idx[2], idx[3]);
+      }
+      break;
+
+   case PIPE_PRIM_LINE_STRIP_ADJACENCY:
+      if (count >= 4) {
+         flags = DRAW_PIPE_RESET_STIPPLE;
+         idx[1] = GET_ELT(0);
+         idx[2] = GET_ELT(1);
+         idx[3] = GET_ELT(2);
+
+         for (i = 1; i + 2 < count; i++, flags = 0) {
+            idx[0] = idx[1];
+            idx[1] = idx[2];
+            idx[2] = idx[3];
+            idx[3] = GET_ELT(i + 2);
+            LINE_ADJ(flags, idx[0], idx[1], idx[2], idx[3]);
+         }
+      }
+      break;
+
+   case PIPE_PRIM_TRIANGLES_ADJACENCY:
+      flags = DRAW_PIPE_RESET_STIPPLE | DRAW_PIPE_EDGE_FLAG_ALL;
+      for (i = 0; i + 5 < count; i += 6) {
+         idx[0] = GET_ELT(i);
+         idx[1] = GET_ELT(i + 1);
+         idx[2] = GET_ELT(i + 2);
+         idx[3] = GET_ELT(i + 3);
+         idx[4] = GET_ELT(i + 4);
+         idx[5] = GET_ELT(i + 5);
+         TRIANGLE_ADJ(flags, idx[0], idx[1], idx[2], idx[3], idx[4], idx[5]);
+      }
+      break;
+
+   case PIPE_PRIM_TRIANGLE_STRIP_ADJACENCY:
+      if (count >= 6) {
+         flags = DRAW_PIPE_RESET_STIPPLE | DRAW_PIPE_EDGE_FLAG_ALL;
+         idx[0] = GET_ELT(1);
+         idx[2] = GET_ELT(0);
+         idx[4] = GET_ELT(2);
+         idx[3] = GET_ELT(4);
+
+         /*
+          * The vertices of the i-th triangle are stored in
+          * idx[0,2,4] = { 2*i, 2*i+2, 2*i+4 };
+          *
+          * The adjacent vertices are stored in
+          * idx[1,3,5] = { 2*i-2, 2*i+6, 2*i+3 }.
+          *
+          * However, there are two exceptions:
+          *
+          * For the first triangle, idx[1] = 1;
+          * For the  last triangle, idx[3] = 2*i+5.
+          */
+         if (last_vertex_last) {
+            for (i = 0; i + 5 < count; i += 2) {
+               idx[1] = idx[0];
+
+               idx[0] = idx[2];
+               idx[2] = idx[4];
+               idx[4] = idx[3];
+
+               idx[3] = GET_ELT(i + ((i + 7 < count) ? 6 : 5));
+               idx[5] = GET_ELT(i + 3);
+
+               /*
+                * alternate the first two vertices (idx[0] and idx[2]) and the
+                * corresponding adjacent vertices (idx[3] and idx[5]) to have
+                * the correct orientation
+                */
+               if (i & 2) {
+                  TRIANGLE_ADJ(flags,
+                        idx[2], idx[1], idx[0], idx[5], idx[4], idx[3]);
+               }
+               else {
+                  TRIANGLE_ADJ(flags,
+                        idx[0], idx[1], idx[2], idx[3], idx[4], idx[5]);
+               }
+            }
+         }
+         else {
+            for (i = 0; i + 5 < count; i += 2) {
+               idx[1] = idx[0];
+
+               idx[0] = idx[2];
+               idx[2] = idx[4];
+               idx[4] = idx[3];
+
+               idx[3] = GET_ELT(i + ((i + 7 < count) ? 6 : 5));
+               idx[5] = GET_ELT(i + 3);
+
+               /*
+                * alternate the last two vertices (idx[2] and idx[4]) and the
+                * corresponding adjacent vertices (idx[1] and idx[5]) to have
+                * the correct orientation
+                */
+               if (i & 2) {
+                  TRIANGLE_ADJ(flags,
+                        idx[0], idx[5], idx[4], idx[3], idx[2], idx[1]);
+               }
+               else {
+                  TRIANGLE_ADJ(flags,
+                        idx[0], idx[1], idx[2], idx[3], idx[4], idx[5]);
+               }
+            }
+         }
+      }
+      break;
+
+   default:
+      assert(0);
+      break;
+   }
+
+   FUNC_EXIT;
+}
+
+#undef LOCAL_VARS
+#undef FUNC_ENTER
+#undef FUNC_EXIT
+#undef LINE_ADJ
+#undef TRIANGLE_ADJ
+
+#undef FUNC
+#undef FUNC_VARS
+#undef GET_ELT
+#undef POINT
+#undef LINE
+#undef TRIANGLE
diff --git a/src/gallium/auxiliary/draw/draw_gs.c b/src/gallium/auxiliary/draw/draw_gs.c
index 79a57a67f3e..4a1013e79a5 100644
--- a/src/gallium/auxiliary/draw/draw_gs.c
+++ b/src/gallium/auxiliary/draw/draw_gs.c
@@ -1,6 +1,6 @@
 /**************************************************************************
  *
- * Copyright 2009 VMWare Inc.
+ * Copyright 2009 VMware, Inc.
  * All Rights Reserved.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
@@ -75,7 +75,10 @@ draw_gs_set_constants(struct draw_context *draw,
                       const void *constants,
                       unsigned size)
 {
-   /* noop */
+   /* noop. added here for symmetry with the VS
+    * code and in case we'll ever want to allign
+    * the constants, e.g. when we'll change to a
+    * different interpreter */
 }
 
 
@@ -370,32 +373,23 @@ static void gs_tri_adj(struct draw_geometry_shader *shader,
    gs_flush(shader, 1);
 }
 
-#define TRIANGLE(gs,i0,i1,i2) gs_tri(gs,i0,i1,i2)
-#define TRI_ADJ(gs,i0,i1,i2,i3,i4,i5)  gs_tri_adj(gs,i0,i1,i2,i3,i4,i5)
-#define LINE(gs,i0,i1)        gs_line(gs,i0,i1)
-#define LINE_ADJ(gs,i0,i1,i2,i3)    gs_line_adj(gs,i0,i1,i2,i3)
-#define POINT(gs,i0)          gs_point(gs,i0)
-#define FUNC gs_run
-#define LOCAL_VARS
+#define FUNC         gs_run
+#define GET_ELT(idx) (idx)
 #include "draw_gs_tmp.h"
 
 
-#define TRIANGLE(gs,i0,i1,i2) gs_tri(gs,elts[i0],elts[i1],elts[i2])
-#define TRI_ADJ(gs,i0,i1,i2,i3,i4,i5)           \
-   gs_tri_adj(gs,elts[i0],elts[i1],elts[i2],elts[i3], \
-              elts[i4],elts[i5])
-#define LINE(gs,i0,i1)        gs_line(gs,elts[i0],elts[i1])
-#define LINE_ADJ(gs,i0,i1,i2,i3)  gs_line_adj(gs,elts[i0],      \
-                                              elts[i1],         \
-                                              elts[i2],elts[i3])
-#define POINT(gs,i0)          gs_point(gs,elts[i0])
-#define FUNC gs_run_elts
-#define LOCAL_VARS                         \
-   const ushort *elts = input_prims->elts;
+#define FUNC         gs_run_elts
+#define LOCAL_VARS   const ushort *elts = input_prims->elts;
+#define GET_ELT(idx) (elts[idx] & ~DRAW_PIPE_FLAG_MASK)
 #include "draw_gs_tmp.h"
 
+
+/**
+ * Execute geometry shader using TGSI interpreter.
+ */
 int draw_geometry_shader_run(struct draw_geometry_shader *shader,
                              const void *constants[PIPE_MAX_CONSTANT_BUFFERS], 
+                             const unsigned constants_size[PIPE_MAX_CONSTANT_BUFFERS], 
                              const struct draw_vertex_info *input_verts,
                              const struct draw_prim_info *input_prim,
                              struct draw_vertex_info *output_verts,
@@ -405,7 +399,6 @@ int draw_geometry_shader_run(struct draw_geometry_shader *shader,
    unsigned input_stride = input_verts->vertex_size;
    unsigned vertex_size = input_verts->vertex_size;
    struct tgsi_exec_machine *machine = shader->machine;
-   unsigned int i;
    unsigned num_input_verts = input_prim->linear ?
                               input_verts->count :
                               input_prim->count;
@@ -447,9 +440,8 @@ int draw_geometry_shader_run(struct draw_geometry_shader *shader,
    }
    shader->primitive_lengths = MALLOC(max_out_prims * sizeof(unsigned));
 
-   for (i = 0; i < PIPE_MAX_CONSTANT_BUFFERS; i++) {
-      machine->Consts[i] = constants[i];
-   }
+   tgsi_exec_set_constant_buffers(machine, PIPE_MAX_CONSTANT_BUFFERS,
+                                  constants, constants_size);
 
    if (input_prim->linear)
       gs_run(shader, input_prim, input_verts,
diff --git a/src/gallium/auxiliary/draw/draw_gs.h b/src/gallium/auxiliary/draw/draw_gs.h
index 2cb634818c2..67bc1aa73ff 100644
--- a/src/gallium/auxiliary/draw/draw_gs.h
+++ b/src/gallium/auxiliary/draw/draw_gs.h
@@ -1,6 +1,6 @@
 /**************************************************************************
  * 
- * Copyright 2009 VMWare Inc.
+ * Copyright 2009 VMware, Inc.
  * All Rights Reserved.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
@@ -73,6 +73,7 @@ struct draw_geometry_shader {
  */
 int draw_geometry_shader_run(struct draw_geometry_shader *shader,
                              const void *constants[PIPE_MAX_CONSTANT_BUFFERS], 
+                             const unsigned constants_size[PIPE_MAX_CONSTANT_BUFFERS], 
                              const struct draw_vertex_info *input_verts,
                              const struct draw_prim_info *input_prim,
                              struct draw_vertex_info *output_verts,
diff --git a/src/gallium/auxiliary/draw/draw_gs_tmp.h b/src/gallium/auxiliary/draw/draw_gs_tmp.h
index 7a8683cf7c2..4a17af0dea3 100644
--- a/src/gallium/auxiliary/draw/draw_gs_tmp.h
+++ b/src/gallium/auxiliary/draw/draw_gs_tmp.h
@@ -1,152 +1,34 @@
-
-static void FUNC( struct draw_geometry_shader *shader,
-                  const struct draw_prim_info *input_prims,
-                  const struct draw_vertex_info *input_verts,
-                  struct draw_prim_info *output_prims,
-                  struct draw_vertex_info *output_verts)
-{
-   struct draw_context *draw = shader->draw;
-
-   boolean flatfirst = (draw->rasterizer->flatshade &&
-                        draw->rasterizer->flatshade_first);
-   unsigned i, j;
-   unsigned count = input_prims->count;
-   LOCAL_VARS
-
-   if (0) debug_printf("%s %d\n", __FUNCTION__, count);
-
-   debug_assert(input_prims->primitive_count == 1);
-
-   switch (input_prims->prim) {
-   case PIPE_PRIM_POINTS:
-      for (i = 0; i < count; i++) {
-	 POINT( shader, i + 0 );
-      }
-      break;
-
-   case PIPE_PRIM_LINES:
-      for (i = 0; i+1 < count; i += 2) {
-         LINE( shader , i + 0 , i + 1 );
-      }
-      break;
-
-   case PIPE_PRIM_LINE_LOOP:
-      if (count >= 2) {
-
-         for (i = 1; i < count; i++) {
-            LINE( shader, i - 1, i );
-         }
-
-	 LINE( shader, i - 1, 0 );
-      }
-      break;
-
-   case PIPE_PRIM_LINE_STRIP:
-      for (i = 1; i < count; i++) {
-         LINE( shader, i - 1, i );
-      }
-      break;
-
-   case PIPE_PRIM_TRIANGLES:
-      for (i = 0; i+2 < count; i += 3) {
-         TRIANGLE( shader, i + 0, i + 1, i + 2 );
-      }
-      break;
-
-   case PIPE_PRIM_TRIANGLE_STRIP:
-      if (flatfirst) {
-         for (i = 0; i+2 < count; i++) {
-            TRIANGLE( shader,
-                      i + 0,
-                      i + 1 + (i&1),
-                      i + 2 - (i&1) );
-         }
-      }
-      else {
-         for (i = 0; i+2 < count; i++) {
-            TRIANGLE( shader,
-                      i + 0 + (i&1),
-                      i + 1 - (i&1),
-                      i + 2 );
-         }
-      }
-      break;
-
-   case PIPE_PRIM_TRIANGLE_FAN:
-      if (count >= 3) {
-         if (flatfirst) {
-            for (i = 0; i+2 < count; i++) {
-               TRIANGLE( shader,
-                         i + 1,
-                         i + 2,
-                         0 );
-            }
-         }
-         else {
-            for (i = 0; i+2 < count; i++) {
-               TRIANGLE( shader,
-                         0,
-                         i + 1,
-                         i + 2 );
-            }
-         }
-      }
-      break;
-
-   case PIPE_PRIM_POLYGON:
-      {
-	 for (i = 0; i+2 < count; i++) {
-
-            if (flatfirst) {
-               TRIANGLE( shader, 0, i + 1, i + 2 );
-            }
-            else {
-               TRIANGLE( shader, i + 1, i + 2, 0 );
-            }
-	 }
-      }
-      break;
-
-   case PIPE_PRIM_LINES_ADJACENCY:
-      for (i = 0; i+3 < count; i += 4) {
-         LINE_ADJ( shader , i + 0 , i + 1, i + 2, i + 3 );
-      }
-      break;
-   case PIPE_PRIM_LINE_STRIP_ADJACENCY:
-      for (i = 1; i + 2 < count; i++) {
-         LINE_ADJ( shader, i - 1, i, i + 1, i + 2 );
-      }
-      break;
-
-   case PIPE_PRIM_TRIANGLES_ADJACENCY:
-      for (i = 0; i+5 < count; i += 5) {
-         TRI_ADJ( shader, i + 0, i + 1, i + 2,
-                  i + 3, i + 4, i + 5);
-      }
-      break;
-   case PIPE_PRIM_TRIANGLE_STRIP_ADJACENCY:
-      for (i = 0, j = 0; i+5 < count; i += 2, ++j) {
-         TRI_ADJ( shader,
-                  i + 0,
-                  i + 1 + 2*(j&1),
-                  i + 2 + 2*(j&1),
-                  i + 3 - 2*(j&1),
-                  i + 4 - 2*(j&1),
-                  i + 5);
-      }
-      break;
-
-   default:
-      debug_assert(!"Unsupported primitive in geometry shader");
-      break;
-   }
-}
-
-
-#undef TRIANGLE
-#undef TRI_ADJ
-#undef POINT
-#undef LINE
-#undef LINE_ADJ
-#undef FUNC
-#undef LOCAL_VARS
+#define FUNC_VARS struct draw_geometry_shader *gs,             \
+                  const struct draw_prim_info *input_prims,    \
+                  const struct draw_vertex_info *input_verts,  \
+                  struct draw_prim_info *output_prims,         \
+                  struct draw_vertex_info *output_verts
+
+#define FUNC_ENTER                                                \
+   /* declare more local vars */                                  \
+   struct draw_context *draw = gs->draw;                          \
+   const unsigned prim = input_prims->prim;                       \
+   const unsigned count = input_prims->count;                     \
+   const boolean last_vertex_last =                               \
+      !(draw->rasterizer->flatshade &&                            \
+        draw->rasterizer->flatshade_first);                       \
+   do {                                                           \
+      debug_assert(input_prims->primitive_count == 1);            \
+      switch (prim) {                                             \
+      case PIPE_PRIM_QUADS:                                       \
+      case PIPE_PRIM_QUAD_STRIP:                                  \
+      case PIPE_PRIM_POLYGON:                                     \
+         debug_assert(!"unexpected primitive type in GS");        \
+         return;                                                  \
+      default:                                                    \
+         break;                                                   \
+      }                                                           \
+   } while (0)                                                    \
+
+#define POINT(i0)                             gs_point(gs,i0)
+#define LINE(flags,i0,i1)                     gs_line(gs,i0,i1)
+#define TRIANGLE(flags,i0,i1,i2)              gs_tri(gs,i0,i1,i2)
+#define LINE_ADJ(flags,i0,i1,i2,i3)           gs_line_adj(gs,i0,i1,i2,i3)
+#define TRIANGLE_ADJ(flags,i0,i1,i2,i3,i4,i5) gs_tri_adj(gs,i0,i1,i2,i3,i4,i5)
+
+#include "draw_decompose_tmp.h"
diff --git a/src/gallium/auxiliary/draw/draw_llvm.c b/src/gallium/auxiliary/draw/draw_llvm.c
index 19f96c37ab4..de99b00a814 100644
--- a/src/gallium/auxiliary/draw/draw_llvm.c
+++ b/src/gallium/auxiliary/draw/draw_llvm.c
@@ -37,6 +37,8 @@
 #include "gallivm/lp_bld_debug.h"
 #include "gallivm/lp_bld_tgsi.h"
 #include "gallivm/lp_bld_printf.h"
+#include "gallivm/lp_bld_intr.h"
+#include "gallivm/lp_bld_init.h"
 
 #include "tgsi/tgsi_exec.h"
 #include "tgsi/tgsi_dump.h"
@@ -793,6 +795,11 @@ draw_llvm_generate(struct draw_llvm *llvm, struct draw_llvm_variant *variant)
 
    sampler->destroy(sampler);
 
+#ifdef PIPE_ARCH_X86
+   /* Avoid corrupting the FPU stack on 32bit OSes. */
+   lp_build_intrinsic(builder, "llvm.x86.mmx.emms", LLVMVoidType(), NULL, 0);
+#endif
+
    LLVMBuildRetVoid(builder);
 
    LLVMDisposeBuilder(builder);
@@ -820,6 +827,7 @@ draw_llvm_generate(struct draw_llvm *llvm, struct draw_llvm_variant *variant)
    if (gallivm_debug & GALLIVM_DEBUG_ASM) {
       lp_disassemble(code);
    }
+   lp_func_delete_body(variant->function);
 }
 
 
@@ -963,6 +971,11 @@ draw_llvm_generate_elts(struct draw_llvm *llvm, struct draw_llvm_variant *varian
 
    sampler->destroy(sampler);
 
+#ifdef PIPE_ARCH_X86
+   /* Avoid corrupting the FPU stack on 32bit OSes. */
+   lp_build_intrinsic(builder, "llvm.x86.mmx.emms", LLVMVoidType(), NULL, 0);
+#endif
+
    LLVMBuildRetVoid(builder);
 
    LLVMDisposeBuilder(builder);
@@ -990,6 +1003,7 @@ draw_llvm_generate_elts(struct draw_llvm *llvm, struct draw_llvm_variant *varian
    if (gallivm_debug & GALLIVM_DEBUG_ASM) {
       lp_disassemble(code);
    }
+   lp_func_delete_body(variant->function_elts);
 }
 
 void
diff --git a/src/gallium/auxiliary/draw/draw_pipe.c b/src/gallium/auxiliary/draw/draw_pipe.c
index 8cd75ecf9a3..070ac803c85 100644
--- a/src/gallium/auxiliary/draw/draw_pipe.c
+++ b/src/gallium/auxiliary/draw/draw_pipe.c
@@ -169,77 +169,40 @@ static void do_triangle( struct draw_context *draw,
 /*
  * Set up macros for draw_pt_decompose.h template code.
  * This code uses vertex indexes / elements.
+ *
+ * Flags are needed by the stipple and unfilled stages.  When the two stages
+ * are active, vcache_run_extras is called and the flags are stored in the
+ * higher bits of i0.  Otherwise, flags do not matter.
  */
 
-/* emit first quad vertex as first vertex in triangles */
-#define QUAD_FIRST_PV(i0,i1,i2,i3)              \
-   do_triangle( draw,                           \
-                ( DRAW_PIPE_RESET_STIPPLE |     \
-                  DRAW_PIPE_EDGE_FLAG_0 |       \
-                  DRAW_PIPE_EDGE_FLAG_1 ),      \
-                verts + stride * (elts[i0] & ~DRAW_PIPE_FLAG_MASK),     \
-                verts + stride * (elts[i1] & ~DRAW_PIPE_FLAG_MASK),     \
-                verts + stride * (elts[i2] & ~DRAW_PIPE_FLAG_MASK));    \
-   do_triangle( draw,                           \
-                ( DRAW_PIPE_EDGE_FLAG_1 |       \
-                  DRAW_PIPE_EDGE_FLAG_2 ),      \
-                verts + stride * (elts[i0] & ~DRAW_PIPE_FLAG_MASK),     \
-                verts + stride * (elts[i2] & ~DRAW_PIPE_FLAG_MASK),     \
-                verts + stride * (elts[i3] & ~DRAW_PIPE_FLAG_MASK))
-
-/* emit last quad vertex as last vertex in triangles */
-#define QUAD_LAST_PV(i0,i1,i2,i3)               \
-   do_triangle( draw,                           \
-                ( DRAW_PIPE_RESET_STIPPLE |     \
-                  DRAW_PIPE_EDGE_FLAG_0 |       \
-                  DRAW_PIPE_EDGE_FLAG_2 ),      \
-                verts + stride * (elts[i0] & ~DRAW_PIPE_FLAG_MASK),     \
-                verts + stride * (elts[i1] & ~DRAW_PIPE_FLAG_MASK),     \
-                verts + stride * (elts[i3] & ~DRAW_PIPE_FLAG_MASK));    \
-   do_triangle( draw,                           \
-                ( DRAW_PIPE_EDGE_FLAG_0 |       \
-                  DRAW_PIPE_EDGE_FLAG_1 ),      \
-                verts + stride * (elts[i1] & ~DRAW_PIPE_FLAG_MASK),     \
-                verts + stride * (elts[i2] & ~DRAW_PIPE_FLAG_MASK),     \
-                verts + stride * (elts[i3] & ~DRAW_PIPE_FLAG_MASK))
-
-#define TRIANGLE(flags,i0,i1,i2)                                        \
-   do_triangle( draw,                                                   \
-                elts[i0],  /* flags */                                  \
-                verts + stride * (elts[i0] & ~DRAW_PIPE_FLAG_MASK),     \
-                verts + stride * (elts[i1] & ~DRAW_PIPE_FLAG_MASK),     \
-                verts + stride * (elts[i2] & ~DRAW_PIPE_FLAG_MASK) );
-
-#define LINE(flags,i0,i1)                                       \
-   do_line( draw,                                               \
-            elts[i0],                                           \
-            verts + stride * (elts[i0] & ~DRAW_PIPE_FLAG_MASK), \
-            verts + stride * (elts[i1] & ~DRAW_PIPE_FLAG_MASK) );
+#define TRIANGLE(flags,i0,i1,i2)                               \
+   do_triangle( draw,                                          \
+                i0,  /* flags */                               \
+                verts + stride * (i0 & ~DRAW_PIPE_FLAG_MASK),  \
+                verts + stride * (i1),                         \
+                verts + stride * (i2) )
+
+#define LINE(flags,i0,i1)                                      \
+   do_line( draw,                                              \
+            i0, /* flags */                                    \
+            verts + stride * (i0 & ~DRAW_PIPE_FLAG_MASK),      \
+            verts + stride * (i1) )
 
 #define POINT(i0)                               \
-   do_point( draw,                              \
-             verts + stride * (elts[i0] & ~DRAW_PIPE_FLAG_MASK) )
+   do_point( draw, verts + stride * (i0) )
 
-#define FUNC pipe_run
-#define ARGS                                    \
+#define GET_ELT(idx) (elts[idx])
+
+#define FUNC pipe_run_elts
+#define FUNC_VARS                               \
     struct draw_context *draw,                  \
     unsigned prim,                              \
     struct vertex_header *vertices,             \
     unsigned stride,                            \
-    const ushort *elts
-
-#define LOCAL_VARS                                           \
-   char *verts = (char *)vertices;                           \
-   boolean flatfirst = (draw->rasterizer->flatshade &&       \
-                        draw->rasterizer->flatshade_first);  \
-   unsigned i;                                               \
-   ushort flags
-
-#define FLUSH
+    const ushort *elts,                         \
+    unsigned count
 
 #include "draw_pt_decompose.h"
-#undef ARGS
-#undef LOCAL_VARS
 
 
 
@@ -269,14 +232,29 @@ void draw_pipeline_run( struct draw_context *draw,
         i < prim_info->primitive_count;
         start += prim_info->primitive_lengths[i], i++)
    {
-      unsigned count = prim_info->primitive_lengths[i];
-
-      pipe_run(draw,
-               prim_info->prim,
-               vert_info->verts,
-               vert_info->stride,
-               prim_info->elts + start,
-               count);
+      const unsigned count = prim_info->primitive_lengths[i];
+
+#if DEBUG
+      /* make sure none of the element indexes go outside the vertex buffer */
+      {
+         unsigned max_index = 0x0, i;
+         /* find the largest element index */
+         for (i = 0; i < count; i++) {
+            unsigned int index = (prim_info->elts[start + i]
+                                  & ~DRAW_PIPE_FLAG_MASK);
+            if (index > max_index)
+               max_index = index;
+         }
+         assert(max_index <= vert_info->count);
+      }
+#endif
+
+      pipe_run_elts(draw,
+                    prim_info->prim,
+                    vert_info->verts,
+                    vert_info->stride,
+                    prim_info->elts + start,
+                    count);
    }
 
    draw->pipeline.verts = NULL;
@@ -289,70 +267,30 @@ void draw_pipeline_run( struct draw_context *draw,
  * This code is for non-indexed (aka linear) rendering (no elts).
  */
 
-/* emit first quad vertex as first vertex in triangles */
-#define QUAD_FIRST_PV(i0,i1,i2,i3)                               \
-   do_triangle( draw,                                            \
-                ( DRAW_PIPE_RESET_STIPPLE |                      \
-                  DRAW_PIPE_EDGE_FLAG_0 |                        \
-                  DRAW_PIPE_EDGE_FLAG_1 ),                       \
-                verts + stride * ((i0) & ~DRAW_PIPE_FLAG_MASK),  \
-                verts + stride * ((i1) & ~DRAW_PIPE_FLAG_MASK),  \
-                verts + stride * ((i2) & ~DRAW_PIPE_FLAG_MASK)); \
-   do_triangle( draw,                                            \
-                ( DRAW_PIPE_EDGE_FLAG_1 |                        \
-                  DRAW_PIPE_EDGE_FLAG_2 ),                       \
-                verts + stride * ((i0) & ~DRAW_PIPE_FLAG_MASK),  \
-                verts + stride * ((i2) & ~DRAW_PIPE_FLAG_MASK),  \
-                verts + stride * ((i3) & ~DRAW_PIPE_FLAG_MASK))
-
-/* emit last quad vertex as last vertex in triangles */
-#define QUAD_LAST_PV(i0,i1,i2,i3)                                \
-   do_triangle( draw,                                            \
-                ( DRAW_PIPE_RESET_STIPPLE |                      \
-                  DRAW_PIPE_EDGE_FLAG_0 |                        \
-                  DRAW_PIPE_EDGE_FLAG_2 ),                       \
-                verts + stride * ((i0) & ~DRAW_PIPE_FLAG_MASK),  \
-                verts + stride * ((i1) & ~DRAW_PIPE_FLAG_MASK),  \
-                verts + stride * ((i3) & ~DRAW_PIPE_FLAG_MASK)); \
-   do_triangle( draw,                                            \
-                ( DRAW_PIPE_EDGE_FLAG_0 |                        \
-                  DRAW_PIPE_EDGE_FLAG_1 ),                       \
-                verts + stride * ((i1) & ~DRAW_PIPE_FLAG_MASK),  \
-                verts + stride * ((i2) & ~DRAW_PIPE_FLAG_MASK),  \
-                verts + stride * ((i3) & ~DRAW_PIPE_FLAG_MASK))
-
-#define TRIANGLE(flags,i0,i1,i2)                                 \
-   do_triangle( draw,                                            \
-                flags,  /* flags */                              \
-                verts + stride * ((i0) & ~DRAW_PIPE_FLAG_MASK),  \
-                verts + stride * ((i1) & ~DRAW_PIPE_FLAG_MASK),  \
-                verts + stride * ((i2) & ~DRAW_PIPE_FLAG_MASK))
-
-#define LINE(flags,i0,i1)                                   \
-   do_line( draw,                                           \
-            flags,                                          \
-            verts + stride * ((i0) & ~DRAW_PIPE_FLAG_MASK), \
-            verts + stride * ((i1) & ~DRAW_PIPE_FLAG_MASK))
+#define TRIANGLE(flags,i0,i1,i2)       \
+   do_triangle( draw, flags,           \
+                verts + stride * (i0), \
+                verts + stride * (i1), \
+                verts + stride * (i2) )
 
-#define POINT(i0)                               \
-   do_point( draw,                              \
-             verts + stride * ((i0) & ~DRAW_PIPE_FLAG_MASK) )
+#define LINE(flags,i0,i1)              \
+   do_line( draw, flags,               \
+            verts + stride * (i0),     \
+            verts + stride * (i1) )
 
-#define FUNC pipe_run_linear
-#define ARGS                                    \
-    struct draw_context *draw,                  \
-    unsigned prim,                              \
-    struct vertex_header *vertices,             \
-    unsigned stride
+#define POINT(i0)                      \
+   do_point( draw, verts + stride * (i0) )
 
-#define LOCAL_VARS                                           \
-   char *verts = (char *)vertices;                           \
-   boolean flatfirst = (draw->rasterizer->flatshade &&       \
-                        draw->rasterizer->flatshade_first);  \
-   unsigned i;                                               \
-   ushort flags
 
-#define FLUSH
+#define GET_ELT(idx) (idx)
+
+#define FUNC pipe_run_linear
+#define FUNC_VARS                      \
+    struct draw_context *draw,         \
+    unsigned prim,                     \
+    struct vertex_header *vertices,    \
+    unsigned stride,                   \
+    unsigned count
 
 #include "draw_pt_decompose.h"
 
@@ -378,6 +316,8 @@ void draw_pipeline_run_linear( struct draw_context *draw,
       draw->pipeline.vertex_stride = vert_info->stride;
       draw->pipeline.vertex_count = count;
 
+      assert(count <= vert_info->count);
+
       pipe_run_linear(draw,
                       prim_info->prim,
                       (struct vertex_header*)verts,
diff --git a/src/gallium/auxiliary/draw/draw_pipe_clip.c b/src/gallium/auxiliary/draw/draw_pipe_clip.c
index 1cf6ee7a7f9..8a3d499febb 100644
--- a/src/gallium/auxiliary/draw/draw_pipe_clip.c
+++ b/src/gallium/auxiliary/draw/draw_pipe_clip.c
@@ -68,8 +68,7 @@ struct clip_stage {
 };
 
 
-/* This is a bit confusing:
- */
+/** Cast wrapper */
 static INLINE struct clip_stage *clip_stage( struct draw_stage *stage )
 {
    return (struct clip_stage *)stage;
@@ -81,18 +80,22 @@ static INLINE struct clip_stage *clip_stage( struct draw_stage *stage )
 
 /* All attributes are float[4], so this is easy:
  */
-static void interp_attr( float *fdst,
+static void interp_attr( float dst[4],
 			 float t,
-			 const float *fin,
-			 const float *fout )
+			 const float in[4],
+			 const float out[4] )
 {  
-   fdst[0] = LINTERP( t, fout[0], fin[0] );
-   fdst[1] = LINTERP( t, fout[1], fin[1] );
-   fdst[2] = LINTERP( t, fout[2], fin[2] );
-   fdst[3] = LINTERP( t, fout[3], fin[3] );
+   dst[0] = LINTERP( t, out[0], in[0] );
+   dst[1] = LINTERP( t, out[1], in[1] );
+   dst[2] = LINTERP( t, out[2], in[2] );
+   dst[3] = LINTERP( t, out[3], in[3] );
 }
 
 
+/**
+ * Copy front/back, primary/secondary colors from src vertex to dst vertex.
+ * Used when flat shading.
+ */
 static void copy_colors( struct draw_stage *stage,
 			 struct vertex_header *dst,
 			 const struct vertex_header *src )
@@ -121,20 +124,17 @@ static void interp( const struct clip_stage *clip,
 
    /* Vertex header.
     */
-   {
-      dst->clipmask = 0;
-      dst->edgeflag = 0;        /* will get overwritten later */
-      dst->pad = 0;
-      dst->vertex_id = UNDEFINED_VERTEX_ID;
-   }
+   dst->clipmask = 0;
+   dst->edgeflag = 0;        /* will get overwritten later */
+   dst->pad = 0;
+   dst->vertex_id = UNDEFINED_VERTEX_ID;
 
-   /* Clip coordinates:  interpolate normally
+   /* Interpolate the clip-space coords.
     */
-   {
-      interp_attr(dst->clip, t, in->clip, out->clip);
-   }
+   interp_attr(dst->clip, t, in->clip, out->clip);
 
-   /* Do the projective divide and insert window coordinates:
+   /* Do the projective divide and viewport transformation to get
+    * new window coordinates:
     */
    {
       const float *pos = dst->clip;
diff --git a/src/gallium/auxiliary/draw/draw_private.h b/src/gallium/auxiliary/draw/draw_private.h
index 058aeedc17a..397d4bf653c 100644
--- a/src/gallium/auxiliary/draw/draw_private.h
+++ b/src/gallium/auxiliary/draw/draw_private.h
@@ -163,9 +163,11 @@ struct draw_context
          /** vertex arrays */
          const void *vbuffer[PIPE_MAX_ATTRIBS];
          
-         /** constant buffer (for vertex/geometry shader) */
+         /** constant buffers (for vertex/geometry shader) */
          const void *vs_constants[PIPE_MAX_CONSTANT_BUFFERS];
+         unsigned vs_constants_size[PIPE_MAX_CONSTANT_BUFFERS];
          const void *gs_constants[PIPE_MAX_CONSTANT_BUFFERS];
+         unsigned gs_constants_size[PIPE_MAX_CONSTANT_BUFFERS];
       } user;
 
       boolean test_fse;         /* enable FSE even though its not correct (eg for softpipe) */
@@ -198,6 +200,7 @@ struct draw_context
    struct pipe_viewport_state viewport;
    boolean identity_viewport;
 
+   /** Vertex shader state */
    struct {
       struct draw_vertex_shader *vertex_shader;
       uint num_vs_outputs;  /**< convenience, from vertex_shader */
@@ -227,6 +230,7 @@ struct draw_context
       struct translate_cache *emit_cache;
    } vs;
 
+   /** Geometry shader state */
    struct {
       struct draw_geometry_shader *geometry_shader;
       uint num_gs_outputs;  /**< convenience, from geometry_shader */
@@ -239,6 +243,7 @@ struct draw_context
       struct tgsi_sampler **samplers;
    } gs;
 
+   /** Stream output (vertex feedback) state */
    struct {
       struct pipe_stream_output_state state;
       void *buffers[PIPE_MAX_SO_BUFFERS];
diff --git a/src/gallium/auxiliary/draw/draw_pt.c b/src/gallium/auxiliary/draw/draw_pt.c
index 92d4113b4c6..248927505da 100644
--- a/src/gallium/auxiliary/draw/draw_pt.c
+++ b/src/gallium/auxiliary/draw/draw_pt.c
@@ -259,6 +259,12 @@ draw_print_arrays(struct draw_context *draw, uint prim, int start, uint count)
       for (j = 0; j < draw->pt.nr_vertex_elements; j++) {
          uint buf = draw->pt.vertex_element[j].vertex_buffer_index;
          ubyte *ptr = (ubyte *) draw->pt.user.vbuffer[buf];
+
+         if (draw->pt.vertex_element[j].instance_divisor) {
+            ii = draw->instance_id / draw->pt.vertex_element[j].instance_divisor;
+         }
+
+         ptr += draw->pt.vertex_buffer[buf].buffer_offset;
          ptr += draw->pt.vertex_buffer[buf].stride * ii;
          ptr += draw->pt.vertex_element[j].src_offset;
 
@@ -341,19 +347,22 @@ draw_arrays_instanced(struct draw_context *draw,
    unsigned reduced_prim = u_reduced_prim(mode);
    unsigned instance;
 
+   assert(instanceCount > 0);
+
    if (reduced_prim != draw->reduced_prim) {
       draw_do_flush(draw, DRAW_FLUSH_STATE_CHANGE);
       draw->reduced_prim = reduced_prim;
    }
 
    if (0)
-      draw_print_arrays(draw, mode, start, MIN2(count, 20));
-
-   if (0) {
-      unsigned int i;
       debug_printf("draw_arrays(mode=%u start=%u count=%u):\n",
                    mode, start, count);
+
+   if (0)
       tgsi_dump(draw->vs.vertex_shader->state.tokens, 0);
+
+   if (0) {
+      unsigned int i;
       debug_printf("Elements:\n");
       for (i = 0; i < draw->pt.nr_vertex_elements; i++) {
          debug_printf("  %u: src_offset=%u  inst_div=%u   vbuf=%u  format=%s\n",
@@ -374,6 +383,9 @@ draw_arrays_instanced(struct draw_context *draw,
       }
    }
 
+   if (0)
+      draw_print_arrays(draw, mode, start, MIN2(count, 20));
+
    for (instance = 0; instance < instanceCount; instance++) {
       draw->instance_id = instance + startInstance;
       draw_pt_arrays(draw, mode, start, count);
diff --git a/src/gallium/auxiliary/draw/draw_pt_decompose.h b/src/gallium/auxiliary/draw/draw_pt_decompose.h
index 52f9593d46e..3127aad7310 100644
--- a/src/gallium/auxiliary/draw/draw_pt_decompose.h
+++ b/src/gallium/auxiliary/draw/draw_pt_decompose.h
@@ -1,194 +1,7 @@
+#define LOCAL_VARS                           \
+   char *verts = (char *) vertices;          \
+   const boolean last_vertex_last =          \
+      !(draw->rasterizer->flatshade &&       \
+        draw->rasterizer->flatshade_first);
 
-
-static void FUNC( ARGS,
-                  unsigned count )
-{
-   LOCAL_VARS;
-
-   switch (prim) {
-   case PIPE_PRIM_POINTS:
-      for (i = 0; i < count; i ++) {
-	 POINT( (i + 0) );
-      }
-      break;
-
-   case PIPE_PRIM_LINES:
-      for (i = 0; i+1 < count; i += 2) {
-         LINE( DRAW_PIPE_RESET_STIPPLE,
-               (i + 0),
-               (i + 1));
-      }
-      break;
-
-   case PIPE_PRIM_LINE_LOOP:
-      if (count >= 2) {
-         flags = DRAW_PIPE_RESET_STIPPLE;
-
-         for (i = 1; i < count; i++, flags = 0) {
-            LINE( flags,
-                  (i - 1),
-                  (i ));
-         }
-
-	 LINE( flags,
-               (i - 1),
-               (0 ));
-      }
-      break;
-
-   case PIPE_PRIM_LINE_STRIP:
-      flags = DRAW_PIPE_RESET_STIPPLE;
-      for (i = 1; i < count; i++, flags = 0) {
-         LINE( flags,
-               (i - 1),
-               (i ));
-      }
-      break;
-
-   case PIPE_PRIM_TRIANGLES:
-      for (i = 0; i+2 < count; i += 3) {
-         TRIANGLE( DRAW_PIPE_RESET_STIPPLE | DRAW_PIPE_EDGE_FLAG_ALL,
-                   (i + 0),
-                   (i + 1),
-                   (i + 2 ));
-      }
-      break;
-
-   case PIPE_PRIM_TRIANGLE_STRIP:
-      if (flatfirst) {
-         for (i = 0; i+2 < count; i++) {
-            /* Emit first triangle vertex as first triangle vertex */
-            TRIANGLE( DRAW_PIPE_RESET_STIPPLE | DRAW_PIPE_EDGE_FLAG_ALL,
-                      (i + 0),
-                      (i + 1 + (i&1)),
-                      (i + 2 - (i&1)) );
-         }
-      }
-      else {
-         for (i = 0; i+2 < count; i++) {
-            /* Emit last triangle vertex as last triangle vertex */
-            TRIANGLE( DRAW_PIPE_RESET_STIPPLE | DRAW_PIPE_EDGE_FLAG_ALL,
-                      (i + 0 + (i&1)),
-                      (i + 1 - (i&1)),
-                      (i + 2 ));
-         }
-      }
-      break;
-
-   case PIPE_PRIM_TRIANGLE_FAN:
-      if (count >= 3) {
-         if (flatfirst) {
-            for (i = 0; i+2 < count; i++) {
-               TRIANGLE( DRAW_PIPE_RESET_STIPPLE | DRAW_PIPE_EDGE_FLAG_ALL,
-                         (i + 1),
-                         (i + 2),
-                         0 );
-            }
-         }
-         else {
-            for (i = 0; i+2 < count; i++) {
-               TRIANGLE( DRAW_PIPE_RESET_STIPPLE | DRAW_PIPE_EDGE_FLAG_ALL,
-                         (0),
-                         (i + 1),
-                         (i + 2 ));
-            }
-         }
-      }
-      break;
-
-
-   case PIPE_PRIM_QUADS:
-      /* GL quads don't follow provoking vertex convention */
-      if (flatfirst) {
-         for (i = 0; i+3 < count; i += 4) {
-            /* emit last quad vertex as first triangle vertex */
-            QUAD_FIRST_PV( (i + 3),
-                           (i + 0),
-                           (i + 1),
-                           (i + 2) );
-         }
-      }
-      else {
-         for (i = 0; i+3 < count; i += 4) {
-            /* emit last quad vertex as last triangle vertex */
-            QUAD_LAST_PV( (i + 0),
-                          (i + 1),
-                          (i + 2),
-                          (i + 3) );
-         }
-      }
-      break;
-
-   case PIPE_PRIM_QUAD_STRIP:
-      /* GL quad strips don't follow provoking vertex convention */
-      if (flatfirst) {
-         for (i = 0; i+3 < count; i += 2) {
-            /* emit last quad vertex as first triangle vertex */
-            QUAD_FIRST_PV( (i + 3),
-                           (i + 2),
-                           (i + 0),
-                           (i + 1) );
-
-         }
-      }
-      else {
-         for (i = 0; i+3 < count; i += 2) {
-            /* emit last quad vertex as last triangle vertex */
-            QUAD_LAST_PV( (i + 2),
-                          (i + 0),
-                          (i + 1),
-                          (i + 3) );
-         }
-      }
-      break;
-
-   case PIPE_PRIM_POLYGON:
-      /* GL polygons don't follow provoking vertex convention */
-      {
-         /* These bitflags look a little odd because we submit the
-          * vertices as (1,2,0) to satisfy flatshade requirements.
-          */
-         const ushort edge_first  = DRAW_PIPE_EDGE_FLAG_2;
-         const ushort edge_middle = DRAW_PIPE_EDGE_FLAG_0;
-         const ushort edge_last   = DRAW_PIPE_EDGE_FLAG_1;
-
-         flags = DRAW_PIPE_RESET_STIPPLE | edge_first | edge_middle;
-
-	 for (i = 0; i+2 < count; i++, flags = edge_middle) {
-
-            if (i + 3 == count)
-               flags |= edge_last;
-
-            if (flatfirst) {
-               /* emit first polygon vertex as first triangle vertex */
-               TRIANGLE( flags,
-                         (0),
-                         (i + 1),
-                         (i + 2) );
-            }
-            else {
-               /* emit first polygon vertex as last triangle vertex */
-               TRIANGLE( flags,
-                         (i + 1),
-                         (i + 2),
-                         (0));
-            }
-	 }
-      }
-      break;
-
-   default:
-      assert(0);
-      break;
-   }
-
-   FLUSH;
-}
-
-
-#undef TRIANGLE
-#undef QUAD_FIRST_PV
-#undef QUAD_LAST_PV
-#undef POINT
-#undef LINE
-#undef FUNC
+#include "draw_decompose_tmp.h"
diff --git a/src/gallium/auxiliary/draw/draw_pt_emit.c b/src/gallium/auxiliary/draw/draw_pt_emit.c
index 0229bcc7fe1..5568fbb9f88 100644
--- a/src/gallium/auxiliary/draw/draw_pt_emit.c
+++ b/src/gallium/auxiliary/draw/draw_pt_emit.c
@@ -182,6 +182,7 @@ void draw_pt_emit( struct pt_emit *emit,
 			 0,
 			 ~0);
 
+   /* fetch/translate vertex attribs to fill hw_verts[] */
    translate->run( translate,
 		   0, 
 		   vertex_count,
diff --git a/src/gallium/auxiliary/draw/draw_pt_fetch_shade_pipeline.c b/src/gallium/auxiliary/draw/draw_pt_fetch_shade_pipeline.c
index 121dfc414a4..5b16c3788e5 100644
--- a/src/gallium/auxiliary/draw/draw_pt_fetch_shade_pipeline.c
+++ b/src/gallium/auxiliary/draw/draw_pt_fetch_shade_pipeline.c
@@ -176,6 +176,7 @@ static void emit(struct pt_emit *emit,
 
 static void draw_vertex_shader_run(struct draw_vertex_shader *vshader,
                                    const void *constants[PIPE_MAX_CONSTANT_BUFFERS], 
+                                   unsigned const_size[PIPE_MAX_CONSTANT_BUFFERS],
                                    const struct draw_vertex_info *input_verts,
                                    struct draw_vertex_info *output_verts )
 {
@@ -190,6 +191,7 @@ static void draw_vertex_shader_run(struct draw_vertex_shader *vshader,
                        (const float (*)[4])input_verts->verts->data,
                        (      float (*)[4])output_verts->verts->data,
                        constants,
+                       const_size,
                        input_verts->count,
                        input_verts->vertex_size,
                        input_verts->vertex_size);
@@ -236,6 +238,7 @@ static void fetch_pipeline_generic( struct draw_pt_middle_end *middle,
    if (fpme->opt & PT_SHADE) {
       draw_vertex_shader_run(vshader,
                              draw->pt.user.vs_constants,
+                             draw->pt.user.vs_constants_size,
                              vert_info,
                              &vs_vert_info);
 
@@ -246,6 +249,7 @@ static void fetch_pipeline_generic( struct draw_pt_middle_end *middle,
    if ((fpme->opt & PT_SHADE) && gshader) {
       draw_geometry_shader_run(gshader,
                                draw->pt.user.gs_constants,
+                               draw->pt.user.gs_constants_size,
                                vert_info,
                                prim_info,
                                &gs_vert_info,
diff --git a/src/gallium/auxiliary/draw/draw_pt_fetch_shade_pipeline_llvm.c b/src/gallium/auxiliary/draw/draw_pt_fetch_shade_pipeline_llvm.c
index bc074df8c2a..4b99bee86a0 100644
--- a/src/gallium/auxiliary/draw/draw_pt_fetch_shade_pipeline_llvm.c
+++ b/src/gallium/auxiliary/draw/draw_pt_fetch_shade_pipeline_llvm.c
@@ -1,6 +1,6 @@
 /**************************************************************************
  *
- * Copyright 2010 VMWare, Inc.
+ * Copyright 2010 VMware, Inc.
  * All Rights Reserved.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
@@ -254,6 +254,7 @@ llvm_pipeline_generic( struct draw_pt_middle_end *middle,
    if ((opt & PT_SHADE) && gshader) {
       draw_geometry_shader_run(gshader,
                                draw->pt.user.gs_constants,
+                               draw->pt.user.gs_constants_size,
                                vert_info,
                                prim_info,
                                &gs_vert_info,
diff --git a/src/gallium/auxiliary/draw/draw_pt_so_emit.c b/src/gallium/auxiliary/draw/draw_pt_so_emit.c
index 5d82934889b..f7f4f24d354 100644
--- a/src/gallium/auxiliary/draw/draw_pt_so_emit.c
+++ b/src/gallium/auxiliary/draw/draw_pt_so_emit.c
@@ -218,25 +218,15 @@ static void so_tri(struct pt_so_emit *so, int i0, int i1, int i2)
 }
 
 
-#define TRIANGLE(gs,i0,i1,i2) so_tri(so,i0,i1,i2)
-#define LINE(gs,i0,i1)        so_line(so,i0,i1)
-#define POINT(gs,i0)          so_point(so,i0)
-#define FUNC so_run_linear
-#define LOCAL_VARS
+#define FUNC         so_run_linear
+#define GET_ELT(idx) (start + (idx))
 #include "draw_so_emit_tmp.h"
-#undef LOCAL_VARS
-#undef FUNC
 
 
-#define TRIANGLE(gs,i0,i1,i2) so_tri(gs,elts[i0],elts[i1],elts[i2])
-#define LINE(gs,i0,i1)        so_line(gs,elts[i0],elts[i1])
-#define POINT(gs,i0)          so_point(gs,elts[i0])
-#define FUNC so_run_elts
-#define LOCAL_VARS                         \
-   const ushort *elts = input_prims->elts;
+#define FUNC         so_run_elts
+#define LOCAL_VARS   const ushort *elts = input_prims->elts;
+#define GET_ELT(idx) (elts[start + (idx)] & ~DRAW_PIPE_FLAG_MASK)
 #include "draw_so_emit_tmp.h"
-#undef LOCAL_VARS
-#undef FUNC
 
 
 void draw_pt_so_emit( struct pt_so_emit *emit,
diff --git a/src/gallium/auxiliary/draw/draw_pt_vcache.c b/src/gallium/auxiliary/draw/draw_pt_vcache.c
index 8ef94c3163c..a848b54f7d2 100644
--- a/src/gallium/auxiliary/draw/draw_pt_vcache.c
+++ b/src/gallium/auxiliary/draw/draw_pt_vcache.c
@@ -95,7 +95,7 @@ static INLINE void
 vcache_check_flush( struct vcache_frontend *vcache )
 {
    if (vcache->draw_count + 6 >= DRAW_MAX ||
-       vcache->fetch_count + 4 >= FETCH_MAX) {
+       vcache->fetch_count + 6 >= FETCH_MAX) {
       vcache_flush( vcache );
    }
 }
@@ -180,59 +180,61 @@ vcache_point( struct vcache_frontend *vcache,
 }
 
 
-static INLINE void 
-vcache_quad( struct vcache_frontend *vcache,
-             unsigned i0,
-             unsigned i1,
-             unsigned i2,
-             unsigned i3 )
+static INLINE void
+vcache_line_adj_flags( struct vcache_frontend *vcache,
+                       unsigned flags,
+                       unsigned a0, unsigned i0, unsigned i1, unsigned a1 )
 {
-   if (vcache->draw->rasterizer->flatshade_first) {
-      /* pass last quad vertex as first triangle vertex */
-      vcache_triangle( vcache, i3, i0, i1 );
-      vcache_triangle( vcache, i3, i1, i2 );
-   }
-   else {
-      /* pass last quad vertex as last triangle vertex */
-      vcache_triangle( vcache, i0, i1, i3 );
-      vcache_triangle( vcache, i1, i2, i3 );
-   }
+   vcache_elt(vcache, a0, 0);
+   vcache_elt(vcache, i0, flags);
+   vcache_elt(vcache, i1, 0);
+   vcache_elt(vcache, a1, 0);
+   vcache_check_flush(vcache);
 }
 
 
-static INLINE void 
-vcache_ef_quad( struct vcache_frontend *vcache,
-                unsigned i0,
-                unsigned i1,
-                unsigned i2,
-                unsigned i3 )
+static INLINE void
+vcache_line_adj( struct vcache_frontend *vcache,
+                 unsigned a0, unsigned i0, unsigned i1, unsigned a1 )
 {
-   if (vcache->draw->rasterizer->flatshade_first) {
-      /* pass last quad vertex as first triangle vertex */
-      vcache_triangle_flags( vcache,
-                             ( DRAW_PIPE_RESET_STIPPLE |
-                               DRAW_PIPE_EDGE_FLAG_0 |
-                               DRAW_PIPE_EDGE_FLAG_1 ),
-                             i3, i0, i1 );
-
-      vcache_triangle_flags( vcache,
-                             ( DRAW_PIPE_EDGE_FLAG_1 |
-                               DRAW_PIPE_EDGE_FLAG_2 ),
-                             i3, i1, i2 );
-   }
-   else {
-      /* pass last quad vertex as last triangle vertex */
-      vcache_triangle_flags( vcache,
-                             ( DRAW_PIPE_RESET_STIPPLE |
-                               DRAW_PIPE_EDGE_FLAG_0 |
-                               DRAW_PIPE_EDGE_FLAG_2 ),
-                             i0, i1, i3 );
-
-      vcache_triangle_flags( vcache,
-                             ( DRAW_PIPE_EDGE_FLAG_0 |
-                               DRAW_PIPE_EDGE_FLAG_1 ),
-                             i1, i2, i3 );
-   }
+   vcache_elt(vcache, a0, 0);
+   vcache_elt(vcache, i0, 0);
+   vcache_elt(vcache, i1, 0);
+   vcache_elt(vcache, a1, 0);
+   vcache_check_flush(vcache);
+}
+
+
+static INLINE void
+vcache_triangle_adj_flags( struct vcache_frontend *vcache,
+                           unsigned flags,
+                           unsigned i0, unsigned a0,
+                           unsigned i1, unsigned a1,
+                           unsigned i2, unsigned a2 )
+{
+   vcache_elt(vcache, i0, flags);
+   vcache_elt(vcache, a0, 0);
+   vcache_elt(vcache, i1, 0);
+   vcache_elt(vcache, a1, 0);
+   vcache_elt(vcache, i2, 0);
+   vcache_elt(vcache, a2, 0);
+   vcache_check_flush(vcache);
+}
+
+
+static INLINE void
+vcache_triangle_adj( struct vcache_frontend *vcache,
+                     unsigned i0, unsigned a0,
+                     unsigned i1, unsigned a1,
+                     unsigned i2, unsigned a2 )
+{
+   vcache_elt(vcache, i0, 0);
+   vcache_elt(vcache, a0, 0);
+   vcache_elt(vcache, i1, 0);
+   vcache_elt(vcache, a1, 0);
+   vcache_elt(vcache, i2, 0);
+   vcache_elt(vcache, a2, 0);
+   vcache_check_flush(vcache);
 }
 
 
@@ -240,17 +242,23 @@ vcache_ef_quad( struct vcache_frontend *vcache,
  * this.  The two paths aren't too different though - it may be
  * possible to reunify them.
  */
-#define TRIANGLE(vc,flags,i0,i1,i2) vcache_triangle_flags(vc,flags,i0,i1,i2)
-#define QUAD(vc,i0,i1,i2,i3)        vcache_ef_quad(vc,i0,i1,i2,i3)
-#define LINE(vc,flags,i0,i1)        vcache_line_flags(vc,flags,i0,i1)
-#define POINT(vc,i0)                vcache_point(vc,i0)
+#define TRIANGLE(flags,i0,i1,i2) vcache_triangle_flags(vcache,flags,i0,i1,i2)
+#define LINE(flags,i0,i1)        vcache_line_flags(vcache,flags,i0,i1)
+#define POINT(i0)                vcache_point(vcache,i0)
+#define LINE_ADJ(flags,a0,i0,i1,a1) \
+   vcache_line_adj_flags(vcache,flags,a0,i0,i1,a1)
+#define TRIANGLE_ADJ(flags,i0,a0,i1,a1,i2,a2) \
+   vcache_triangle_adj_flags(vcache,flags,i0,a0,i1,a1,i2,a2)
 #define FUNC vcache_run_extras
 #include "draw_pt_vcache_tmp.h"
 
-#define TRIANGLE(vc,flags,i0,i1,i2) vcache_triangle(vc,i0,i1,i2)
-#define QUAD(vc,i0,i1,i2,i3)        vcache_quad(vc,i0,i1,i2,i3)
-#define LINE(vc,flags,i0,i1)        vcache_line(vc,i0,i1)
-#define POINT(vc,i0)                vcache_point(vc,i0)
+#define TRIANGLE(flags,i0,i1,i2) vcache_triangle(vcache,i0,i1,i2)
+#define LINE(flags,i0,i1)        vcache_line(vcache,i0,i1)
+#define POINT(i0)                vcache_point(vcache,i0)
+#define LINE_ADJ(flags,a0,i0,i1,a1) \
+   vcache_line_adj(vcache,a0,i0,i1,a1)
+#define TRIANGLE_ADJ(flags,i0,a0,i1,a1,i2,a2) \
+   vcache_triangle_adj(vcache,i0,a0,i1,a1,i2,a2)
 #define FUNC vcache_run
 #include "draw_pt_vcache_tmp.h"
 
@@ -339,6 +347,25 @@ format_from_get_elt( pt_elt_func get_elt )
 #endif
 
 
+/**
+ * Check if any vertex attributes use instance divisors.
+ * Note that instance divisors complicate vertex fetching so we need
+ * to take the vcache path when they're in use.
+ */
+static boolean
+any_instance_divisors(const struct draw_context *draw)
+{
+   uint i;
+
+   for (i = 0; i < draw->pt.nr_vertex_elements; i++) {
+      uint div = draw->pt.vertex_element[i].instance_divisor;
+      if (div)
+         return TRUE;
+   }
+   return FALSE;
+}
+
+
 static INLINE void 
 vcache_check_run( struct draw_pt_front_end *frontend, 
                   pt_elt_func get_elt,
@@ -382,6 +409,9 @@ vcache_check_run( struct draw_pt_front_end *frontend,
    if (max_index >= (unsigned) DRAW_PIPE_MAX_VERTICES)
       goto fail;
 
+   if (any_instance_divisors(draw))
+      goto fail;
+
    fetch_count = max_index + 1 - min_index;
 
    if (0)
@@ -518,7 +548,18 @@ vcache_prepare( struct draw_pt_front_end *frontend,
     * which is a separate issue.
     */
    vcache->input_prim = in_prim;
-   vcache->output_prim = u_reduced_prim(in_prim);
+   switch (in_prim) {
+   case PIPE_PRIM_LINES_ADJACENCY:
+   case PIPE_PRIM_LINE_STRIP_ADJACENCY:
+      vcache->output_prim = PIPE_PRIM_LINES_ADJACENCY;
+      break;
+   case PIPE_PRIM_TRIANGLES_ADJACENCY:
+   case PIPE_PRIM_TRIANGLE_STRIP_ADJACENCY:
+      vcache->output_prim = PIPE_PRIM_TRIANGLES_ADJACENCY;
+      break;
+   default:
+      vcache->output_prim = u_reduced_prim(in_prim);
+   }
 
    vcache->middle = middle;
    vcache->opt = opt;
diff --git a/src/gallium/auxiliary/draw/draw_pt_vcache_tmp.h b/src/gallium/auxiliary/draw/draw_pt_vcache_tmp.h
index dac68ad4398..1a3748d5f0b 100644
--- a/src/gallium/auxiliary/draw/draw_pt_vcache_tmp.h
+++ b/src/gallium/auxiliary/draw/draw_pt_vcache_tmp.h
@@ -1,198 +1,19 @@
+#define FUNC_VARS                      \
+   struct draw_pt_front_end *frontend, \
+   pt_elt_func get_elt,                \
+   const void *elts,                   \
+   int elt_bias,                       \
+   unsigned count
 
+#define LOCAL_VARS \
+   struct vcache_frontend *vcache = (struct vcache_frontend *) frontend;   \
+   struct draw_context *draw = vcache->draw;                               \
+   const unsigned prim = vcache->input_prim;                               \
+   const boolean last_vertex_last = !(draw->rasterizer->flatshade &&       \
+                                      draw->rasterizer->flatshade_first);
 
-static void FUNC( struct draw_pt_front_end *frontend, 
-                  pt_elt_func get_elt,
-                  const void *elts,
-                  int elt_bias,
-                  unsigned count )
-{
-   struct vcache_frontend *vcache = (struct vcache_frontend *)frontend;
-   struct draw_context *draw = vcache->draw;
+#define GET_ELT(idx) (get_elt(elts, idx) + elt_bias)
 
-   boolean flatfirst = (draw->rasterizer->flatshade && 
-                        draw->rasterizer->flatshade_first);
-   unsigned i;
-   ushort flags;
+#define FUNC_EXIT do { vcache_flush(vcache); } while (0)
 
-   if (0) debug_printf("%s %d\n", __FUNCTION__, count);
-
-
-   switch (vcache->input_prim) {
-   case PIPE_PRIM_POINTS:
-      for (i = 0; i < count; i ++) {
-	 POINT( vcache,
-                get_elt(elts, i + 0) + elt_bias );
-      }
-      break;
-
-   case PIPE_PRIM_LINES:
-      for (i = 0; i+1 < count; i += 2) {
-         LINE( vcache, 
-               DRAW_PIPE_RESET_STIPPLE,
-               get_elt(elts, i + 0) + elt_bias,
-               get_elt(elts, i + 1) + elt_bias);
-      }
-      break;
-
-   case PIPE_PRIM_LINE_LOOP:  
-      if (count >= 2) {
-         flags = DRAW_PIPE_RESET_STIPPLE;
-
-         for (i = 1; i < count; i++, flags = 0) {
-            LINE( vcache, 
-                  flags,
-                  get_elt(elts, i - 1) + elt_bias,
-                  get_elt(elts, i ) + elt_bias);
-         }
-
-	 LINE( vcache, 
-               flags,
-               get_elt(elts, i - 1) + elt_bias,
-               get_elt(elts, 0 ) + elt_bias);
-      }
-      break;
-
-   case PIPE_PRIM_LINE_STRIP:
-      flags = DRAW_PIPE_RESET_STIPPLE;
-      for (i = 1; i < count; i++, flags = 0) {
-         LINE( vcache, 
-               flags,
-               get_elt(elts, i - 1) + elt_bias,
-               get_elt(elts, i ) + elt_bias);
-      }
-      break;
-
-   case PIPE_PRIM_TRIANGLES:
-      for (i = 0; i+2 < count; i += 3) {
-         TRIANGLE( vcache,
-                   DRAW_PIPE_RESET_STIPPLE | DRAW_PIPE_EDGE_FLAG_ALL, 
-                   get_elt(elts, i + 0) + elt_bias,
-                   get_elt(elts, i + 1) + elt_bias,
-                   get_elt(elts, i + 2 ) + elt_bias);
-      }
-      break;
-
-   case PIPE_PRIM_TRIANGLE_STRIP:
-      if (flatfirst) {
-         for (i = 0; i+2 < count; i++) {
-            TRIANGLE( vcache,
-                      DRAW_PIPE_RESET_STIPPLE | DRAW_PIPE_EDGE_FLAG_ALL, 
-                      get_elt(elts, i + 0) + elt_bias,
-                      get_elt(elts, i + 1 + (i&1)) + elt_bias,
-                      get_elt(elts, i + 2 - (i&1)) + elt_bias);
-         }
-      }
-      else {
-         for (i = 0; i+2 < count; i++) {
-            TRIANGLE( vcache,
-                      DRAW_PIPE_RESET_STIPPLE | DRAW_PIPE_EDGE_FLAG_ALL, 
-                      get_elt(elts, i + 0 + (i&1)) + elt_bias,
-                      get_elt(elts, i + 1 - (i&1)) + elt_bias,
-                      get_elt(elts, i + 2 ) + elt_bias);
-         }
-      }
-      break;
-
-   case PIPE_PRIM_TRIANGLE_FAN:
-      if (count >= 3) {
-         if (flatfirst) {
-            for (i = 0; i+2 < count; i++) {
-               TRIANGLE( vcache,
-                         DRAW_PIPE_RESET_STIPPLE | DRAW_PIPE_EDGE_FLAG_ALL, 
-                         get_elt(elts, i + 1) + elt_bias,
-                         get_elt(elts, i + 2) + elt_bias,
-                         get_elt(elts, 0 ) + elt_bias);
-            }
-         }
-         else {
-            for (i = 0; i+2 < count; i++) {
-               TRIANGLE( vcache,
-                         DRAW_PIPE_RESET_STIPPLE | DRAW_PIPE_EDGE_FLAG_ALL, 
-                         get_elt(elts, 0) + elt_bias,
-                         get_elt(elts, i + 1) + elt_bias,
-                         get_elt(elts, i + 2 ) + elt_bias);
-            }
-         }
-      }
-      break;
-
-
-   case PIPE_PRIM_QUADS:
-      for (i = 0; i+3 < count; i += 4) {
-         QUAD( vcache,
-               get_elt(elts, i + 0) + elt_bias,
-               get_elt(elts, i + 1) + elt_bias,
-               get_elt(elts, i + 2) + elt_bias,
-               get_elt(elts, i + 3) + elt_bias );
-      }
-      break;
-
-   case PIPE_PRIM_QUAD_STRIP:
-      for (i = 0; i+3 < count; i += 2) {
-         QUAD( vcache,
-               get_elt(elts, i + 2) + elt_bias,
-               get_elt(elts, i + 0) + elt_bias,
-               get_elt(elts, i + 1) + elt_bias,
-               get_elt(elts, i + 3) + elt_bias );
-      }
-      break;
-
-   case PIPE_PRIM_POLYGON:
-      {
-         /* These bitflags look a little odd because we submit the
-          * vertices as (1,2,0) to satisfy flatshade requirements.  
-          */
-         ushort edge_next, edge_finish;
-
-         if (flatfirst) {
-            flags = (DRAW_PIPE_RESET_STIPPLE |
-                     DRAW_PIPE_EDGE_FLAG_1 |
-                     DRAW_PIPE_EDGE_FLAG_2);
-            edge_next = DRAW_PIPE_EDGE_FLAG_2;
-            edge_finish = DRAW_PIPE_EDGE_FLAG_0;
-         }
-         else {
-            flags = (DRAW_PIPE_RESET_STIPPLE |
-                     DRAW_PIPE_EDGE_FLAG_2 |
-                     DRAW_PIPE_EDGE_FLAG_0);
-            edge_next = DRAW_PIPE_EDGE_FLAG_0;
-            edge_finish = DRAW_PIPE_EDGE_FLAG_1;
-         }
-
-	 for (i = 0; i+2 < count; i++, flags = edge_next) {
-
-            if (i + 3 == count)
-               flags |= edge_finish;
-
-            if (flatfirst) {
-               TRIANGLE( vcache,
-                         flags,
-                         get_elt(elts, 0) + elt_bias,
-                         get_elt(elts, i + 1) + elt_bias,
-                         get_elt(elts, i + 2) + elt_bias );
-            }
-            else {
-               TRIANGLE( vcache,
-                         flags,
-                         get_elt(elts, i + 1) + elt_bias,
-                         get_elt(elts, i + 2) + elt_bias,
-                         get_elt(elts, 0) + elt_bias);
-            }
-	 }
-      }
-      break;
-
-   default:
-      assert(0);
-      break;
-   }
-   
-   vcache_flush( vcache );
-}
-
-
-#undef TRIANGLE
-#undef QUAD
-#undef POINT
-#undef LINE
-#undef FUNC
+#include "draw_decompose_tmp.h"
diff --git a/src/gallium/auxiliary/draw/draw_so_emit_tmp.h b/src/gallium/auxiliary/draw/draw_so_emit_tmp.h
index 01212a8e536..6d8937a0b41 100644
--- a/src/gallium/auxiliary/draw/draw_so_emit_tmp.h
+++ b/src/gallium/auxiliary/draw/draw_so_emit_tmp.h
@@ -1,123 +1,33 @@
-
-static void FUNC( struct pt_so_emit *so,
-                  const struct draw_prim_info *input_prims,
-                  const struct draw_vertex_info *input_verts,
-                  unsigned start,
-                  unsigned count)
-{
-   struct draw_context *draw = so->draw;
-
-   boolean flatfirst = (draw->rasterizer->flatshade &&
-                        draw->rasterizer->flatshade_first);
-   unsigned i;
-   LOCAL_VARS
-
-   if (0) debug_printf("%s %d\n", __FUNCTION__, count);
-
-   debug_assert(input_prims->primitive_count == 1);
-
-   switch (input_prims->prim) {
-   case PIPE_PRIM_POINTS:
-      for (i = 0; i < count; i++) {
-	 POINT( so, start + i + 0 );
-      }
-      break;
-
-   case PIPE_PRIM_LINES:
-      for (i = 0; i+1 < count; i += 2) {
-         LINE( so , start + i + 0 , start + i + 1 );
-      }
-      break;
-
-   case PIPE_PRIM_LINE_LOOP:
-      if (count >= 2) {
-
-         for (i = 1; i < count; i++) {
-            LINE( so, start + i - 1, start + i );
-         }
-
-	 LINE( so, start + i - 1, start );
-      }
-      break;
-
-   case PIPE_PRIM_LINE_STRIP:
-      for (i = 1; i < count; i++) {
-         LINE( so, start + i - 1, start + i );
-      }
-      break;
-
-   case PIPE_PRIM_TRIANGLES:
-      for (i = 0; i+2 < count; i += 3) {
-         TRIANGLE( so, start + i + 0, start + i + 1, start + i + 2 );
-      }
-      break;
-
-   case PIPE_PRIM_TRIANGLE_STRIP:
-      if (flatfirst) {
-         for (i = 0; i+2 < count; i++) {
-            TRIANGLE( so,
-                      start + i + 0,
-                      start + i + 1 + (i&1),
-                      start + i + 2 - (i&1) );
-         }
-      }
-      else {
-         for (i = 0; i+2 < count; i++) {
-            TRIANGLE( so,
-                      start + i + 0 + (i&1),
-                      start + i + 1 - (i&1),
-                      start + i + 2 );
-         }
-      }
-      break;
-
-   case PIPE_PRIM_TRIANGLE_FAN:
-      if (count >= 3) {
-         if (flatfirst) {
-            for (i = 0; i+2 < count; i++) {
-               TRIANGLE( so,
-                         start + i + 1,
-                         start + i + 2,
-                         start );
-            }
-         }
-         else {
-            for (i = 0; i+2 < count; i++) {
-               TRIANGLE( so,
-                         start,
-                         start + i + 1,
-                         start + i + 2 );
-            }
-         }
-      }
-      break;
-
-   case PIPE_PRIM_POLYGON:
-      {
-         /* These bitflags look a little odd because we submit the
-          * vertices as (1,2,0) to satisfy flatshade requirements.
-          */
-
-	 for (i = 0; i+2 < count; i++) {
-
-            if (flatfirst) {
-               TRIANGLE( so, start + 0, start + i + 1, start + i + 2 );
-            }
-            else {
-               TRIANGLE( so, start + i + 1, start + i + 2, start + 0 );
-            }
-	 }
-      }
-      break;
-
-   default:
-      debug_assert(!"Unsupported primitive in stream output");
-      break;
-   }
-}
-
-
-#undef TRIANGLE
-#undef POINT
-#undef LINE
-#undef FUNC
+#define FUNC_VARS                               \
+   struct pt_so_emit *so,                       \
+   const struct draw_prim_info *input_prims,    \
+   const struct draw_vertex_info *input_verts,  \
+   unsigned start,                              \
+   unsigned count
+
+#define FUNC_ENTER                                                \
+   /* declare more local vars */                                  \
+   struct draw_context *draw = so->draw;                          \
+   const unsigned prim = input_prims->prim;                       \
+   const boolean last_vertex_last =                               \
+      !(draw->rasterizer->flatshade &&                            \
+        draw->rasterizer->flatshade_first);                       \
+   do {                                                           \
+      debug_assert(input_prims->primitive_count == 1);            \
+      switch (prim) {                                             \
+      case PIPE_PRIM_LINES_ADJACENCY:                             \
+      case PIPE_PRIM_LINE_STRIP_ADJACENCY:                        \
+      case PIPE_PRIM_TRIANGLES_ADJACENCY:                         \
+      case PIPE_PRIM_TRIANGLE_STRIP_ADJACENCY:                    \
+         debug_assert(!"unexpected primitive type in stream output"); \
+         return;                                                  \
+      default:                                                    \
+         break;                                                   \
+      }                                                           \
+   } while (0)                                                    \
+
+#define POINT(i0)                so_point(so,i0)
+#define LINE(flags,i0,i1)        so_line(so,i0,i1)
+#define TRIANGLE(flags,i0,i1,i2) so_tri(so,i0,i1,i2)
+
+#include "draw_decompose_tmp.h"
diff --git a/src/gallium/auxiliary/draw/draw_vertex.h b/src/gallium/auxiliary/draw/draw_vertex.h
index 3af31ffe126..e63cf5f4f98 100644
--- a/src/gallium/auxiliary/draw/draw_vertex.h
+++ b/src/gallium/auxiliary/draw/draw_vertex.h
@@ -166,7 +166,7 @@ static INLINE enum pipe_format draw_translate_vinfo_format(enum attrib_emit emit
    }
 }
 
-static INLINE enum attrib_emit draw_translate_vinfo_size(enum attrib_emit emit)
+static INLINE unsigned draw_translate_vinfo_size(enum attrib_emit emit)
 {
    switch (emit) {
    case EMIT_OMIT:
diff --git a/src/gallium/auxiliary/draw/draw_vs.c b/src/gallium/auxiliary/draw/draw_vs.c
index 57ea63fc060..fb665b08fff 100644
--- a/src/gallium/auxiliary/draw/draw_vs.c
+++ b/src/gallium/auxiliary/draw/draw_vs.c
@@ -48,18 +48,30 @@
 
 DEBUG_GET_ONCE_BOOL_OPTION(gallium_dump_vs, "GALLIUM_DUMP_VS", FALSE)
 
+
+/**
+ * Set a vertex shader constant buffer.
+ * \param slot  which constant buffer in [0, PIPE_MAX_CONSTANT_BUFFERS-1]
+ * \param constants  the mapped buffer
+ * \param size  size of buffer in bytes
+ */
 void
 draw_vs_set_constants(struct draw_context *draw,
                       unsigned slot,
                       const void *constants,
                       unsigned size)
 {
-   if (((uintptr_t)constants) & 0xf) {
+   const int alignment = 16;
+
+   /* check if buffer is 16-byte aligned */
+   if (((uintptr_t)constants) & (alignment - 1)) {
+      /* if not, copy the constants into a new, 16-byte aligned buffer */
       if (size > draw->vs.const_storage_size[slot]) {
          if (draw->vs.aligned_constant_storage[slot]) {
             align_free((void *)draw->vs.aligned_constant_storage[slot]);
          }
-         draw->vs.aligned_constant_storage[slot] = align_malloc(size, 16);
+         draw->vs.aligned_constant_storage[slot] =
+            align_malloc(size, alignment);
       }
       assert(constants);
       memcpy((void *)draw->vs.aligned_constant_storage[slot],
diff --git a/src/gallium/auxiliary/draw/draw_vs.h b/src/gallium/auxiliary/draw/draw_vs.h
index a7319945234..f9a038788fb 100644
--- a/src/gallium/auxiliary/draw/draw_vs.h
+++ b/src/gallium/auxiliary/draw/draw_vs.h
@@ -133,7 +133,8 @@ struct draw_vertex_shader {
    void (*run_linear)( struct draw_vertex_shader *shader,
 		       const float (*input)[4],
 		       float (*output)[4],
-                      const void *constants[PIPE_MAX_CONSTANT_BUFFERS],
+                       const void *constants[PIPE_MAX_CONSTANT_BUFFERS],
+                       const unsigned const_size[PIPE_MAX_CONSTANT_BUFFERS],
 		       unsigned count,
 		       unsigned input_stride,
 		       unsigned output_stride );
diff --git a/src/gallium/auxiliary/draw/draw_vs_exec.c b/src/gallium/auxiliary/draw/draw_vs_exec.c
index bc34d390dae..dab3eb1ca8e 100644
--- a/src/gallium/auxiliary/draw/draw_vs_exec.c
+++ b/src/gallium/auxiliary/draw/draw_vs_exec.c
@@ -85,7 +85,8 @@ static void
 vs_exec_run_linear( struct draw_vertex_shader *shader,
 		    const float (*input)[4],
 		    float (*output)[4],
-                   const void *constants[PIPE_MAX_CONSTANT_BUFFERS],
+                    const void *constants[PIPE_MAX_CONSTANT_BUFFERS],
+                    const unsigned const_size[PIPE_MAX_CONSTANT_BUFFERS],
 		    unsigned count,
 		    unsigned input_stride,
 		    unsigned output_stride )
@@ -95,9 +96,8 @@ vs_exec_run_linear( struct draw_vertex_shader *shader,
    unsigned int i, j;
    unsigned slot;
 
-   for (i = 0; i < PIPE_MAX_CONSTANT_BUFFERS; i++) {
-      machine->Consts[i] = constants[i];
-   }
+   tgsi_exec_set_constant_buffers(machine, PIPE_MAX_CONSTANT_BUFFERS,
+                                  constants, const_size);
 
    for (i = 0; i < count; i += MAX_TGSI_VERTICES) {
       unsigned int max_vertices = MIN2(MAX_TGSI_VERTICES, count - i);
diff --git a/src/gallium/auxiliary/draw/draw_vs_llvm.c b/src/gallium/auxiliary/draw/draw_vs_llvm.c
index 6c13df79132..d13ad24fff0 100644
--- a/src/gallium/auxiliary/draw/draw_vs_llvm.c
+++ b/src/gallium/auxiliary/draw/draw_vs_llvm.c
@@ -49,6 +49,7 @@ vs_llvm_run_linear( struct draw_vertex_shader *shader,
 		    const float (*input)[4],
 		    float (*output)[4],
                     const void *constants[PIPE_MAX_CONSTANT_BUFFERS],
+                    const unsigned constants_size[PIPE_MAX_CONSTANT_BUFFERS],
 		    unsigned count,
 		    unsigned input_stride,
 		    unsigned output_stride )
diff --git a/src/gallium/auxiliary/draw/draw_vs_sse.c b/src/gallium/auxiliary/draw/draw_vs_sse.c
index 14c95082a9d..0b0c6077c6f 100644
--- a/src/gallium/auxiliary/draw/draw_vs_sse.c
+++ b/src/gallium/auxiliary/draw/draw_vs_sse.c
@@ -84,6 +84,7 @@ vs_sse_run_linear( struct draw_vertex_shader *base,
 		   const float (*input)[4],
 		   float (*output)[4],
                   const void *constants[PIPE_MAX_CONSTANT_BUFFERS],
+		  const unsigned const_size[PIPE_MAX_CONSTANT_BUFFERS],
 		   unsigned count,
 		   unsigned input_stride,
 		   unsigned output_stride )
diff --git a/src/gallium/auxiliary/draw/draw_vs_varient.c b/src/gallium/auxiliary/draw/draw_vs_varient.c
index 6eb26927f27..eacd1601877 100644
--- a/src/gallium/auxiliary/draw/draw_vs_varient.c
+++ b/src/gallium/auxiliary/draw/draw_vs_varient.c
@@ -149,7 +149,8 @@ static void PIPE_CDECL vsvg_run_elts( struct draw_vs_varient *varient,
    vsvg->base.vs->run_linear( vsvg->base.vs, 
                               temp_buffer,
                               temp_buffer,
-                             vsvg->base.vs->draw->pt.user.vs_constants,
+                              vsvg->base.vs->draw->pt.user.vs_constants,
+                              vsvg->base.vs->draw->pt.user.vs_constants_size,
                               count,
                               temp_vertex_stride, 
                               temp_vertex_stride);
@@ -214,7 +215,8 @@ static void PIPE_CDECL vsvg_run_linear( struct draw_vs_varient *varient,
    vsvg->base.vs->run_linear( vsvg->base.vs, 
                               temp_buffer,
                               temp_buffer,
-                             vsvg->base.vs->draw->pt.user.vs_constants,
+                              vsvg->base.vs->draw->pt.user.vs_constants,
+                              vsvg->base.vs->draw->pt.user.vs_constants_size,
                               count,
                               temp_vertex_stride, 
                               temp_vertex_stride);
diff --git a/src/gallium/auxiliary/gallivm/lp_bld_assert.c b/src/gallium/auxiliary/gallivm/lp_bld_assert.c
new file mode 100644
index 00000000000..f2ebd868a8d
--- /dev/null
+++ b/src/gallium/auxiliary/gallivm/lp_bld_assert.c
@@ -0,0 +1,101 @@
+/**************************************************************************
+ *
+ * Copyright 2010 VMware, Inc.
+ * All Rights Reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sub license, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the
+ * next paragraph) shall be included in all copies or substantial portions
+ * of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
+ * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
+ * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ *
+ **************************************************************************/
+
+#include "util/u_debug.h"
+#include "util/u_memory.h"
+#include "lp_bld_assert.h"
+#include "lp_bld_init.h"
+#include "lp_bld_printf.h"
+
+
+/**
+ * A call to lp_build_assert() will build a function call to this function.
+ */
+static void
+lp_assert(int condition, const char *msg)
+{
+   if (!condition) {
+      debug_printf("LLVM assertion '%s' failed!\n", msg);
+      assert(condition);
+   }
+}
+
+
+
+/**
+ * lp_build_assert.
+ *
+ * Build an assertion in LLVM IR by building a function call to the
+ * lp_assert() function above.
+ *
+ * \param condition should be an 'i1' or 'i32' value
+ * \param msg  a string to print if the assertion fails.
+ */
+LLVMValueRef
+lp_build_assert(LLVMBuilderRef builder, LLVMValueRef condition,
+                const char *msg)
+{
+   LLVMModuleRef module;
+   LLVMTypeRef arg_types[2];
+   LLVMValueRef msg_string, assert_func, params[2], r;
+
+   module = LLVMGetGlobalParent(LLVMGetBasicBlockParent(
+                            LLVMGetInsertBlock(builder)));
+
+   msg_string = lp_build_const_string_variable(module, msg, strlen(msg) + 1);
+
+   arg_types[0] = LLVMInt32Type();
+   arg_types[1] = LLVMPointerType(LLVMInt8Type(), 0);
+
+   /* lookup the lp_assert function */
+   assert_func = LLVMGetNamedFunction(module, "lp_assert");
+
+   /* Create the assertion function if not found */
+   if (!assert_func) {
+      LLVMTypeRef func_type =
+         LLVMFunctionType(LLVMVoidType(), arg_types, 2, 0);
+
+      assert_func = LLVMAddFunction(module, "lp_assert", func_type);
+      LLVMSetFunctionCallConv(assert_func, LLVMCCallConv);
+      LLVMSetLinkage(assert_func, LLVMExternalLinkage);
+      LLVMAddGlobalMapping(lp_build_engine, assert_func,
+                           func_to_pointer((func_pointer)lp_assert));
+   }
+   assert(assert_func);
+
+   /* build function call param list */
+   params[0] = LLVMBuildZExt(builder, condition, arg_types[0], "");
+   params[1] = LLVMBuildBitCast(builder, msg_string, arg_types[1], "");
+
+   /* check arg types */
+   assert(LLVMTypeOf(params[0]) == arg_types[0]);
+   assert(LLVMTypeOf(params[1]) == arg_types[1]);
+
+   r = LLVMBuildCall(builder, assert_func, params, 2, "");
+
+   return r;
+}
diff --git a/src/gallium/auxiliary/gallivm/lp_bld_assert.h b/src/gallium/auxiliary/gallivm/lp_bld_assert.h
new file mode 100644
index 00000000000..ddd879dc2c6
--- /dev/null
+++ b/src/gallium/auxiliary/gallivm/lp_bld_assert.h
@@ -0,0 +1,41 @@
+/**************************************************************************
+ *
+ * Copyright 2010 VMware, Inc.
+ * All Rights Reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sub license, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the
+ * next paragraph) shall be included in all copies or substantial portions
+ * of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
+ * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
+ * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ *
+ **************************************************************************/
+
+#ifndef LP_BLD_ASSERT_H
+#define LP_BLD_ASSERT_H
+
+
+#include "lp_bld.h"
+
+
+LLVMValueRef
+lp_build_assert(LLVMBuilderRef builder, LLVMValueRef condition,
+                const char *msg);
+
+
+#endif
+
diff --git a/src/gallium/auxiliary/gallivm/lp_bld_init.c b/src/gallium/auxiliary/gallivm/lp_bld_init.c
index 69353dea09e..ef0888079c7 100644
--- a/src/gallium/auxiliary/gallivm/lp_bld_init.c
+++ b/src/gallium/auxiliary/gallivm/lp_bld_init.c
@@ -45,6 +45,8 @@ static const struct debug_named_value lp_bld_debug_flags[] = {
    { "nopt",   GALLIVM_DEBUG_NO_OPT, NULL },
    DEBUG_NAMED_VALUE_END
 };
+
+DEBUG_GET_ONCE_FLAGS_OPTION(gallivm_debug, "GALLIVM_DEBUG", lp_bld_debug_flags, 0);
 #endif
 
 
@@ -89,7 +91,7 @@ void
 lp_build_init(void)
 {
 #ifdef DEBUG
-   gallivm_debug = debug_get_flags_option("GALLIVM_DEBUG", lp_bld_debug_flags, 0 );
+   gallivm_debug = debug_get_option_gallivm_debug();
 #endif
 
    lp_set_target_options();
diff --git a/src/gallium/auxiliary/gallivm/lp_bld_init.h b/src/gallium/auxiliary/gallivm/lp_bld_init.h
index a32ced9b4c3..f26fdac4663 100644
--- a/src/gallium/auxiliary/gallivm/lp_bld_init.h
+++ b/src/gallium/auxiliary/gallivm/lp_bld_init.h
@@ -44,5 +44,7 @@ extern LLVMPassManagerRef lp_build_pass;
 void
 lp_build_init(void);
 
+extern void
+lp_func_delete_body(LLVMValueRef func);
 
 #endif /* !LP_BLD_INIT_H */
diff --git a/src/gallium/auxiliary/gallivm/lp_bld_logic.c b/src/gallium/auxiliary/gallivm/lp_bld_logic.c
index 39854e43b19..ab4ddb81c40 100644
--- a/src/gallium/auxiliary/gallivm/lp_bld_logic.c
+++ b/src/gallium/auxiliary/gallivm/lp_bld_logic.c
@@ -363,9 +363,52 @@ lp_build_cmp(struct lp_build_context *bld,
 
 
 /**
+ * Return (mask & a) | (~mask & b);
+ */
+LLVMValueRef
+lp_build_select_bitwise(struct lp_build_context *bld,
+                        LLVMValueRef mask,
+                        LLVMValueRef a,
+                        LLVMValueRef b)
+{
+   struct lp_type type = bld->type;
+   LLVMValueRef res;
+
+   if (a == b) {
+      return a;
+   }
+
+   if(type.floating) {
+      LLVMTypeRef int_vec_type = lp_build_int_vec_type(type);
+      a = LLVMBuildBitCast(bld->builder, a, int_vec_type, "");
+      b = LLVMBuildBitCast(bld->builder, b, int_vec_type, "");
+   }
+
+   a = LLVMBuildAnd(bld->builder, a, mask, "");
+
+   /* This often gets translated to PANDN, but sometimes the NOT is
+    * pre-computed and stored in another constant. The best strategy depends
+    * on available registers, so it is not a big deal -- hopefully LLVM does
+    * the right decision attending the rest of the program.
+    */
+   b = LLVMBuildAnd(bld->builder, b, LLVMBuildNot(bld->builder, mask, ""), "");
+
+   res = LLVMBuildOr(bld->builder, a, b, "");
+
+   if(type.floating) {
+      LLVMTypeRef vec_type = lp_build_vec_type(type);
+      res = LLVMBuildBitCast(bld->builder, res, vec_type, "");
+   }
+
+   return res;
+}
+
+
+/**
  * Return mask ? a : b;
  *
- * mask is a bitwise mask, composed of 0 or ~0 for each element.
+ * mask is a bitwise mask, composed of 0 or ~0 for each element. Any other value
+ * will yield unpredictable results.
  */
 LLVMValueRef
 lp_build_select(struct lp_build_context *bld,
@@ -424,27 +467,7 @@ lp_build_select(struct lp_build_context *bld,
       }
    }
    else {
-      if(type.floating) {
-         LLVMTypeRef int_vec_type = lp_build_int_vec_type(type);
-         a = LLVMBuildBitCast(bld->builder, a, int_vec_type, "");
-         b = LLVMBuildBitCast(bld->builder, b, int_vec_type, "");
-      }
-
-      a = LLVMBuildAnd(bld->builder, a, mask, "");
-
-      /* This often gets translated to PANDN, but sometimes the NOT is
-       * pre-computed and stored in another constant. The best strategy depends
-       * on available registers, so it is not a big deal -- hopefully LLVM does
-       * the right decision attending the rest of the program.
-       */
-      b = LLVMBuildAnd(bld->builder, b, LLVMBuildNot(bld->builder, mask, ""), "");
-
-      res = LLVMBuildOr(bld->builder, a, b, "");
-
-      if(type.floating) {
-         LLVMTypeRef vec_type = lp_build_vec_type(type);
-         res = LLVMBuildBitCast(bld->builder, res, vec_type, "");
-      }
+      res = lp_build_select_bitwise(bld, mask, a, b);
    }
 
    return res;
diff --git a/src/gallium/auxiliary/gallivm/lp_bld_logic.h b/src/gallium/auxiliary/gallivm/lp_bld_logic.h
index 29f9fc3b205..4e7b4c9938e 100644
--- a/src/gallium/auxiliary/gallivm/lp_bld_logic.h
+++ b/src/gallium/auxiliary/gallivm/lp_bld_logic.h
@@ -63,6 +63,11 @@ lp_build_cmp(struct lp_build_context *bld,
              LLVMValueRef a,
              LLVMValueRef b);
 
+LLVMValueRef
+lp_build_select_bitwise(struct lp_build_context *bld,
+                        LLVMValueRef mask,
+                        LLVMValueRef a,
+                        LLVMValueRef b);
 
 LLVMValueRef
 lp_build_select(struct lp_build_context *bld,
diff --git a/src/gallium/auxiliary/gallivm/lp_bld_misc.cpp b/src/gallium/auxiliary/gallivm/lp_bld_misc.cpp
index 5a9488b5f79..6d5410d9701 100644
--- a/src/gallium/auxiliary/gallivm/lp_bld_misc.cpp
+++ b/src/gallium/auxiliary/gallivm/lp_bld_misc.cpp
@@ -39,6 +39,7 @@
 #include <llvm/Target/TargetOptions.h>
 #include <llvm/ExecutionEngine/ExecutionEngine.h>
 #include <llvm/ExecutionEngine/JITEventListener.h>
+#include <llvm/Support/CommandLine.h>
 
 #include "pipe/p_config.h"
 #include "util/u_debug.h"
@@ -141,4 +142,35 @@ lp_set_target_options(void)
 #if 0
    llvm::UnsafeFPMath = true;
 #endif
+
+#if 0
+   /*
+    * LLVM will generate MMX instructions for vectors <= 64 bits, leading to
+    * innefficient code, and in 32bit systems, to the corruption of the FPU
+    * stack given that it expects the user to generate the EMMS instructions.
+    *
+    * See also:
+    * - http://llvm.org/bugs/show_bug.cgi?id=3287
+    * - http://l4.me.uk/post/2009/06/07/llvm-wrinkle-3-configuration-what-configuration/
+    *
+    * XXX: Unfortunately this is not working.
+    */
+   static boolean first = FALSE;
+   if (first) {
+      static const char* options[] = {
+         "prog",
+         "-disable-mmx"
+      };
+      llvm::cl::ParseCommandLineOptions(2, const_cast<char**>(options));
+      first = FALSE;
+   }
+#endif
+}
+
+
+extern "C" void
+lp_func_delete_body(LLVMValueRef FF)
+{
+   llvm::Function *func = llvm::unwrap<llvm::Function>(FF);
+   func->deleteBody();
 }
diff --git a/src/gallium/auxiliary/tgsi/tgsi_exec.c b/src/gallium/auxiliary/tgsi/tgsi_exec.c
index 5275faa5e22..298f3d0a8bb 100644
--- a/src/gallium/auxiliary/tgsi/tgsi_exec.c
+++ b/src/gallium/auxiliary/tgsi/tgsi_exec.c
@@ -557,6 +557,23 @@ print_temp(const struct tgsi_exec_machine *mach, uint index)
 #endif
 
 
+void
+tgsi_exec_set_constant_buffers(struct tgsi_exec_machine *mach,
+                               unsigned num_bufs,
+                               const void **bufs,
+                               const unsigned *buf_sizes)
+{
+   unsigned i;
+
+   for (i = 0; i < num_bufs; i++) {
+      mach->Consts[i] = bufs[i];
+      mach->ConstsSize[i] = buf_sizes[i];
+   }
+}
+
+
+
+
 /**
  * Check if there's a potential src/dst register data dependency when
  * using SOA execution.
@@ -632,6 +649,10 @@ tgsi_exec_machine_bind_shader(
 
    util_init_math();
 
+   if (numSamplers) {
+      assert(samplers);
+   }
+
    mach->Tokens = tokens;
    mach->Samplers = samplers;
 
@@ -1040,6 +1061,8 @@ fetch_src_file_channel(const struct tgsi_exec_machine *mach,
 {
    uint i;
 
+   assert(swizzle < 4);
+
    switch (file) {
    case TGSI_FILE_CONSTANT:
       for (i = 0; i < QUAD_SIZE; i++) {
@@ -1049,9 +1072,23 @@ fetch_src_file_channel(const struct tgsi_exec_machine *mach,
          if (index->i[i] < 0) {
             chan->u[i] = 0;
          } else {
-            const uint *p = (const uint *)mach->Consts[index2D->i[i]];
-
-            chan->u[i] = p[index->i[i] * 4 + swizzle];
+            /* NOTE: copying the const value as a uint instead of float */
+            const uint constbuf = index2D->i[i];
+            const uint *buf = (const uint *)mach->Consts[constbuf];
+            const int pos = index->i[i] * 4 + swizzle;
+            /* const buffer bounds check */
+            if (pos < 0 || pos >= mach->ConstsSize[constbuf]) {
+               if (0) {
+                  /* Debug: print warning */
+                  static int count = 0;
+                  if (count++ < 100)
+                     debug_printf("TGSI Exec: const buffer index %d"
+                                  " out of bounds\n", pos);
+               }
+               chan->u[i] = 0;
+            }
+            else
+               chan->u[i] = buf[pos];
          }
       }
       break;
@@ -1065,9 +1102,10 @@ fetch_src_file_channel(const struct tgsi_exec_machine *mach,
                          index2D->i[i] * TGSI_EXEC_MAX_INPUT_ATTRIBS + index->i[i],
                          index2D->i[i], index->i[i]);
                          }*/
-         chan->u[i] = mach->Inputs[index2D->i[i] *
-                                   TGSI_EXEC_MAX_INPUT_ATTRIBS +
-                                   index->i[i]].xyzw[swizzle].u[i];
+         int pos = index2D->i[i] * TGSI_EXEC_MAX_INPUT_ATTRIBS + index->i[i];
+         assert(pos >= 0);
+         assert(pos < Elements(mach->Inputs));
+         chan->u[i] = mach->Inputs[pos].xyzw[swizzle].u[i];
       }
       break;
 
@@ -1187,7 +1225,7 @@ fetch_source(const struct tgsi_exec_machine *mach,
       index2.i[1] =
       index2.i[2] =
       index2.i[3] = reg->Indirect.Index;
-
+      assert(reg->Indirect.File == TGSI_FILE_ADDRESS);
       /* get current value of address register[swizzle] */
       swizzle = tgsi_util_get_src_register_swizzle( &reg->Indirect, CHAN_X );
       fetch_src_file_channel(mach,
diff --git a/src/gallium/auxiliary/tgsi/tgsi_exec.h b/src/gallium/auxiliary/tgsi/tgsi_exec.h
index ccf80ca6fd9..6dee362d589 100644
--- a/src/gallium/auxiliary/tgsi/tgsi_exec.h
+++ b/src/gallium/auxiliary/tgsi/tgsi_exec.h
@@ -253,7 +253,10 @@ struct tgsi_exec_machine
    struct tgsi_sampler           **Samplers;
 
    unsigned                      ImmLimit;
+
    const void *Consts[PIPE_MAX_CONSTANT_BUFFERS];
+   unsigned ConstsSize[PIPE_MAX_CONSTANT_BUFFERS];
+
    const struct tgsi_token       *Tokens;   /**< Declarations, instructions */
    unsigned                      Processor; /**< TGSI_PROCESSOR_x */
 
@@ -367,6 +370,13 @@ tgsi_set_exec_mask(struct tgsi_exec_machine *mach,
 }
 
 
+extern void
+tgsi_exec_set_constant_buffers(struct tgsi_exec_machine *mach,
+                               unsigned num_bufs,
+                               const void **bufs,
+                               const unsigned *buf_sizes);
+
+
 #if defined __cplusplus
 } /* extern "C" */
 #endif
diff --git a/src/gallium/auxiliary/tgsi/tgsi_sanity.c b/src/gallium/auxiliary/tgsi/tgsi_sanity.c
index 97148dbe233..9e02040f6c4 100644
--- a/src/gallium/auxiliary/tgsi/tgsi_sanity.c
+++ b/src/gallium/auxiliary/tgsi/tgsi_sanity.c
@@ -33,6 +33,10 @@
 #include "tgsi_info.h"
 #include "tgsi_iterate.h"
 
+
+DEBUG_GET_ONCE_BOOL_OPTION(print_sanity, "TGSI_PRINT_SANITY", TRUE);
+
+
 typedef struct {
    uint file : 28;
    /* max 2 dimensions */
@@ -54,6 +58,8 @@ struct sanity_check_ctx
    uint errors;
    uint warnings;
    uint implied_array_size;
+
+   boolean print;
 };
 
 static INLINE unsigned
@@ -148,6 +154,9 @@ report_error(
 {
    va_list args;
 
+   if (!ctx->print)
+      return;
+
    debug_printf( "Error  : " );
    va_start( args, format );
    _debug_vprintf( format, args );
@@ -164,6 +173,9 @@ report_warning(
 {
    va_list args;
 
+   if (!ctx->print)
+      return;
+
    debug_printf( "Warning: " );
    va_start( args, format );
    _debug_vprintf( format, args );
@@ -539,6 +551,7 @@ tgsi_sanity_check(
    ctx.errors = 0;
    ctx.warnings = 0;
    ctx.implied_array_size = 0;
+   ctx.print = debug_get_option_print_sanity();
 
    if (!tgsi_iterate_shader( tokens, &ctx.iter ))
       return FALSE;
diff --git a/src/gallium/auxiliary/tgsi/tgsi_sanity.h b/src/gallium/auxiliary/tgsi/tgsi_sanity.h
index 52263ff8832..46d8d184190 100644
--- a/src/gallium/auxiliary/tgsi/tgsi_sanity.h
+++ b/src/gallium/auxiliary/tgsi/tgsi_sanity.h
@@ -35,7 +35,8 @@ extern "C" {
 #endif
 
 /* Check the given token stream for errors and common mistakes.
- * Diagnostic messages are printed out to the debug output.
+ * Diagnostic messages are printed out to the debug output, and is
+ * controlled by the debug option TGSI_PRINT_SANITY (default true).
  * Returns TRUE if there are no errors, even though there could be some warnings.
  */
 boolean
diff --git a/src/gallium/auxiliary/translate/translate_generic.c b/src/gallium/auxiliary/translate/translate_generic.c
index 0e43a512ee8..4d1977229e2 100644
--- a/src/gallium/auxiliary/translate/translate_generic.c
+++ b/src/gallium/auxiliary/translate/translate_generic.c
@@ -368,23 +368,23 @@ static void PIPE_CDECL generic_run_elts( struct translate *translate,
    /* loop over vertex attributes (vertex shader inputs)
     */
    for (i = 0; i < count; i++) {
-      unsigned elt = *elts++;
+      const unsigned elt = *elts++;
 
       for (attr = 0; attr < nr_attrs; attr++) {
 	 float data[4];
-         const uint8_t *src;
-         unsigned index;
-
-	 char *dst = (vert + 
-		      tg->attrib[attr].output_offset);
+	 char *dst = vert + tg->attrib[attr].output_offset;
 
          if (tg->attrib[attr].type == TRANSLATE_ELEMENT_NORMAL) {
+            const uint8_t *src;
+            unsigned index;
+
             if (tg->attrib[attr].instance_divisor) {
                index = instance_id / tg->attrib[attr].instance_divisor;
             } else {
                index = elt;
             }
 
+            /* clamp to void going out of bounds */
             index = MIN2(index, tg->attrib[attr].max_index);
 
             src = tg->attrib[attr].input_ptr +
@@ -392,11 +392,23 @@ static void PIPE_CDECL generic_run_elts( struct translate *translate,
 
             tg->attrib[attr].fetch( data, src, 0, 0 );
 
+            if (0)
+               debug_printf("Fetch elt attr %d  from %p  stride %d  div %u  max %u  index %d:  "
+                            " %f, %f, %f, %f \n",
+                            attr,
+                            tg->attrib[attr].input_ptr,
+                            tg->attrib[attr].input_stride,
+                            tg->attrib[attr].instance_divisor,
+                            tg->attrib[attr].max_index,
+                            index,
+                            data[0], data[1],data[2], data[3]);
          } else {
             data[0] = (float)instance_id;
          }
-         if (0) debug_printf("vert %d/%d attr %d: %f %f %f %f\n",
-                             i, elt, attr, data[0], data[1], data[2], data[3]);
+
+         if (0)
+            debug_printf("vert %d/%d attr %d: %f %f %f %f\n",
+                         i, elt, attr, data[0], data[1], data[2], data[3]);
 
 	 tg->attrib[attr].emit( data, dst );
       }
@@ -425,29 +437,42 @@ static void PIPE_CDECL generic_run( struct translate *translate,
 
       for (attr = 0; attr < nr_attrs; attr++) {
 	 float data[4];
-
-	 char *dst = (vert + 
-		      tg->attrib[attr].output_offset);
+	 char *dst = vert + tg->attrib[attr].output_offset;
 
          if (tg->attrib[attr].type == TRANSLATE_ELEMENT_NORMAL) {
             const uint8_t *src;
+            unsigned index;
 
             if (tg->attrib[attr].instance_divisor) {
-               src = tg->attrib[attr].input_ptr +
-                     tg->attrib[attr].input_stride *
-                     (instance_id / tg->attrib[attr].instance_divisor);
-            } else {
-               src = tg->attrib[attr].input_ptr +
-                     tg->attrib[attr].input_stride * elt;
+               index = instance_id / tg->attrib[attr].instance_divisor;
             }
+            else {
+               index = elt;
+            }
+
+            /* clamp to void going out of bounds */
+            index = MIN2(index, tg->attrib[attr].max_index);
+
+            src = tg->attrib[attr].input_ptr +
+                  tg->attrib[attr].input_stride * index;
 
             tg->attrib[attr].fetch( data, src, 0, 0 );
+
+            if (0)
+               debug_printf("Fetch linear attr %d  from %p  stride %d  index %d: "
+                            " %f, %f, %f, %f \n",
+                            attr,
+                            tg->attrib[attr].input_ptr,
+                            tg->attrib[attr].input_stride,
+                            index,
+                            data[0], data[1],data[2], data[3]);
          } else {
             data[0] = (float)instance_id;
          }
 
-         if (0) debug_printf("vert %d attr %d: %f %f %f %f\n",
-                             i, attr, data[0], data[1], data[2], data[3]);
+         if (0)
+            debug_printf("vert %d attr %d: %f %f %f %f\n",
+                         i, attr, data[0], data[1], data[2], data[3]);
 
 	 tg->attrib[attr].emit( data, dst );
       }
diff --git a/src/gallium/auxiliary/util/u_blitter.c b/src/gallium/auxiliary/util/u_blitter.c
index 0d94aaae956..b5b86b72142 100644
--- a/src/gallium/auxiliary/util/u_blitter.c
+++ b/src/gallium/auxiliary/util/u_blitter.c
@@ -87,6 +87,7 @@ struct blitter_context_priv
    void *dsa_write_depth_keep_stencil;
    void *dsa_keep_depth_stencil;
    void *dsa_keep_depth_write_stencil;
+   void *dsa_flush_depth_stencil;
 
    void *velem_state;
 
@@ -156,6 +157,10 @@ struct blitter_context *util_blitter_create(struct pipe_context *pipe)
    ctx->dsa_keep_depth_stencil =
       pipe->create_depth_stencil_alpha_state(pipe, &dsa);
 
+   dsa.depth.writemask = 1;
+   ctx->dsa_flush_depth_stencil =
+      pipe->create_depth_stencil_alpha_state(pipe, &dsa);
+
    dsa.depth.enabled = 1;
    dsa.depth.writemask = 1;
    dsa.depth.func = PIPE_FUNC_ALWAYS;
@@ -940,3 +945,42 @@ void util_blitter_clear_depth_stencil(struct blitter_context *blitter,
                            UTIL_BLITTER_ATTRIB_NONE, NULL);
    blitter_restore_CSOs(ctx);
 }
+
+/* Clear a region of a depth stencil surface. */
+void util_blitter_flush_depth_stencil(struct blitter_context *blitter,
+                                      struct pipe_surface *dstsurf)
+{
+   struct blitter_context_priv *ctx = (struct blitter_context_priv*)blitter;
+   struct pipe_context *pipe = ctx->base.pipe;
+   struct pipe_framebuffer_state fb_state;
+
+   assert(dstsurf->texture);
+   if (!dstsurf->texture)
+      return;
+
+   /* check the saved state */
+   blitter_check_saved_CSOs(ctx);
+   assert(blitter->saved_fb_state.nr_cbufs != ~0);
+
+   /* bind CSOs */
+   pipe->bind_blend_state(pipe, ctx->blend_keep_color);
+   pipe->bind_depth_stencil_alpha_state(pipe, ctx->dsa_flush_depth_stencil);
+
+   pipe->bind_rasterizer_state(pipe, ctx->rs_state);
+   pipe->bind_fs_state(pipe, blitter_get_fs_col(ctx, 0));
+   pipe->bind_vs_state(pipe, ctx->vs_col);
+   pipe->bind_vertex_elements_state(pipe, ctx->velem_state);
+
+   /* set a framebuffer state */
+   fb_state.width = dstsurf->width;
+   fb_state.height = dstsurf->height;
+   fb_state.nr_cbufs = 0;
+   fb_state.cbufs[0] = 0;
+   fb_state.zsbuf = dstsurf;
+   pipe->set_framebuffer_state(pipe, &fb_state);
+
+   blitter_set_dst_dimensions(ctx, dstsurf->width, dstsurf->height);
+   blitter->draw_rectangle(blitter, 0, 0, dstsurf->width, dstsurf->height, 0,
+                           UTIL_BLITTER_ATTRIB_NONE, NULL);
+   blitter_restore_CSOs(ctx);
+}
diff --git a/src/gallium/auxiliary/util/u_blitter.h b/src/gallium/auxiliary/util/u_blitter.h
index ba3f92eca85..f316587dea0 100644
--- a/src/gallium/auxiliary/util/u_blitter.h
+++ b/src/gallium/auxiliary/util/u_blitter.h
@@ -200,6 +200,8 @@ void util_blitter_clear_depth_stencil(struct blitter_context *blitter,
                                       unsigned dstx, unsigned dsty,
                                       unsigned width, unsigned height);
 
+void util_blitter_flush_depth_stencil(struct blitter_context *blitter,
+                                      struct pipe_surface *dstsurf);
 /* The functions below should be used to save currently bound constant state
  * objects inside a driver. The objects are automatically restored at the end
  * of the util_blitter_{clear, copy_region, fill_region} functions and then
diff --git a/src/gallium/auxiliary/util/u_cpu_detect.c b/src/gallium/auxiliary/util/u_cpu_detect.c
index 23d33af4e41..6f38d222854 100644
--- a/src/gallium/auxiliary/util/u_cpu_detect.c
+++ b/src/gallium/auxiliary/util/u_cpu_detect.c
@@ -73,9 +73,15 @@
 #endif
 
 
+DEBUG_GET_ONCE_BOOL_OPTION(dump_cpu, "GALLIUM_DUMP_CPU", TRUE);
+
+
 struct util_cpu_caps util_cpu_caps;
 
+#if defined(PIPE_ARCH_X86) || defined(PIPE_ARCH_X86_64)
 static int has_cpuid(void);
+#endif
+
 
 #if defined(PIPE_ARCH_X86)
 
@@ -497,23 +503,25 @@ util_cpu_detect(void)
 #endif /* PIPE_ARCH_PPC */
 
 #ifdef DEBUG
-   debug_printf("util_cpu_caps.arch = %i\n", util_cpu_caps.arch);
-   debug_printf("util_cpu_caps.nr_cpus = %u\n", util_cpu_caps.nr_cpus);
-
-   debug_printf("util_cpu_caps.x86_cpu_type = %u\n", util_cpu_caps.x86_cpu_type);
-   debug_printf("util_cpu_caps.cacheline = %u\n", util_cpu_caps.cacheline);
-
-   debug_printf("util_cpu_caps.has_tsc = %u\n", util_cpu_caps.has_tsc);
-   debug_printf("util_cpu_caps.has_mmx = %u\n", util_cpu_caps.has_mmx);
-   debug_printf("util_cpu_caps.has_mmx2 = %u\n", util_cpu_caps.has_mmx2);
-   debug_printf("util_cpu_caps.has_sse = %u\n", util_cpu_caps.has_sse);
-   debug_printf("util_cpu_caps.has_sse2 = %u\n", util_cpu_caps.has_sse2);
-   debug_printf("util_cpu_caps.has_sse3 = %u\n", util_cpu_caps.has_sse3);
-   debug_printf("util_cpu_caps.has_ssse3 = %u\n", util_cpu_caps.has_ssse3);
-   debug_printf("util_cpu_caps.has_sse4_1 = %u\n", util_cpu_caps.has_sse4_1);
-   debug_printf("util_cpu_caps.has_3dnow = %u\n", util_cpu_caps.has_3dnow);
-   debug_printf("util_cpu_caps.has_3dnow_ext = %u\n", util_cpu_caps.has_3dnow_ext);
-   debug_printf("util_cpu_caps.has_altivec = %u\n", util_cpu_caps.has_altivec);
+   if (debug_get_option_dump_cpu()) {
+      debug_printf("util_cpu_caps.arch = %i\n", util_cpu_caps.arch);
+      debug_printf("util_cpu_caps.nr_cpus = %u\n", util_cpu_caps.nr_cpus);
+
+      debug_printf("util_cpu_caps.x86_cpu_type = %u\n", util_cpu_caps.x86_cpu_type);
+      debug_printf("util_cpu_caps.cacheline = %u\n", util_cpu_caps.cacheline);
+
+      debug_printf("util_cpu_caps.has_tsc = %u\n", util_cpu_caps.has_tsc);
+      debug_printf("util_cpu_caps.has_mmx = %u\n", util_cpu_caps.has_mmx);
+      debug_printf("util_cpu_caps.has_mmx2 = %u\n", util_cpu_caps.has_mmx2);
+      debug_printf("util_cpu_caps.has_sse = %u\n", util_cpu_caps.has_sse);
+      debug_printf("util_cpu_caps.has_sse2 = %u\n", util_cpu_caps.has_sse2);
+      debug_printf("util_cpu_caps.has_sse3 = %u\n", util_cpu_caps.has_sse3);
+      debug_printf("util_cpu_caps.has_ssse3 = %u\n", util_cpu_caps.has_ssse3);
+      debug_printf("util_cpu_caps.has_sse4_1 = %u\n", util_cpu_caps.has_sse4_1);
+      debug_printf("util_cpu_caps.has_3dnow = %u\n", util_cpu_caps.has_3dnow);
+      debug_printf("util_cpu_caps.has_3dnow_ext = %u\n", util_cpu_caps.has_3dnow_ext);
+      debug_printf("util_cpu_caps.has_altivec = %u\n", util_cpu_caps.has_altivec);
+   }
 #endif
 
    util_cpu_detect_initialized = TRUE;
diff --git a/src/gallium/auxiliary/util/u_draw.h b/src/gallium/auxiliary/util/u_draw.h
new file mode 100644
index 00000000000..2a91ea0f9ae
--- /dev/null
+++ b/src/gallium/auxiliary/util/u_draw.h
@@ -0,0 +1,138 @@
+/**************************************************************************
+ *
+ * Copyright 2008 Tungsten Graphics, Inc., Cedar Park, Texas.
+ * All Rights Reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sub license, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the
+ * next paragraph) shall be included in all copies or substantial portions
+ * of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
+ * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
+ * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ *
+ **************************************************************************/
+
+#ifndef U_DRAW_H
+#define U_DRAW_H
+
+
+#include "pipe/p_compiler.h"
+#include "pipe/p_context.h"
+
+
+static INLINE void
+util_draw_init_info(struct pipe_draw_info *info)
+{
+   memset(info, 0, sizeof(*info));
+   info->instance_count = 1;
+   info->max_index = 0xffffffff;
+}
+
+
+static INLINE void
+util_draw_arrays(struct pipe_context *pipe, uint mode, uint start, uint count)
+{
+   struct pipe_draw_info info;
+
+   util_draw_init_info(&info);
+   info.mode = mode;
+   info.start = start;
+   info.count = count;
+   info.min_index = start;
+   info.max_index = start + count - 1;
+
+   pipe->draw_vbo(pipe, &info);
+}
+
+static INLINE void
+util_draw_elements(struct pipe_context *pipe, int index_bias,
+                   uint mode, uint start, uint count)
+{
+   struct pipe_draw_info info;
+
+   util_draw_init_info(&info);
+   info.indexed = TRUE;
+   info.mode = mode;
+   info.start = start;
+   info.count = count;
+   info.index_bias = index_bias;
+
+   pipe->draw_vbo(pipe, &info);
+}
+
+static INLINE void
+util_draw_arrays_instanced(struct pipe_context *pipe,
+                           uint mode, uint start, uint count,
+                           uint start_instance,
+                           uint instance_count)
+{
+   struct pipe_draw_info info;
+
+   util_draw_init_info(&info);
+   info.mode = mode;
+   info.start = start;
+   info.count = count;
+   info.start_instance = start_instance;
+   info.instance_count = instance_count;
+   info.min_index = start;
+   info.max_index = start + count - 1;
+
+   pipe->draw_vbo(pipe, &info);
+}
+
+static INLINE void
+util_draw_elements_instanced(struct pipe_context *pipe,
+                             int index_bias,
+                             uint mode, uint start, uint count,
+                             uint start_instance,
+                             uint instance_count)
+{
+   struct pipe_draw_info info;
+
+   util_draw_init_info(&info);
+   info.indexed = TRUE;
+   info.mode = mode;
+   info.start = start;
+   info.count = count;
+   info.index_bias = index_bias;
+   info.start_instance = start_instance;
+   info.instance_count = instance_count;
+
+   pipe->draw_vbo(pipe, &info);
+}
+
+static INLINE void
+util_draw_range_elements(struct pipe_context *pipe,
+                         int index_bias,
+                         uint min_index,
+                         uint max_index,
+                         uint mode, uint start, uint count)
+{
+   struct pipe_draw_info info;
+
+   util_draw_init_info(&info);
+   info.indexed = TRUE;
+   info.mode = mode;
+   info.start = start;
+   info.count = count;
+   info.index_bias = index_bias;
+   info.min_index = min_index;
+   info.max_index = max_index;
+
+   pipe->draw_vbo(pipe, &info);
+}
+
+#endif
diff --git a/src/gallium/auxiliary/util/u_draw_quad.c b/src/gallium/auxiliary/util/u_draw_quad.c
index b37b48b5aef..0b6dc5880f3 100644
--- a/src/gallium/auxiliary/util/u_draw_quad.c
+++ b/src/gallium/auxiliary/util/u_draw_quad.c
@@ -60,7 +60,7 @@ util_draw_vertex_buffer(struct pipe_context *pipe,
    /* note: vertex elements already set by caller */
 
    /* draw */
-   pipe->draw_arrays(pipe, prim_type, 0, num_verts);
+   util_draw_arrays(pipe, prim_type, 0, num_verts);
 }
 
 
diff --git a/src/gallium/auxiliary/util/u_draw_quad.h b/src/gallium/auxiliary/util/u_draw_quad.h
index 42eb1844289..52994fe05c3 100644
--- a/src/gallium/auxiliary/util/u_draw_quad.h
+++ b/src/gallium/auxiliary/util/u_draw_quad.h
@@ -29,12 +29,18 @@
 #define U_DRAWQUAD_H
 
 
+#include "pipe/p_compiler.h"
+#include "pipe/p_context.h"
+
+
 #ifdef __cplusplus
 extern "C" {
 #endif
 
 struct pipe_resource;
 
+#include "util/u_draw.h"
+
 extern void 
 util_draw_vertex_buffer(struct pipe_context *pipe,
                         struct pipe_resource *vbuf, uint offset,
diff --git a/src/gallium/auxiliary/util/u_format_other.c b/src/gallium/auxiliary/util/u_format_other.c
index 723fa8c3bf9..fa42ec37138 100644
--- a/src/gallium/auxiliary/util/u_format_other.c
+++ b/src/gallium/auxiliary/util/u_format_other.c
@@ -121,6 +121,15 @@ util_format_r1_unorm_pack_rgba_8unorm(uint8_t *dst_row, unsigned dst_stride,
  * A.k.a. D3DFMT_CxV8U8
  */
 
+static uint8_t
+r8g8bx_derive(int16_t r, int16_t g)
+{
+   /* Derive blue from red and green components.
+    * Apparently, we must always use integers to perform calculations,
+    * otherwise the results won't match D3D's CxV8U8 definition.
+    */
+   return (uint8_t)sqrtf(0x7f * 0x7f - r * r - g * g) * 0xff / 0x7f;
+}
 
 void
 util_format_r8g8bx_snorm_unpack_rgba_float(float *dst_row, unsigned dst_stride,
@@ -145,7 +154,7 @@ util_format_r8g8bx_snorm_unpack_rgba_float(float *dst_row, unsigned dst_stride,
 
          dst[0] = (float)(r * (1.0f/0x7f)); /* r */
          dst[1] = (float)(g * (1.0f/0x7f)); /* g */
-         dst[2] = sqrtf(1.0f - dst[0] * dst[0] - dst[1] * dst[1]); /* b */
+         dst[2] = r8g8bx_derive(r, g) * (1.0f/0xff); /* b */
          dst[3] = 1.0f; /* a */
          dst += 4;
       }
@@ -177,7 +186,7 @@ util_format_r8g8bx_snorm_unpack_rgba_8unorm(uint8_t *dst_row, unsigned dst_strid
 
          dst[0] = (uint8_t)(((uint16_t)MAX2(r, 0)) * 0xff / 0x7f); /* r */
          dst[1] = (uint8_t)(((uint16_t)MAX2(g, 0)) * 0xff / 0x7f); /* g */
-         dst[2] = (uint8_t)sqrtf(0x7f*0x7f - r * r - g * g) * 0xff / 0x7f; /* b */
+         dst[2] = r8g8bx_derive(r, g); /* b */
          dst[3] = 255; /* a */
          dst += 4;
       }
@@ -262,6 +271,6 @@ util_format_r8g8bx_snorm_fetch_rgba_float(float *dst, const uint8_t *src,
 
    dst[0] = r * (1.0f/0x7f); /* r */
    dst[1] = g * (1.0f/0x7f); /* g */
-   dst[2] = sqrtf(1.0f - dst[0] * dst[0] - dst[1] * dst[1]); /* b */
+   dst[2] = r8g8bx_derive(r, g) * (1.0f/0xff); /* b */
    dst[3] = 1.0f; /* a */
 }
diff --git a/src/gallium/auxiliary/util/u_pack_color.h b/src/gallium/auxiliary/util/u_pack_color.h
index 3ebef9fb749..5f113f742b1 100644
--- a/src/gallium/auxiliary/util/u_pack_color.h
+++ b/src/gallium/auxiliary/util/u_pack_color.h
@@ -425,6 +425,53 @@ util_pack_color(const float rgba[4], enum pipe_format format, union util_color *
    }
 }
  
+/* Integer versions of util_pack_z and util_pack_z_stencil - useful for
+ * constructing clear masks.
+ */
+static INLINE uint
+util_pack_uint_z(enum pipe_format format, unsigned z)
+{
+   switch (format) {
+   case PIPE_FORMAT_Z16_UNORM:
+      return z & 0xffff;
+   case PIPE_FORMAT_Z32_UNORM:
+   case PIPE_FORMAT_Z32_FLOAT:
+      return z;
+   case PIPE_FORMAT_Z24_UNORM_S8_USCALED:
+   case PIPE_FORMAT_Z24X8_UNORM:
+      return z & 0xffffff;
+   case PIPE_FORMAT_S8_USCALED_Z24_UNORM:
+   case PIPE_FORMAT_X8Z24_UNORM:
+      return (z & 0xffffff) << 8;
+   case PIPE_FORMAT_S8_USCALED:
+      return 0;
+   default:
+      debug_print_format("gallium: unhandled format in util_pack_z()", format);
+      assert(0);
+      return 0;
+   }
+}
+
+static INLINE uint
+util_pack_uint_z_stencil(enum pipe_format format, double z, uint s)
+{
+   unsigned packed = util_pack_uint_z(format, z);
+
+   s &= 0xff;
+
+   switch (format) {
+   case PIPE_FORMAT_Z24_UNORM_S8_USCALED:
+      return packed | (s << 24);
+   case PIPE_FORMAT_S8_USCALED_Z24_UNORM:
+      return packed | s;
+   case PIPE_FORMAT_S8_USCALED:
+      return packed | s;
+   default:
+      return packed;
+   }
+}
+
+
 
 /**
  * Note: it's assumed that z is in [0,1]
diff --git a/src/gallium/auxiliary/util/u_prim.h b/src/gallium/auxiliary/util/u_prim.h
index 606b9b5c6b9..3c851f73401 100644
--- a/src/gallium/auxiliary/util/u_prim.h
+++ b/src/gallium/auxiliary/util/u_prim.h
@@ -108,6 +108,20 @@ static INLINE boolean u_trim_pipe_prim( unsigned pipe_prim, unsigned *nr )
       ok = (*nr >= 4);
       *nr -= (*nr % 2);
       break;
+   case PIPE_PRIM_LINES_ADJACENCY:
+      ok = (*nr >= 4);
+      *nr -= (*nr % 4);
+      break;
+   case PIPE_PRIM_LINE_STRIP_ADJACENCY:
+      ok = (*nr >= 4);
+      break;
+   case PIPE_PRIM_TRIANGLES_ADJACENCY:
+      ok = (*nr >= 6);
+      *nr -= (*nr % 5);
+      break;
+   case PIPE_PRIM_TRIANGLE_STRIP_ADJACENCY:
+      ok = (*nr >= 4);
+      break;
    default:
       ok = 0;
       break;
diff --git a/src/gallium/auxiliary/util/u_sse.h b/src/gallium/auxiliary/util/u_sse.h
index e2a8491e62c..6145e34aa3f 100644
--- a/src/gallium/auxiliary/util/u_sse.h
+++ b/src/gallium/auxiliary/util/u_sse.h
@@ -41,7 +41,6 @@
 
 #if defined(PIPE_ARCH_SSE)
 
-#include <xmmintrin.h>
 #include <emmintrin.h>