Merge remote branch 'origin/master' into pipe-video

Conflicts:
	configure.ac
	src/gallium/drivers/r600/eg_asm.c
	src/gallium/drivers/r600/r600_asm.c
	src/gallium/drivers/r600/r600_asm.h
	src/gallium/include/pipe/p_format.h
	src/gallium/targets/dri-nouveau/Makefile

17 files changed, 341 insertions, 196 deletions

diff --git a/src/gallium/auxiliary/draw/draw_context.c b/src/gallium/auxiliary/draw/draw_context.c
index 73d5b6e403f..e045313b94f 100644
--- a/src/gallium/auxiliary/draw/draw_context.c
+++ b/src/gallium/auxiliary/draw/draw_context.c
@@ -35,6 +35,7 @@
 #include "util/u_memory.h"
 #include "util/u_math.h"
 #include "util/u_cpu_detect.h"
+#include "util/u_inlines.h"
 #include "draw_context.h"
 #include "draw_vs.h"
 #include "draw_gs.h"
@@ -164,6 +165,10 @@ void draw_destroy( struct draw_context *draw )
       }
    }
 
+   for (i = 0; i < draw->pt.nr_vertex_buffers; i++) {
+      pipe_resource_reference(&draw->pt.vertex_buffer[i].buffer, NULL);
+   }
+
    /* Not so fast -- we're just borrowing this at the moment.
     * 
    if (draw->render)
@@ -307,8 +312,9 @@ draw_set_vertex_buffers(struct draw_context *draw,
 {
    assert(count <= PIPE_MAX_ATTRIBS);
 
-   memcpy(draw->pt.vertex_buffer, buffers, count * sizeof(buffers[0]));
-   draw->pt.nr_vertex_buffers = count;
+   util_copy_vertex_buffers(draw->pt.vertex_buffer,
+                            &draw->pt.nr_vertex_buffers,
+                            buffers, count);
 }
 
 
diff --git a/src/gallium/auxiliary/gallivm/lp_bld_const.h b/src/gallium/auxiliary/gallivm/lp_bld_const.h
index 680211fbbd7..69718eb4b3d 100644
--- a/src/gallium/auxiliary/gallivm/lp_bld_const.h
+++ b/src/gallium/auxiliary/gallivm/lp_bld_const.h
@@ -134,7 +134,7 @@ lp_build_const_int_pointer(struct gallivm_state *gallivm, const void *ptr)
 
    /* int type large enough to hold a pointer */
    int_type = LLVMIntTypeInContext(gallivm->context, 8 * sizeof(void *));
-   v = LLVMConstInt(int_type, (unsigned long long) ptr, 0);
+   v = LLVMConstInt(int_type, (uintptr_t) ptr, 0);
    v = LLVMBuildIntToPtr(gallivm->builder, v,
                          LLVMPointerType(int_type, 0),
                          "cast int to ptr");
diff --git a/src/gallium/auxiliary/gallivm/lp_bld_misc.cpp b/src/gallium/auxiliary/gallivm/lp_bld_misc.cpp
index f56ddee7fd7..46dd00d8224 100644
--- a/src/gallium/auxiliary/gallivm/lp_bld_misc.cpp
+++ b/src/gallium/auxiliary/gallivm/lp_bld_misc.cpp
@@ -144,6 +144,7 @@ lp_set_target_options(void)
    llvm::UnsafeFPMath = true;
 #endif
 
+#if HAVE_LLVM < 0x0209
    /*
     * LLVM will generate MMX instructions for vectors <= 64 bits, leading to
     * innefficient code, and in 32bit systems, to the corruption of the FPU
@@ -162,6 +163,7 @@ lp_set_target_options(void)
       llvm::cl::ParseCommandLineOptions(2, const_cast<char**>(options));
       first = FALSE;
    }
+#endif
 
    /*
     * By default LLVM adds a signal handler to output a pretty stack trace.
diff --git a/src/gallium/auxiliary/tgsi/tgsi_dump.c b/src/gallium/auxiliary/tgsi/tgsi_dump.c
index 77bde86684e..82cd8eaa969 100644
--- a/src/gallium/auxiliary/tgsi/tgsi_dump.c
+++ b/src/gallium/auxiliary/tgsi/tgsi_dump.c
@@ -138,7 +138,7 @@ static const char *immediate_type_names[] =
 };
 
 const char *
-tgsi_swizzle_names[] =
+tgsi_swizzle_names[4] =
 {
    "x",
    "y",
@@ -147,7 +147,7 @@ tgsi_swizzle_names[] =
 };
 
 const char *
-tgsi_texture_names[] =
+tgsi_texture_names[TGSI_TEXTURE_COUNT] =
 {
    "UNKNOWN",
    "1D",
@@ -160,16 +160,17 @@ tgsi_texture_names[] =
    "SHADOWRECT"
 };
 
-static const char *property_names[] =
+const char *tgsi_property_names[TGSI_PROPERTY_COUNT] =
 {
    "GS_INPUT_PRIMITIVE",
    "GS_OUTPUT_PRIMITIVE",
    "GS_MAX_OUTPUT_VERTICES",
    "FS_COORD_ORIGIN",
-   "FS_COORD_PIXEL_CENTER"
+   "FS_COORD_PIXEL_CENTER",
+   "FS_COLOR0_WRITES_ALL_CBUFS",
 };
 
-static const char *primitive_names[] =
+const char *tgsi_primitive_names[PIPE_PRIM_MAX] =
 {
    "POINTS",
    "LINES",
@@ -187,13 +188,13 @@ static const char *primitive_names[] =
    "TRIANGLE_STRIP_ADJACENCY"
 };
 
-static const char *fs_coord_origin_names[] =
+const char *tgsi_fs_coord_origin_names[2] =
 {
    "UPPER_LEFT",
    "LOWER_LEFT"
 };
 
-static const char *fs_coord_pixel_center_names[] =
+const char *tgsi_fs_coord_pixel_center_names[2] =
 {
    "HALF_INTEGER",
    "INTEGER"
@@ -484,10 +485,10 @@ iter_property(
    int i;
    struct dump_ctx *ctx = (struct dump_ctx *)iter;
 
-   assert(Elements(property_names) == TGSI_PROPERTY_COUNT);
+   assert(Elements(tgsi_property_names) == TGSI_PROPERTY_COUNT);
 
    TXT( "PROPERTY " );
-   ENM(prop->Property.PropertyName, property_names);
+   ENM(prop->Property.PropertyName, tgsi_property_names);
 
    if (prop->Property.NrTokens > 1)
       TXT(" ");
@@ -496,13 +497,13 @@ iter_property(
       switch (prop->Property.PropertyName) {
       case TGSI_PROPERTY_GS_INPUT_PRIM:
       case TGSI_PROPERTY_GS_OUTPUT_PRIM:
-         ENM(prop->u[i].Data, primitive_names);
+         ENM(prop->u[i].Data, tgsi_primitive_names);
          break;
       case TGSI_PROPERTY_FS_COORD_ORIGIN:
-         ENM(prop->u[i].Data, fs_coord_origin_names);
+         ENM(prop->u[i].Data, tgsi_fs_coord_origin_names);
          break;
       case TGSI_PROPERTY_FS_COORD_PIXEL_CENTER:
-         ENM(prop->u[i].Data, fs_coord_pixel_center_names);
+         ENM(prop->u[i].Data, tgsi_fs_coord_pixel_center_names);
          break;
       default:
          SID( prop->u[i].Data );
diff --git a/src/gallium/auxiliary/tgsi/tgsi_dump.h b/src/gallium/auxiliary/tgsi/tgsi_dump.h
index fc0429ad8d9..2491e9198a2 100644
--- a/src/gallium/auxiliary/tgsi/tgsi_dump.h
+++ b/src/gallium/auxiliary/tgsi/tgsi_dump.h
@@ -29,6 +29,7 @@
 #define TGSI_DUMP_H
 
 #include "pipe/p_compiler.h"
+#include "pipe/p_defines.h"
 #include "pipe/p_shader_tokens.h"
 
 #if defined __cplusplus
@@ -39,10 +40,22 @@ extern const char *
 tgsi_file_names[TGSI_FILE_COUNT];
 
 extern const char *
-tgsi_swizzle_names[];
+tgsi_swizzle_names[4];
 
 extern const char *
-tgsi_texture_names[];
+tgsi_texture_names[TGSI_TEXTURE_COUNT];
+
+extern const char *
+tgsi_property_names[TGSI_PROPERTY_COUNT];
+
+extern const char *
+tgsi_primitive_names[PIPE_PRIM_MAX];
+
+extern const char *
+tgsi_fs_coord_origin_names[2];
+
+extern const char *
+tgsi_fs_coord_pixel_center_names[2];
 
 void
 tgsi_dump_str(
diff --git a/src/gallium/auxiliary/tgsi/tgsi_sanity.h b/src/gallium/auxiliary/tgsi/tgsi_sanity.h
index 73f0f414e3f..707d11ec6dd 100644
--- a/src/gallium/auxiliary/tgsi/tgsi_sanity.h
+++ b/src/gallium/auxiliary/tgsi/tgsi_sanity.h
@@ -28,12 +28,14 @@
 #ifndef TGSI_SANITY_H
 #define TGSI_SANITY_H
 
-#include "pipe/p_shader_tokens.h"
-
 #if defined __cplusplus
 extern "C" {
 #endif
 
+#include "pipe/p_compiler.h"
+
+struct tgsi_token;
+
 /* Check the given token stream for errors and common mistakes.
  * Diagnostic messages are printed out to the debug output, and is
  * controlled by the debug option TGSI_PRINT_SANITY (default false).
diff --git a/src/gallium/auxiliary/tgsi/tgsi_text.c b/src/gallium/auxiliary/tgsi/tgsi_text.c
index 9a38c37979c..819b0725a1f 100644
--- a/src/gallium/auxiliary/tgsi/tgsi_text.c
+++ b/src/gallium/auxiliary/tgsi/tgsi_text.c
@@ -36,6 +36,7 @@
 #include "tgsi_parse.h"
 #include "tgsi_sanity.h"
 #include "tgsi_util.h"
+#include "tgsi_dump.h"
 
 static boolean is_alpha_underscore( const char *cur )
 {
@@ -1258,42 +1259,6 @@ static boolean parse_immediate( struct translate_ctx *ctx )
    return TRUE;
 }
 
-static const char *property_names[] =
-{
-   "GS_INPUT_PRIMITIVE",
-   "GS_OUTPUT_PRIMITIVE",
-   "GS_MAX_OUTPUT_VERTICES",
-   "FS_COORD_ORIGIN",
-   "FS_COORD_PIXEL_CENTER"
-};
-
-static const char *primitive_names[] =
-{
-   "POINTS",
-   "LINES",
-   "LINE_LOOP",
-   "LINE_STRIP",
-   "TRIANGLES",
-   "TRIANGLE_STRIP",
-   "TRIANGLE_FAN",
-   "QUADS",
-   "QUAD_STRIP",
-   "POLYGON"
-};
-
-static const char *fs_coord_origin_names[] =
-{
-   "UPPER_LEFT",
-   "LOWER_LEFT"
-};
-
-static const char *fs_coord_pixel_center_names[] =
-{
-   "HALF_INTEGER",
-   "INTEGER"
-};
-
-
 static boolean
 parse_primitive( const char **pcur, uint *primitive )
 {
@@ -1302,7 +1267,7 @@ parse_primitive( const char **pcur, uint *primitive )
    for (i = 0; i < PIPE_PRIM_MAX; i++) {
       const char *cur = *pcur;
 
-      if (str_match_no_case( &cur, primitive_names[i])) {
+      if (str_match_no_case( &cur, tgsi_primitive_names[i])) {
          *primitive = i;
          *pcur = cur;
          return TRUE;
@@ -1316,10 +1281,10 @@ parse_fs_coord_origin( const char **pcur, uint *fs_coord_origin )
 {
    uint i;
 
-   for (i = 0; i < sizeof(fs_coord_origin_names) / sizeof(fs_coord_origin_names[0]); i++) {
+   for (i = 0; i < Elements(tgsi_fs_coord_origin_names); i++) {
       const char *cur = *pcur;
 
-      if (str_match_no_case( &cur, fs_coord_origin_names[i])) {
+      if (str_match_no_case( &cur, tgsi_fs_coord_origin_names[i])) {
          *fs_coord_origin = i;
          *pcur = cur;
          return TRUE;
@@ -1333,10 +1298,10 @@ parse_fs_coord_pixel_center( const char **pcur, uint *fs_coord_pixel_center )
 {
    uint i;
 
-   for (i = 0; i < sizeof(fs_coord_pixel_center_names) / sizeof(fs_coord_pixel_center_names[0]); i++) {
+   for (i = 0; i < Elements(tgsi_fs_coord_pixel_center_names); i++) {
       const char *cur = *pcur;
 
-      if (str_match_no_case( &cur, fs_coord_pixel_center_names[i])) {
+      if (str_match_no_case( &cur, tgsi_fs_coord_pixel_center_names[i])) {
          *fs_coord_pixel_center = i;
          *pcur = cur;
          return TRUE;
@@ -1364,7 +1329,7 @@ static boolean parse_property( struct translate_ctx *ctx )
    }
    for (property_name = 0; property_name < TGSI_PROPERTY_COUNT;
         ++property_name) {
-      if (streq_nocase_uprcase(property_names[property_name], id)) {
+      if (streq_nocase_uprcase(tgsi_property_names[property_name], id)) {
          break;
       }
    }
@@ -1398,6 +1363,7 @@ static boolean parse_property( struct translate_ctx *ctx )
          return FALSE;
       }
       break;
+   case TGSI_PROPERTY_FS_COLOR0_WRITES_ALL_CBUFS:
    default:
       if (!parse_uint(&ctx->cur, &values[0] )) {
          report_error( ctx, "Expected unsigned integer as property!" );
diff --git a/src/gallium/auxiliary/tgsi/tgsi_text.h b/src/gallium/auxiliary/tgsi/tgsi_text.h
index 8eeeeef1402..5fe53cf007d 100644
--- a/src/gallium/auxiliary/tgsi/tgsi_text.h
+++ b/src/gallium/auxiliary/tgsi/tgsi_text.h
@@ -28,12 +28,14 @@
 #ifndef TGSI_TEXT_H
 #define TGSI_TEXT_H
 
-#include "pipe/p_shader_tokens.h"
-
 #if defined __cplusplus
 extern "C" {
 #endif
 
+#include "pipe/p_compiler.h"
+
+struct tgsi_token;
+
 boolean
 tgsi_text_translate(
    const char *text,
diff --git a/src/gallium/auxiliary/tgsi/tgsi_ureg.c b/src/gallium/auxiliary/tgsi/tgsi_ureg.c
index 7d13a17bdbc..02de12d77d5 100644
--- a/src/gallium/auxiliary/tgsi/tgsi_ureg.c
+++ b/src/gallium/auxiliary/tgsi/tgsi_ureg.c
@@ -148,6 +148,7 @@ struct ureg_program
    unsigned property_gs_max_vertices;
    unsigned char property_fs_coord_origin; /* = TGSI_FS_COORD_ORIGIN_* */
    unsigned char property_fs_coord_pixel_center; /* = TGSI_FS_COORD_PIXEL_CENTER_* */
+   unsigned char property_fs_color0_writes_all_cbufs; /* = TGSI_FS_COLOR0_WRITES_ALL_CBUFS * */
 
    unsigned nr_addrs;
    unsigned nr_preds;
@@ -284,7 +285,12 @@ ureg_property_fs_coord_pixel_center(struct ureg_program *ureg,
    ureg->property_fs_coord_pixel_center = fs_coord_pixel_center;
 }
 
-
+void
+ureg_property_fs_color0_writes_all_cbufs(struct ureg_program *ureg,
+                            unsigned fs_color0_writes_all_cbufs)
+{
+   ureg->property_fs_color0_writes_all_cbufs = fs_color0_writes_all_cbufs;
+}
 
 struct ureg_src
 ureg_DECL_fs_input_cyl_centroid(struct ureg_program *ureg,
@@ -1278,6 +1284,14 @@ static void emit_decls( struct ureg_program *ureg )
                     ureg->property_fs_coord_pixel_center);
    }
 
+   if (ureg->property_fs_color0_writes_all_cbufs) {
+      assert(ureg->processor == TGSI_PROCESSOR_FRAGMENT);
+
+      emit_property(ureg,
+                    TGSI_PROPERTY_FS_COLOR0_WRITES_ALL_CBUFS,
+                    ureg->property_fs_color0_writes_all_cbufs);
+   }
+
    if (ureg->processor == TGSI_PROCESSOR_VERTEX) {
       for (i = 0; i < UREG_MAX_INPUT; i++) {
          if (ureg->vs_inputs[i/32] & (1 << (i%32))) {
diff --git a/src/gallium/auxiliary/tgsi/tgsi_ureg.h b/src/gallium/auxiliary/tgsi/tgsi_ureg.h
index acc463200a6..807128a5e52 100644
--- a/src/gallium/auxiliary/tgsi/tgsi_ureg.h
+++ b/src/gallium/auxiliary/tgsi/tgsi_ureg.h
@@ -153,6 +153,10 @@ void
 ureg_property_fs_coord_pixel_center(struct ureg_program *ureg,
                             unsigned fs_coord_pixel_center);
 
+void
+ureg_property_fs_color0_writes_all_cbufs(struct ureg_program *ureg,
+                            unsigned fs_color0_writes_all_cbufs);
+
 /***********************************************************************
  * Build shader declarations:
  */
diff --git a/src/gallium/auxiliary/util/u_blitter.h b/src/gallium/auxiliary/util/u_blitter.h
index c5660cf2d00..922a8580ac1 100644
--- a/src/gallium/auxiliary/util/u_blitter.h
+++ b/src/gallium/auxiliary/util/u_blitter.h
@@ -318,21 +318,13 @@ util_blitter_save_vertex_buffers(struct blitter_context *blitter,
                                          int num_vertex_buffers,
                                          struct pipe_vertex_buffer *vertex_buffers)
 {
-   unsigned i;
    assert(num_vertex_buffers <= Elements(blitter->saved_vertex_buffers));
 
-   blitter->saved_num_vertex_buffers = num_vertex_buffers;
-
-   for (i = 0; i < num_vertex_buffers; i++) {
-      if (vertex_buffers[i].buffer) {
-         pipe_resource_reference(&blitter->saved_vertex_buffers[i].buffer,
-                                 vertex_buffers[i].buffer);
-      }
-   }
-
-   memcpy(blitter->saved_vertex_buffers,
-          vertex_buffers,
-          num_vertex_buffers * sizeof(struct pipe_vertex_buffer));
+   blitter->saved_num_vertex_buffers = 0;
+   util_copy_vertex_buffers(blitter->saved_vertex_buffers,
+                            (unsigned*)&blitter->saved_num_vertex_buffers,
+                            vertex_buffers,
+                            num_vertex_buffers);
 }
 
 #ifdef __cplusplus
diff --git a/src/gallium/auxiliary/util/u_format.csv b/src/gallium/auxiliary/util/u_format.csv
index 8e5d4487a67..0ae15c9cc0a 100644
--- a/src/gallium/auxiliary/util/u_format.csv
+++ b/src/gallium/auxiliary/util/u_format.csv
@@ -76,6 +76,7 @@ PIPE_FORMAT_B4G4R4X4_UNORM        , plain, 1, 1, un4 , un4 , un4 , x4  , zyx1, r
 PIPE_FORMAT_B5G6R5_UNORM          , plain, 1, 1, un5 , un6 , un5 ,     , zyx1, rgb
 PIPE_FORMAT_R10G10B10A2_UNORM     , plain, 1, 1, un10, un10, un10, un2 , xyzw, rgb
 PIPE_FORMAT_B10G10R10A2_UNORM     , plain, 1, 1, un10, un10, un10, un2 , zyxw, rgb
+PIPE_FORMAT_B2G3R3_UNORM          , plain, 1, 1, un2 , un3 , un3 ,     , zyx1, rgb
 
 # Luminance/Intensity/Alpha formats
 PIPE_FORMAT_L8_UNORM              , plain, 1, 1, un8 ,     ,     ,     , xxx1, rgb
@@ -84,6 +85,9 @@ PIPE_FORMAT_I8_UNORM              , plain, 1, 1, un8 ,     ,     ,     , xxxx, r
 PIPE_FORMAT_L4A4_UNORM            , plain, 1, 1, un4 , un4 ,     ,     , xxxy, rgb
 PIPE_FORMAT_L8A8_UNORM            , plain, 1, 1, un8 , un8 ,     ,     , xxxy, rgb
 PIPE_FORMAT_L16_UNORM             , plain, 1, 1, un16,     ,     ,     , xxx1, rgb
+PIPE_FORMAT_A16_UNORM             , plain, 1, 1, un16,     ,     ,     , 000x, rgb
+PIPE_FORMAT_I16_UNORM             , plain, 1, 1, un16,     ,     ,     , xxxx, rgb
+PIPE_FORMAT_L16A16_UNORM          , plain, 1, 1, un16, un16,     ,     , xxxy, rgb
 
 # SRGB formats
 PIPE_FORMAT_L8_SRGB               , plain, 1, 1, un8 ,     ,     ,     , xxx1, srgb 
diff --git a/src/gallium/auxiliary/util/u_index_modify.c b/src/gallium/auxiliary/util/u_index_modify.c
index 3822f60e71d..f2c9db3caf1 100644
--- a/src/gallium/auxiliary/util/u_index_modify.c
+++ b/src/gallium/auxiliary/util/u_index_modify.c
@@ -24,26 +24,70 @@
 #include "util/u_index_modify.h"
 #include "util/u_inlines.h"
 
+/* Ubyte indices. */
+
+void util_shorten_ubyte_elts_to_userptr(struct pipe_context *context,
+					struct pipe_resource *elts,
+					int index_bias,
+					unsigned start,
+					unsigned count,
+					void *out)
+{
+    struct pipe_transfer *src_transfer;
+    unsigned char *in_map;
+    unsigned short *out_map = out;
+    unsigned i;
+
+    in_map = pipe_buffer_map(context, elts, PIPE_TRANSFER_READ, &src_transfer);
+    in_map += start;
+
+    for (i = 0; i < count; i++) {
+        *out_map = (unsigned short)(*in_map + index_bias);
+        in_map++;
+        out_map++;
+    }
+
+    pipe_buffer_unmap(context, src_transfer);
+}
+
 void util_shorten_ubyte_elts(struct pipe_context *context,
 			     struct pipe_resource **elts,
 			     int index_bias,
 			     unsigned start,
 			     unsigned count)
 {
-    struct pipe_screen* screen = context->screen;
     struct pipe_resource* new_elts;
-    unsigned char *in_map;
     unsigned short *out_map;
-    struct pipe_transfer *src_transfer, *dst_transfer;
-    unsigned i;
+    struct pipe_transfer *dst_transfer;
 
-    new_elts = pipe_buffer_create(screen,
+    new_elts = pipe_buffer_create(context->screen,
                                   PIPE_BIND_INDEX_BUFFER,
                                   2 * count);
 
-    in_map = pipe_buffer_map(context, *elts, PIPE_TRANSFER_READ, &src_transfer);
-    out_map = pipe_buffer_map(context, new_elts, PIPE_TRANSFER_WRITE, &dst_transfer);
+    out_map = pipe_buffer_map(context, new_elts, PIPE_TRANSFER_WRITE,
+                              &dst_transfer);
+    util_shorten_ubyte_elts_to_userptr(context, *elts, index_bias,
+                                       start, count, out_map);
+    pipe_buffer_unmap(context, dst_transfer);
+
+    *elts = new_elts;
+}
+
+
+/* Ushort indices. */
+
+void util_rebuild_ushort_elts_to_userptr(struct pipe_context *context,
+					 struct pipe_resource *elts,
+					 int index_bias,
+					 unsigned start, unsigned count,
+					 void *out)
+{
+    struct pipe_transfer *in_transfer = NULL;
+    unsigned short *in_map;
+    unsigned short *out_map = out;
+    unsigned i;
 
+    in_map = pipe_buffer_map(context, elts, PIPE_TRANSFER_READ, &in_transfer);
     in_map += start;
 
     for (i = 0; i < count; i++) {
@@ -52,10 +96,7 @@ void util_shorten_ubyte_elts(struct pipe_context *context,
         out_map++;
     }
 
-    pipe_buffer_unmap(context, src_transfer);
-    pipe_buffer_unmap(context, dst_transfer);
-
-    *elts = new_elts;
+    pipe_buffer_unmap(context, in_transfer);
 }
 
 void util_rebuild_ushort_elts(struct pipe_context *context,
@@ -63,33 +104,47 @@ void util_rebuild_ushort_elts(struct pipe_context *context,
 			      int index_bias,
 			      unsigned start, unsigned count)
 {
-    struct pipe_transfer *in_transfer = NULL;
     struct pipe_transfer *out_transfer = NULL;
     struct pipe_resource *new_elts;
-    unsigned short *in_map;
     unsigned short *out_map;
-    unsigned i;
 
     new_elts = pipe_buffer_create(context->screen,
                                   PIPE_BIND_INDEX_BUFFER,
                                   2 * count);
 
-    in_map = pipe_buffer_map(context, *elts,
-                             PIPE_TRANSFER_READ, &in_transfer);
     out_map = pipe_buffer_map(context, new_elts,
                               PIPE_TRANSFER_WRITE, &out_transfer);
+    util_rebuild_ushort_elts_to_userptr(context, *elts, index_bias,
+                                        start, count, out_map);
+    pipe_buffer_unmap(context, out_transfer);
+
+    *elts = new_elts;
+}
+
 
+/* Uint indices. */
+
+void util_rebuild_uint_elts_to_userptr(struct pipe_context *context,
+				       struct pipe_resource *elts,
+				       int index_bias,
+				       unsigned start, unsigned count,
+				       void *out)
+{
+    struct pipe_transfer *in_transfer = NULL;
+    unsigned int *in_map;
+    unsigned int *out_map = out;
+    unsigned i;
+
+    in_map = pipe_buffer_map(context, elts, PIPE_TRANSFER_READ, &in_transfer);
     in_map += start;
+
     for (i = 0; i < count; i++) {
-        *out_map = (unsigned short)(*in_map + index_bias);
+        *out_map = (unsigned int)(*in_map + index_bias);
         in_map++;
         out_map++;
     }
 
     pipe_buffer_unmap(context, in_transfer);
-    pipe_buffer_unmap(context, out_transfer);
-
-    *elts = new_elts;
 }
 
 void util_rebuild_uint_elts(struct pipe_context *context,
@@ -97,30 +152,18 @@ void util_rebuild_uint_elts(struct pipe_context *context,
 			    int index_bias,
 			    unsigned start, unsigned count)
 {
-    struct pipe_transfer *in_transfer = NULL;
     struct pipe_transfer *out_transfer = NULL;
     struct pipe_resource *new_elts;
-    unsigned int *in_map;
     unsigned int *out_map;
-    unsigned i;
 
     new_elts = pipe_buffer_create(context->screen,
                                   PIPE_BIND_INDEX_BUFFER,
                                   2 * count);
 
-    in_map = pipe_buffer_map(context, *elts,
-                             PIPE_TRANSFER_READ, &in_transfer);
     out_map = pipe_buffer_map(context, new_elts,
                               PIPE_TRANSFER_WRITE, &out_transfer);
-
-    in_map += start;
-    for (i = 0; i < count; i++) {
-        *out_map = (unsigned int)(*in_map + index_bias);
-        in_map++;
-        out_map++;
-    }
-
-    pipe_buffer_unmap(context, in_transfer);
+    util_rebuild_uint_elts_to_userptr(context, *elts, index_bias,
+                                      start, count, out_map);
     pipe_buffer_unmap(context, out_transfer);
 
     *elts = new_elts;
diff --git a/src/gallium/auxiliary/util/u_index_modify.h b/src/gallium/auxiliary/util/u_index_modify.h
index 01a6cae94fc..1e9de3dfac8 100644
--- a/src/gallium/auxiliary/util/u_index_modify.h
+++ b/src/gallium/auxiliary/util/u_index_modify.h
@@ -23,19 +23,46 @@
 #ifndef UTIL_INDEX_MODIFY_H
 #define UTIL_INDEX_MODIFY_H
 
+struct pipe_context;
+struct pipe_resource;
+
+void util_shorten_ubyte_elts_to_userptr(struct pipe_context *context,
+					struct pipe_resource *elts,
+					int index_bias,
+					unsigned start,
+					unsigned count,
+					void *out);
+
 void util_shorten_ubyte_elts(struct pipe_context *context,
 			     struct pipe_resource **elts,
 			     int index_bias,
 			     unsigned start,
 			     unsigned count);
 
+
+
+void util_rebuild_ushort_elts_to_userptr(struct pipe_context *context,
+					 struct pipe_resource *elts,
+					 int index_bias,
+					 unsigned start, unsigned count,
+					 void *out);
+
 void util_rebuild_ushort_elts(struct pipe_context *context,
 			      struct pipe_resource **elts,
 			      int index_bias,
 			      unsigned start, unsigned count);
 
+
+
+void util_rebuild_uint_elts_to_userptr(struct pipe_context *context,
+				       struct pipe_resource *elts,
+				       int index_bias,
+				       unsigned start, unsigned count,
+				       void *out);
+
 void util_rebuild_uint_elts(struct pipe_context *context,
 			    struct pipe_resource **elts,
 			    int index_bias,
 			    unsigned start, unsigned count);
+
 #endif
diff --git a/src/gallium/auxiliary/util/u_inlines.h b/src/gallium/auxiliary/util/u_inlines.h
index 9184b6aa4db..b4870bce981 100644
--- a/src/gallium/auxiliary/util/u_inlines.h
+++ b/src/gallium/auxiliary/util/u_inlines.h
@@ -400,6 +400,34 @@ static INLINE boolean util_get_offset(
    }
 }
 
+/**
+ * This function is used to copy an array of pipe_vertex_buffer structures,
+ * while properly referencing the pipe_vertex_buffer::buffer member.
+ *
+ * \sa util_copy_framebuffer_state
+ */
+static INLINE void util_copy_vertex_buffers(struct pipe_vertex_buffer *dst,
+                                            unsigned *dst_count,
+                                            const struct pipe_vertex_buffer *src,
+                                            unsigned src_count)
+{
+   unsigned i;
+
+   /* Reference the buffers of 'src' in 'dst'. */
+   for (i = 0; i < src_count; i++) {
+      pipe_resource_reference(&dst[i].buffer, src[i].buffer);
+   }
+   /* Unreference the rest of the buffers in 'dst'. */
+   for (; i < *dst_count; i++) {
+      pipe_resource_reference(&dst[i].buffer, NULL);
+   }
+
+   /* Update the size of 'dst' and copy over the other members
+    * of pipe_vertex_buffer. */
+   *dst_count = src_count;
+   memcpy(dst, src, src_count * sizeof(struct pipe_vertex_buffer));
+}
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/src/gallium/auxiliary/util/u_upload_mgr.c b/src/gallium/auxiliary/util/u_upload_mgr.c
index 4daa55d6638..3b3d5b418fe 100644
--- a/src/gallium/auxiliary/util/u_upload_mgr.c
+++ b/src/gallium/auxiliary/util/u_upload_mgr.c
@@ -41,22 +41,23 @@
 struct u_upload_mgr {
    struct pipe_context *pipe;
 
-   unsigned default_size;
-   unsigned alignment;
-   unsigned usage;
-
-   /* The active buffer:
-    */
-   struct pipe_resource *buffer;
-   unsigned size;
-   unsigned offset;
+   unsigned default_size;  /* Minimum size of the upload buffer, in bytes. */
+   unsigned alignment;     /* Alignment of each sub-allocation. */
+   unsigned bind;          /* Bitmask of PIPE_BIND_* flags. */
+
+   struct pipe_resource *buffer;   /* Upload buffer. */
+   struct pipe_transfer *transfer; /* Transfer object for the upload buffer. */
+   uint8_t *map;    /* Pointer to the mapped upload buffer. */
+   unsigned size;   /* Actual size of the upload buffer. */
+   unsigned offset; /* Aligned offset to the upload buffer, pointing
+                     * at the first unused byte. */
 };
 
 
 struct u_upload_mgr *u_upload_create( struct pipe_context *pipe,
                                       unsigned default_size,
                                       unsigned alignment,
-                                      unsigned usage )
+                                      unsigned bind )
 {
    struct u_upload_mgr *upload = CALLOC_STRUCT( u_upload_mgr );
    if (!upload)
@@ -65,54 +66,12 @@ struct u_upload_mgr *u_upload_create( struct pipe_context *pipe,
    upload->pipe = pipe;
    upload->default_size = default_size;
    upload->alignment = alignment;
-   upload->usage = usage;
+   upload->bind = bind;
    upload->buffer = NULL;
 
    return upload;
 }
 
-/* Slightly specialized version of buffer_write designed to maximize
- * chances of the driver consolidating successive writes into a single
- * upload.
- *
- * dirty_size may be slightly greater than size to cope with
- * alignment.  We don't want to leave holes between succesively mapped
- * regions as that may prevent the driver from consolidating uploads.
- * 
- * Note that the 'data' pointer has probably come from the application
- * and we cannot read even a byte past its end without risking
- * segfaults, or at least complaints from valgrind..
- */
-static INLINE enum pipe_error
-my_buffer_write(struct pipe_context *pipe,
-                struct pipe_resource *buf,
-                unsigned offset, unsigned size, unsigned dirty_size,
-                const void *data)
-{
-   struct pipe_transfer *transfer = NULL;
-   uint8_t *map;
-   
-   assert(offset < buf->width0);
-   assert(offset + size <= buf->width0);
-   assert(dirty_size >= size);
-   assert(size);
-
-   map = pipe_buffer_map_range(pipe, buf, offset, dirty_size,
-                               PIPE_TRANSFER_WRITE |
-                               PIPE_TRANSFER_FLUSH_EXPLICIT |
-                               PIPE_TRANSFER_DISCARD |
-                               PIPE_TRANSFER_UNSYNCHRONIZED,
-			       &transfer);
-   if (map == NULL) 
-      return PIPE_ERROR_OUT_OF_MEMORY;
-
-   memcpy(map + offset, data, size);
-   pipe_buffer_flush_mapped_range(pipe, transfer, offset, dirty_size);
-   pipe_buffer_unmap(pipe, transfer);
-
-   return PIPE_OK;
-}
-
 /* Release old buffer.
  * 
  * This must usually be called prior to firing the command stream
@@ -124,6 +83,11 @@ my_buffer_write(struct pipe_context *pipe,
  */
 void u_upload_flush( struct u_upload_mgr *upload )
 {
+   /* Unmap and unreference the upload buffer. */
+   if (upload->transfer) {
+      pipe_transfer_unmap(upload->pipe, upload->transfer);
+      upload->transfer = NULL;
+   }
    pipe_resource_reference( &upload->buffer, NULL );
    upload->size = 0;
 }
@@ -142,7 +106,7 @@ u_upload_alloc_buffer( struct u_upload_mgr *upload,
 {
    unsigned size;
 
-   /* Release old buffer, if present:
+   /* Release the old buffer, if present:
     */
    u_upload_flush( upload );
 
@@ -151,10 +115,14 @@ u_upload_alloc_buffer( struct u_upload_mgr *upload,
    size = align(MAX2(upload->default_size, min_size), 4096);
 
    upload->buffer = pipe_buffer_create( upload->pipe->screen,
-                                        upload->usage,
+                                        upload->bind,
                                         size );
    if (upload->buffer == NULL) 
       goto fail;
+
+   /* Map the new buffer. */
+   upload->map = pipe_buffer_map(upload->pipe, upload->buffer,
+                                 PIPE_TRANSFER_WRITE, &upload->transfer);
    
    upload->size = size;
 
@@ -168,38 +136,62 @@ fail:
    return PIPE_ERROR_OUT_OF_MEMORY;
 }
 
-
-enum pipe_error u_upload_data( struct u_upload_mgr *upload,
-                               unsigned size,
-                               const void *data,
-                               unsigned *out_offset,
-                               struct pipe_resource **outbuf )
+enum pipe_error u_upload_alloc( struct u_upload_mgr *upload,
+                                unsigned min_out_offset,
+                                unsigned size,
+                                unsigned *out_offset,
+                                struct pipe_resource **outbuf,
+                                boolean *flushed,
+                                void **ptr )
 {
    unsigned alloc_size = align( size, upload->alignment );
-   enum pipe_error ret = PIPE_OK;
+   unsigned alloc_offset = align(min_out_offset, upload->alignment);
+   unsigned offset;
 
-   if (upload->offset + alloc_size > upload->size) {
-      ret = u_upload_alloc_buffer( upload, alloc_size );
+   /* Make sure we have enough space in the upload buffer
+    * for the sub-allocation. */
+   if (MAX2(upload->offset, alloc_offset) + alloc_size > upload->size) {
+      enum pipe_error ret = u_upload_alloc_buffer(upload,
+                                                  alloc_offset + alloc_size);
       if (ret)
          return ret;
+
+      *flushed = TRUE;
+   } else {
+      *flushed = FALSE;
    }
 
-   /* Copy the data, using map_range if available:
-    */
-   ret = my_buffer_write( upload->pipe, 
-                          upload->buffer,
-                          upload->offset,
-                          size, 
-                          alloc_size,
-                          data );
+   offset = MAX2(upload->offset, alloc_offset);
+
+   assert(offset < upload->buffer->width0);
+   assert(offset + size <= upload->buffer->width0);
+   assert(size);
+
+   /* Emit the return values: */
+   *ptr = upload->map + offset;
+   pipe_resource_reference( outbuf, upload->buffer );
+   *out_offset = offset;
+
+   upload->offset = offset + alloc_size;
+   return PIPE_OK;
+}
+
+enum pipe_error u_upload_data( struct u_upload_mgr *upload,
+                               unsigned min_out_offset,
+                               unsigned size,
+                               const void *data,
+                               unsigned *out_offset,
+                               struct pipe_resource **outbuf,
+                               boolean *flushed )
+{
+   uint8_t *ptr;
+   enum pipe_error ret = u_upload_alloc(upload, min_out_offset, size,
+                                        out_offset, outbuf, flushed,
+                                        (void**)&ptr);
    if (ret)
       return ret;
 
-   /* Emit the return values:
-    */
-   pipe_resource_reference( outbuf, upload->buffer );
-   *out_offset = upload->offset;
-   upload->offset += alloc_size;
+   memcpy(ptr, data, size);
    return PIPE_OK;
 }
 
@@ -210,11 +202,13 @@ enum pipe_error u_upload_data( struct u_upload_mgr *upload,
  * renders or DrawElements calls.
  */
 enum pipe_error u_upload_buffer( struct u_upload_mgr *upload,
+                                 unsigned min_out_offset,
                                  unsigned offset,
                                  unsigned size,
                                  struct pipe_resource *inbuf,
                                  unsigned *out_offset,
-                                 struct pipe_resource **outbuf )
+                                 struct pipe_resource **outbuf,
+                                 boolean *flushed )
 {
    enum pipe_error ret = PIPE_OK;
    struct pipe_transfer *transfer = NULL;
@@ -233,13 +227,12 @@ enum pipe_error u_upload_buffer( struct u_upload_mgr *upload,
    if (0)
       debug_printf("upload ptr %p ofs %d sz %d\n", map, offset, size);
 
-   ret = u_upload_data( upload, 
+   ret = u_upload_data( upload,
+                        min_out_offset,
                         size,
                         map + offset,
                         out_offset,
-                        outbuf );
-   if (ret)
-      goto done;
+                        outbuf, flushed );
 
 done:
    if (map)
diff --git a/src/gallium/auxiliary/util/u_upload_mgr.h b/src/gallium/auxiliary/util/u_upload_mgr.h
index de016df02e0..c9a2ffeb572 100644
--- a/src/gallium/auxiliary/util/u_upload_mgr.h
+++ b/src/gallium/auxiliary/util/u_upload_mgr.h
@@ -32,15 +32,28 @@
 #ifndef U_UPLOAD_MGR_H
 #define U_UPLOAD_MGR_H
 
+#include "pipe/p_compiler.h"
+
 struct pipe_context;
 struct pipe_resource;
 
 
+/**
+ * Create the upload manager.
+ *
+ * \param pipe          Pipe driver.
+ * \param default_size  Minimum size of the upload buffer, in bytes.
+ * \param alignment     Alignment of each suballocation in the upload buffer.
+ * \param bind          Bitmask of PIPE_BIND_* flags.
+ */
 struct u_upload_mgr *u_upload_create( struct pipe_context *pipe,
                                       unsigned default_size,
                                       unsigned alignment,
-                                      unsigned usage );
+                                      unsigned bind );
 
+/**
+ * Destroy the upload manager.
+ */
 void u_upload_destroy( struct u_upload_mgr *upload );
 
 /* Unmap and release old buffer.
@@ -53,20 +66,55 @@ void u_upload_destroy( struct u_upload_mgr *upload );
  */
 void u_upload_flush( struct u_upload_mgr *upload );
 
+/**
+ * Sub-allocate new memory from the upload buffer.
+ *
+ * \param upload           Upload manager
+ * \param min_out_offset   Minimum offset that should be returned in out_offset.
+ * \param size             Size of the allocation.
+ * \param out_offset       Pointer to where the new buffer offset will be returned.
+ * \param outbuf           Pointer to where the upload buffer will be returned.
+ * \param flushed          Whether the upload buffer was flushed.
+ * \param ptr              Pointer to the allocated memory that is returned.
+ */
+enum pipe_error u_upload_alloc( struct u_upload_mgr *upload,
+                                unsigned min_out_offset,
+                                unsigned size,
+                                unsigned *out_offset,
+                                struct pipe_resource **outbuf,
+                                boolean *flushed,
+                                void **ptr );
 
+
+/**
+ * Allocate and write data to the upload buffer.
+ *
+ * Same as u_upload_alloc, but in addition to that, it copies "data"
+ * to the pointer returned from u_upload_alloc.
+ */
 enum pipe_error u_upload_data( struct u_upload_mgr *upload,
+                               unsigned min_out_offset,
                                unsigned size,
                                const void *data,
                                unsigned *out_offset,
-                               struct pipe_resource **outbuf );
+                               struct pipe_resource **outbuf,
+                               boolean *flushed );
 
 
+/**
+ * Allocate and copy an input buffer to the upload buffer.
+ *
+ * Same as u_upload_data, except that the input data comes from a buffer
+ * instead of a user pointer.
+ */
 enum pipe_error u_upload_buffer( struct u_upload_mgr *upload,
+                                 unsigned min_out_offset,
                                  unsigned offset,
                                  unsigned size,
                                  struct pipe_resource *inbuf,
                                  unsigned *out_offset,
-                                 struct pipe_resource **outbuf );
+                                 struct pipe_resource **outbuf,
+                                 boolean *flushed );