Merge remote branch 'origin/master' into pipe-video

Conflicts:
	configure.ac
	src/gallium/drivers/r600/eg_asm.c
	src/gallium/drivers/r600/r600_asm.c
	src/gallium/drivers/r600/r600_asm.h
	src/gallium/include/pipe/p_format.h
	src/gallium/targets/dri-nouveau/Makefile

396 files changed, 24405 insertions, 4969 deletions

diff --git a/src/egl/main/eglcurrent.c b/src/egl/main/eglcurrent.c
index cbca9ff2c2f..4221a9be3e1 100644
--- a/src/egl/main/eglcurrent.c
+++ b/src/egl/main/eglcurrent.c
@@ -286,6 +286,9 @@ _eglError(EGLint errCode, const char *msg)
       case EGL_BAD_SURFACE:
          s = "EGL_BAD_SURFACE";
          break;
+      case EGL_NOT_INITIALIZED:
+         s = "EGL_NOT_INITIALIZED";
+         break;
 #ifdef EGL_MESA_screen_surface
       case EGL_BAD_SCREEN_MESA:
          s = "EGL_BAD_SCREEN_MESA";
@@ -295,7 +298,7 @@ _eglError(EGLint errCode, const char *msg)
          break;
 #endif
       default:
-         s = "other";
+         s = "other EGL error";
       }
       _eglLog(_EGL_DEBUG, "EGL user error 0x%x (%s) in %s\n", errCode, s, msg);
    }
diff --git a/src/gallium/auxiliary/draw/draw_context.c b/src/gallium/auxiliary/draw/draw_context.c
index 73d5b6e403f..e045313b94f 100644
--- a/src/gallium/auxiliary/draw/draw_context.c
+++ b/src/gallium/auxiliary/draw/draw_context.c
@@ -35,6 +35,7 @@
 #include "util/u_memory.h"
 #include "util/u_math.h"
 #include "util/u_cpu_detect.h"
+#include "util/u_inlines.h"
 #include "draw_context.h"
 #include "draw_vs.h"
 #include "draw_gs.h"
@@ -164,6 +165,10 @@ void draw_destroy( struct draw_context *draw )
       }
    }
 
+   for (i = 0; i < draw->pt.nr_vertex_buffers; i++) {
+      pipe_resource_reference(&draw->pt.vertex_buffer[i].buffer, NULL);
+   }
+
    /* Not so fast -- we're just borrowing this at the moment.
     * 
    if (draw->render)
@@ -307,8 +312,9 @@ draw_set_vertex_buffers(struct draw_context *draw,
 {
    assert(count <= PIPE_MAX_ATTRIBS);
 
-   memcpy(draw->pt.vertex_buffer, buffers, count * sizeof(buffers[0]));
-   draw->pt.nr_vertex_buffers = count;
+   util_copy_vertex_buffers(draw->pt.vertex_buffer,
+                            &draw->pt.nr_vertex_buffers,
+                            buffers, count);
 }
 
 
diff --git a/src/gallium/auxiliary/gallivm/lp_bld_const.h b/src/gallium/auxiliary/gallivm/lp_bld_const.h
index 680211fbbd7..69718eb4b3d 100644
--- a/src/gallium/auxiliary/gallivm/lp_bld_const.h
+++ b/src/gallium/auxiliary/gallivm/lp_bld_const.h
@@ -134,7 +134,7 @@ lp_build_const_int_pointer(struct gallivm_state *gallivm, const void *ptr)
 
    /* int type large enough to hold a pointer */
    int_type = LLVMIntTypeInContext(gallivm->context, 8 * sizeof(void *));
-   v = LLVMConstInt(int_type, (unsigned long long) ptr, 0);
+   v = LLVMConstInt(int_type, (uintptr_t) ptr, 0);
    v = LLVMBuildIntToPtr(gallivm->builder, v,
                          LLVMPointerType(int_type, 0),
                          "cast int to ptr");
diff --git a/src/gallium/auxiliary/gallivm/lp_bld_misc.cpp b/src/gallium/auxiliary/gallivm/lp_bld_misc.cpp
index f56ddee7fd7..46dd00d8224 100644
--- a/src/gallium/auxiliary/gallivm/lp_bld_misc.cpp
+++ b/src/gallium/auxiliary/gallivm/lp_bld_misc.cpp
@@ -144,6 +144,7 @@ lp_set_target_options(void)
    llvm::UnsafeFPMath = true;
 #endif
 
+#if HAVE_LLVM < 0x0209
    /*
     * LLVM will generate MMX instructions for vectors <= 64 bits, leading to
     * innefficient code, and in 32bit systems, to the corruption of the FPU
@@ -162,6 +163,7 @@ lp_set_target_options(void)
       llvm::cl::ParseCommandLineOptions(2, const_cast<char**>(options));
       first = FALSE;
    }
+#endif
 
    /*
     * By default LLVM adds a signal handler to output a pretty stack trace.
diff --git a/src/gallium/auxiliary/tgsi/tgsi_dump.c b/src/gallium/auxiliary/tgsi/tgsi_dump.c
index 77bde86684e..82cd8eaa969 100644
--- a/src/gallium/auxiliary/tgsi/tgsi_dump.c
+++ b/src/gallium/auxiliary/tgsi/tgsi_dump.c
@@ -138,7 +138,7 @@ static const char *immediate_type_names[] =
 };
 
 const char *
-tgsi_swizzle_names[] =
+tgsi_swizzle_names[4] =
 {
    "x",
    "y",
@@ -147,7 +147,7 @@ tgsi_swizzle_names[] =
 };
 
 const char *
-tgsi_texture_names[] =
+tgsi_texture_names[TGSI_TEXTURE_COUNT] =
 {
    "UNKNOWN",
    "1D",
@@ -160,16 +160,17 @@ tgsi_texture_names[] =
    "SHADOWRECT"
 };
 
-static const char *property_names[] =
+const char *tgsi_property_names[TGSI_PROPERTY_COUNT] =
 {
    "GS_INPUT_PRIMITIVE",
    "GS_OUTPUT_PRIMITIVE",
    "GS_MAX_OUTPUT_VERTICES",
    "FS_COORD_ORIGIN",
-   "FS_COORD_PIXEL_CENTER"
+   "FS_COORD_PIXEL_CENTER",
+   "FS_COLOR0_WRITES_ALL_CBUFS",
 };
 
-static const char *primitive_names[] =
+const char *tgsi_primitive_names[PIPE_PRIM_MAX] =
 {
    "POINTS",
    "LINES",
@@ -187,13 +188,13 @@ static const char *primitive_names[] =
    "TRIANGLE_STRIP_ADJACENCY"
 };
 
-static const char *fs_coord_origin_names[] =
+const char *tgsi_fs_coord_origin_names[2] =
 {
    "UPPER_LEFT",
    "LOWER_LEFT"
 };
 
-static const char *fs_coord_pixel_center_names[] =
+const char *tgsi_fs_coord_pixel_center_names[2] =
 {
    "HALF_INTEGER",
    "INTEGER"
@@ -484,10 +485,10 @@ iter_property(
    int i;
    struct dump_ctx *ctx = (struct dump_ctx *)iter;
 
-   assert(Elements(property_names) == TGSI_PROPERTY_COUNT);
+   assert(Elements(tgsi_property_names) == TGSI_PROPERTY_COUNT);
 
    TXT( "PROPERTY " );
-   ENM(prop->Property.PropertyName, property_names);
+   ENM(prop->Property.PropertyName, tgsi_property_names);
 
    if (prop->Property.NrTokens > 1)
       TXT(" ");
@@ -496,13 +497,13 @@ iter_property(
       switch (prop->Property.PropertyName) {
       case TGSI_PROPERTY_GS_INPUT_PRIM:
       case TGSI_PROPERTY_GS_OUTPUT_PRIM:
-         ENM(prop->u[i].Data, primitive_names);
+         ENM(prop->u[i].Data, tgsi_primitive_names);
          break;
       case TGSI_PROPERTY_FS_COORD_ORIGIN:
-         ENM(prop->u[i].Data, fs_coord_origin_names);
+         ENM(prop->u[i].Data, tgsi_fs_coord_origin_names);
          break;
       case TGSI_PROPERTY_FS_COORD_PIXEL_CENTER:
-         ENM(prop->u[i].Data, fs_coord_pixel_center_names);
+         ENM(prop->u[i].Data, tgsi_fs_coord_pixel_center_names);
          break;
       default:
          SID( prop->u[i].Data );
diff --git a/src/gallium/auxiliary/tgsi/tgsi_dump.h b/src/gallium/auxiliary/tgsi/tgsi_dump.h
index fc0429ad8d9..2491e9198a2 100644
--- a/src/gallium/auxiliary/tgsi/tgsi_dump.h
+++ b/src/gallium/auxiliary/tgsi/tgsi_dump.h
@@ -29,6 +29,7 @@
 #define TGSI_DUMP_H
 
 #include "pipe/p_compiler.h"
+#include "pipe/p_defines.h"
 #include "pipe/p_shader_tokens.h"
 
 #if defined __cplusplus
@@ -39,10 +40,22 @@ extern const char *
 tgsi_file_names[TGSI_FILE_COUNT];
 
 extern const char *
-tgsi_swizzle_names[];
+tgsi_swizzle_names[4];
 
 extern const char *
-tgsi_texture_names[];
+tgsi_texture_names[TGSI_TEXTURE_COUNT];
+
+extern const char *
+tgsi_property_names[TGSI_PROPERTY_COUNT];
+
+extern const char *
+tgsi_primitive_names[PIPE_PRIM_MAX];
+
+extern const char *
+tgsi_fs_coord_origin_names[2];
+
+extern const char *
+tgsi_fs_coord_pixel_center_names[2];
 
 void
 tgsi_dump_str(
diff --git a/src/gallium/auxiliary/tgsi/tgsi_sanity.h b/src/gallium/auxiliary/tgsi/tgsi_sanity.h
index 73f0f414e3f..707d11ec6dd 100644
--- a/src/gallium/auxiliary/tgsi/tgsi_sanity.h
+++ b/src/gallium/auxiliary/tgsi/tgsi_sanity.h
@@ -28,12 +28,14 @@
 #ifndef TGSI_SANITY_H
 #define TGSI_SANITY_H
 
-#include "pipe/p_shader_tokens.h"
-
 #if defined __cplusplus
 extern "C" {
 #endif
 
+#include "pipe/p_compiler.h"
+
+struct tgsi_token;
+
 /* Check the given token stream for errors and common mistakes.
  * Diagnostic messages are printed out to the debug output, and is
  * controlled by the debug option TGSI_PRINT_SANITY (default false).
diff --git a/src/gallium/auxiliary/tgsi/tgsi_text.c b/src/gallium/auxiliary/tgsi/tgsi_text.c
index 9a38c37979c..819b0725a1f 100644
--- a/src/gallium/auxiliary/tgsi/tgsi_text.c
+++ b/src/gallium/auxiliary/tgsi/tgsi_text.c
@@ -36,6 +36,7 @@
 #include "tgsi_parse.h"
 #include "tgsi_sanity.h"
 #include "tgsi_util.h"
+#include "tgsi_dump.h"
 
 static boolean is_alpha_underscore( const char *cur )
 {
@@ -1258,42 +1259,6 @@ static boolean parse_immediate( struct translate_ctx *ctx )
    return TRUE;
 }
 
-static const char *property_names[] =
-{
-   "GS_INPUT_PRIMITIVE",
-   "GS_OUTPUT_PRIMITIVE",
-   "GS_MAX_OUTPUT_VERTICES",
-   "FS_COORD_ORIGIN",
-   "FS_COORD_PIXEL_CENTER"
-};
-
-static const char *primitive_names[] =
-{
-   "POINTS",
-   "LINES",
-   "LINE_LOOP",
-   "LINE_STRIP",
-   "TRIANGLES",
-   "TRIANGLE_STRIP",
-   "TRIANGLE_FAN",
-   "QUADS",
-   "QUAD_STRIP",
-   "POLYGON"
-};
-
-static const char *fs_coord_origin_names[] =
-{
-   "UPPER_LEFT",
-   "LOWER_LEFT"
-};
-
-static const char *fs_coord_pixel_center_names[] =
-{
-   "HALF_INTEGER",
-   "INTEGER"
-};
-
-
 static boolean
 parse_primitive( const char **pcur, uint *primitive )
 {
@@ -1302,7 +1267,7 @@ parse_primitive( const char **pcur, uint *primitive )
    for (i = 0; i < PIPE_PRIM_MAX; i++) {
       const char *cur = *pcur;
 
-      if (str_match_no_case( &cur, primitive_names[i])) {
+      if (str_match_no_case( &cur, tgsi_primitive_names[i])) {
          *primitive = i;
          *pcur = cur;
          return TRUE;
@@ -1316,10 +1281,10 @@ parse_fs_coord_origin( const char **pcur, uint *fs_coord_origin )
 {
    uint i;
 
-   for (i = 0; i < sizeof(fs_coord_origin_names) / sizeof(fs_coord_origin_names[0]); i++) {
+   for (i = 0; i < Elements(tgsi_fs_coord_origin_names); i++) {
       const char *cur = *pcur;
 
-      if (str_match_no_case( &cur, fs_coord_origin_names[i])) {
+      if (str_match_no_case( &cur, tgsi_fs_coord_origin_names[i])) {
          *fs_coord_origin = i;
          *pcur = cur;
          return TRUE;
@@ -1333,10 +1298,10 @@ parse_fs_coord_pixel_center( const char **pcur, uint *fs_coord_pixel_center )
 {
    uint i;
 
-   for (i = 0; i < sizeof(fs_coord_pixel_center_names) / sizeof(fs_coord_pixel_center_names[0]); i++) {
+   for (i = 0; i < Elements(tgsi_fs_coord_pixel_center_names); i++) {
       const char *cur = *pcur;
 
-      if (str_match_no_case( &cur, fs_coord_pixel_center_names[i])) {
+      if (str_match_no_case( &cur, tgsi_fs_coord_pixel_center_names[i])) {
          *fs_coord_pixel_center = i;
          *pcur = cur;
          return TRUE;
@@ -1364,7 +1329,7 @@ static boolean parse_property( struct translate_ctx *ctx )
    }
    for (property_name = 0; property_name < TGSI_PROPERTY_COUNT;
         ++property_name) {
-      if (streq_nocase_uprcase(property_names[property_name], id)) {
+      if (streq_nocase_uprcase(tgsi_property_names[property_name], id)) {
          break;
       }
    }
@@ -1398,6 +1363,7 @@ static boolean parse_property( struct translate_ctx *ctx )
          return FALSE;
       }
       break;
+   case TGSI_PROPERTY_FS_COLOR0_WRITES_ALL_CBUFS:
    default:
       if (!parse_uint(&ctx->cur, &values[0] )) {
          report_error( ctx, "Expected unsigned integer as property!" );
diff --git a/src/gallium/auxiliary/tgsi/tgsi_text.h b/src/gallium/auxiliary/tgsi/tgsi_text.h
index 8eeeeef1402..5fe53cf007d 100644
--- a/src/gallium/auxiliary/tgsi/tgsi_text.h
+++ b/src/gallium/auxiliary/tgsi/tgsi_text.h
@@ -28,12 +28,14 @@
 #ifndef TGSI_TEXT_H
 #define TGSI_TEXT_H
 
-#include "pipe/p_shader_tokens.h"
-
 #if defined __cplusplus
 extern "C" {
 #endif
 
+#include "pipe/p_compiler.h"
+
+struct tgsi_token;
+
 boolean
 tgsi_text_translate(
    const char *text,
diff --git a/src/gallium/auxiliary/tgsi/tgsi_ureg.c b/src/gallium/auxiliary/tgsi/tgsi_ureg.c
index 7d13a17bdbc..02de12d77d5 100644
--- a/src/gallium/auxiliary/tgsi/tgsi_ureg.c
+++ b/src/gallium/auxiliary/tgsi/tgsi_ureg.c
@@ -148,6 +148,7 @@ struct ureg_program
    unsigned property_gs_max_vertices;
    unsigned char property_fs_coord_origin; /* = TGSI_FS_COORD_ORIGIN_* */
    unsigned char property_fs_coord_pixel_center; /* = TGSI_FS_COORD_PIXEL_CENTER_* */
+   unsigned char property_fs_color0_writes_all_cbufs; /* = TGSI_FS_COLOR0_WRITES_ALL_CBUFS * */
 
    unsigned nr_addrs;
    unsigned nr_preds;
@@ -284,7 +285,12 @@ ureg_property_fs_coord_pixel_center(struct ureg_program *ureg,
    ureg->property_fs_coord_pixel_center = fs_coord_pixel_center;
 }
 
-
+void
+ureg_property_fs_color0_writes_all_cbufs(struct ureg_program *ureg,
+                            unsigned fs_color0_writes_all_cbufs)
+{
+   ureg->property_fs_color0_writes_all_cbufs = fs_color0_writes_all_cbufs;
+}
 
 struct ureg_src
 ureg_DECL_fs_input_cyl_centroid(struct ureg_program *ureg,
@@ -1278,6 +1284,14 @@ static void emit_decls( struct ureg_program *ureg )
                     ureg->property_fs_coord_pixel_center);
    }
 
+   if (ureg->property_fs_color0_writes_all_cbufs) {
+      assert(ureg->processor == TGSI_PROCESSOR_FRAGMENT);
+
+      emit_property(ureg,
+                    TGSI_PROPERTY_FS_COLOR0_WRITES_ALL_CBUFS,
+                    ureg->property_fs_color0_writes_all_cbufs);
+   }
+
    if (ureg->processor == TGSI_PROCESSOR_VERTEX) {
       for (i = 0; i < UREG_MAX_INPUT; i++) {
          if (ureg->vs_inputs[i/32] & (1 << (i%32))) {
diff --git a/src/gallium/auxiliary/tgsi/tgsi_ureg.h b/src/gallium/auxiliary/tgsi/tgsi_ureg.h
index acc463200a6..807128a5e52 100644
--- a/src/gallium/auxiliary/tgsi/tgsi_ureg.h
+++ b/src/gallium/auxiliary/tgsi/tgsi_ureg.h
@@ -153,6 +153,10 @@ void
 ureg_property_fs_coord_pixel_center(struct ureg_program *ureg,
                             unsigned fs_coord_pixel_center);
 
+void
+ureg_property_fs_color0_writes_all_cbufs(struct ureg_program *ureg,
+                            unsigned fs_color0_writes_all_cbufs);
+
 /***********************************************************************
  * Build shader declarations:
  */
diff --git a/src/gallium/auxiliary/util/u_blitter.h b/src/gallium/auxiliary/util/u_blitter.h
index c5660cf2d00..922a8580ac1 100644
--- a/src/gallium/auxiliary/util/u_blitter.h
+++ b/src/gallium/auxiliary/util/u_blitter.h
@@ -318,21 +318,13 @@ util_blitter_save_vertex_buffers(struct blitter_context *blitter,
                                          int num_vertex_buffers,
                                          struct pipe_vertex_buffer *vertex_buffers)
 {
-   unsigned i;
    assert(num_vertex_buffers <= Elements(blitter->saved_vertex_buffers));
 
-   blitter->saved_num_vertex_buffers = num_vertex_buffers;
-
-   for (i = 0; i < num_vertex_buffers; i++) {
-      if (vertex_buffers[i].buffer) {
-         pipe_resource_reference(&blitter->saved_vertex_buffers[i].buffer,
-                                 vertex_buffers[i].buffer);
-      }
-   }
-
-   memcpy(blitter->saved_vertex_buffers,
-          vertex_buffers,
-          num_vertex_buffers * sizeof(struct pipe_vertex_buffer));
+   blitter->saved_num_vertex_buffers = 0;
+   util_copy_vertex_buffers(blitter->saved_vertex_buffers,
+                            (unsigned*)&blitter->saved_num_vertex_buffers,
+                            vertex_buffers,
+                            num_vertex_buffers);
 }
 
 #ifdef __cplusplus
diff --git a/src/gallium/auxiliary/util/u_format.csv b/src/gallium/auxiliary/util/u_format.csv
index 8e5d4487a67..0ae15c9cc0a 100644
--- a/src/gallium/auxiliary/util/u_format.csv
+++ b/src/gallium/auxiliary/util/u_format.csv
@@ -76,6 +76,7 @@ PIPE_FORMAT_B4G4R4X4_UNORM        , plain, 1, 1, un4 , un4 , un4 , x4  , zyx1, r
 PIPE_FORMAT_B5G6R5_UNORM          , plain, 1, 1, un5 , un6 , un5 ,     , zyx1, rgb
 PIPE_FORMAT_R10G10B10A2_UNORM     , plain, 1, 1, un10, un10, un10, un2 , xyzw, rgb
 PIPE_FORMAT_B10G10R10A2_UNORM     , plain, 1, 1, un10, un10, un10, un2 , zyxw, rgb
+PIPE_FORMAT_B2G3R3_UNORM          , plain, 1, 1, un2 , un3 , un3 ,     , zyx1, rgb
 
 # Luminance/Intensity/Alpha formats
 PIPE_FORMAT_L8_UNORM              , plain, 1, 1, un8 ,     ,     ,     , xxx1, rgb
@@ -84,6 +85,9 @@ PIPE_FORMAT_I8_UNORM              , plain, 1, 1, un8 ,     ,     ,     , xxxx, r
 PIPE_FORMAT_L4A4_UNORM            , plain, 1, 1, un4 , un4 ,     ,     , xxxy, rgb
 PIPE_FORMAT_L8A8_UNORM            , plain, 1, 1, un8 , un8 ,     ,     , xxxy, rgb
 PIPE_FORMAT_L16_UNORM             , plain, 1, 1, un16,     ,     ,     , xxx1, rgb
+PIPE_FORMAT_A16_UNORM             , plain, 1, 1, un16,     ,     ,     , 000x, rgb
+PIPE_FORMAT_I16_UNORM             , plain, 1, 1, un16,     ,     ,     , xxxx, rgb
+PIPE_FORMAT_L16A16_UNORM          , plain, 1, 1, un16, un16,     ,     , xxxy, rgb
 
 # SRGB formats
 PIPE_FORMAT_L8_SRGB               , plain, 1, 1, un8 ,     ,     ,     , xxx1, srgb 
diff --git a/src/gallium/auxiliary/util/u_index_modify.c b/src/gallium/auxiliary/util/u_index_modify.c
index 3822f60e71d..f2c9db3caf1 100644
--- a/src/gallium/auxiliary/util/u_index_modify.c
+++ b/src/gallium/auxiliary/util/u_index_modify.c
@@ -24,26 +24,70 @@
 #include "util/u_index_modify.h"
 #include "util/u_inlines.h"
 
+/* Ubyte indices. */
+
+void util_shorten_ubyte_elts_to_userptr(struct pipe_context *context,
+					struct pipe_resource *elts,
+					int index_bias,
+					unsigned start,
+					unsigned count,
+					void *out)
+{
+    struct pipe_transfer *src_transfer;
+    unsigned char *in_map;
+    unsigned short *out_map = out;
+    unsigned i;
+
+    in_map = pipe_buffer_map(context, elts, PIPE_TRANSFER_READ, &src_transfer);
+    in_map += start;
+
+    for (i = 0; i < count; i++) {
+        *out_map = (unsigned short)(*in_map + index_bias);
+        in_map++;
+        out_map++;
+    }
+
+    pipe_buffer_unmap(context, src_transfer);
+}
+
 void util_shorten_ubyte_elts(struct pipe_context *context,
 			     struct pipe_resource **elts,
 			     int index_bias,
 			     unsigned start,
 			     unsigned count)
 {
-    struct pipe_screen* screen = context->screen;
     struct pipe_resource* new_elts;
-    unsigned char *in_map;
     unsigned short *out_map;
-    struct pipe_transfer *src_transfer, *dst_transfer;
-    unsigned i;
+    struct pipe_transfer *dst_transfer;
 
-    new_elts = pipe_buffer_create(screen,
+    new_elts = pipe_buffer_create(context->screen,
                                   PIPE_BIND_INDEX_BUFFER,
                                   2 * count);
 
-    in_map = pipe_buffer_map(context, *elts, PIPE_TRANSFER_READ, &src_transfer);
-    out_map = pipe_buffer_map(context, new_elts, PIPE_TRANSFER_WRITE, &dst_transfer);
+    out_map = pipe_buffer_map(context, new_elts, PIPE_TRANSFER_WRITE,
+                              &dst_transfer);
+    util_shorten_ubyte_elts_to_userptr(context, *elts, index_bias,
+                                       start, count, out_map);
+    pipe_buffer_unmap(context, dst_transfer);
+
+    *elts = new_elts;
+}
+
+
+/* Ushort indices. */
+
+void util_rebuild_ushort_elts_to_userptr(struct pipe_context *context,
+					 struct pipe_resource *elts,
+					 int index_bias,
+					 unsigned start, unsigned count,
+					 void *out)
+{
+    struct pipe_transfer *in_transfer = NULL;
+    unsigned short *in_map;
+    unsigned short *out_map = out;
+    unsigned i;
 
+    in_map = pipe_buffer_map(context, elts, PIPE_TRANSFER_READ, &in_transfer);
     in_map += start;
 
     for (i = 0; i < count; i++) {
@@ -52,10 +96,7 @@ void util_shorten_ubyte_elts(struct pipe_context *context,
         out_map++;
     }
 
-    pipe_buffer_unmap(context, src_transfer);
-    pipe_buffer_unmap(context, dst_transfer);
-
-    *elts = new_elts;
+    pipe_buffer_unmap(context, in_transfer);
 }
 
 void util_rebuild_ushort_elts(struct pipe_context *context,
@@ -63,33 +104,47 @@ void util_rebuild_ushort_elts(struct pipe_context *context,
 			      int index_bias,
 			      unsigned start, unsigned count)
 {
-    struct pipe_transfer *in_transfer = NULL;
     struct pipe_transfer *out_transfer = NULL;
     struct pipe_resource *new_elts;
-    unsigned short *in_map;
     unsigned short *out_map;
-    unsigned i;
 
     new_elts = pipe_buffer_create(context->screen,
                                   PIPE_BIND_INDEX_BUFFER,
                                   2 * count);
 
-    in_map = pipe_buffer_map(context, *elts,
-                             PIPE_TRANSFER_READ, &in_transfer);
     out_map = pipe_buffer_map(context, new_elts,
                               PIPE_TRANSFER_WRITE, &out_transfer);
+    util_rebuild_ushort_elts_to_userptr(context, *elts, index_bias,
+                                        start, count, out_map);
+    pipe_buffer_unmap(context, out_transfer);
+
+    *elts = new_elts;
+}
+
 
+/* Uint indices. */
+
+void util_rebuild_uint_elts_to_userptr(struct pipe_context *context,
+				       struct pipe_resource *elts,
+				       int index_bias,
+				       unsigned start, unsigned count,
+				       void *out)
+{
+    struct pipe_transfer *in_transfer = NULL;
+    unsigned int *in_map;
+    unsigned int *out_map = out;
+    unsigned i;
+
+    in_map = pipe_buffer_map(context, elts, PIPE_TRANSFER_READ, &in_transfer);
     in_map += start;
+
     for (i = 0; i < count; i++) {
-        *out_map = (unsigned short)(*in_map + index_bias);
+        *out_map = (unsigned int)(*in_map + index_bias);
         in_map++;
         out_map++;
     }
 
     pipe_buffer_unmap(context, in_transfer);
-    pipe_buffer_unmap(context, out_transfer);
-
-    *elts = new_elts;
 }
 
 void util_rebuild_uint_elts(struct pipe_context *context,
@@ -97,30 +152,18 @@ void util_rebuild_uint_elts(struct pipe_context *context,
 			    int index_bias,
 			    unsigned start, unsigned count)
 {
-    struct pipe_transfer *in_transfer = NULL;
     struct pipe_transfer *out_transfer = NULL;
     struct pipe_resource *new_elts;
-    unsigned int *in_map;
     unsigned int *out_map;
-    unsigned i;
 
     new_elts = pipe_buffer_create(context->screen,
                                   PIPE_BIND_INDEX_BUFFER,
                                   2 * count);
 
-    in_map = pipe_buffer_map(context, *elts,
-                             PIPE_TRANSFER_READ, &in_transfer);
     out_map = pipe_buffer_map(context, new_elts,
                               PIPE_TRANSFER_WRITE, &out_transfer);
-
-    in_map += start;
-    for (i = 0; i < count; i++) {
-        *out_map = (unsigned int)(*in_map + index_bias);
-        in_map++;
-        out_map++;
-    }
-
-    pipe_buffer_unmap(context, in_transfer);
+    util_rebuild_uint_elts_to_userptr(context, *elts, index_bias,
+                                      start, count, out_map);
     pipe_buffer_unmap(context, out_transfer);
 
     *elts = new_elts;
diff --git a/src/gallium/auxiliary/util/u_index_modify.h b/src/gallium/auxiliary/util/u_index_modify.h
index 01a6cae94fc..1e9de3dfac8 100644
--- a/src/gallium/auxiliary/util/u_index_modify.h
+++ b/src/gallium/auxiliary/util/u_index_modify.h
@@ -23,19 +23,46 @@
 #ifndef UTIL_INDEX_MODIFY_H
 #define UTIL_INDEX_MODIFY_H
 
+struct pipe_context;
+struct pipe_resource;
+
+void util_shorten_ubyte_elts_to_userptr(struct pipe_context *context,
+					struct pipe_resource *elts,
+					int index_bias,
+					unsigned start,
+					unsigned count,
+					void *out);
+
 void util_shorten_ubyte_elts(struct pipe_context *context,
 			     struct pipe_resource **elts,
 			     int index_bias,
 			     unsigned start,
 			     unsigned count);
 
+
+
+void util_rebuild_ushort_elts_to_userptr(struct pipe_context *context,
+					 struct pipe_resource *elts,
+					 int index_bias,
+					 unsigned start, unsigned count,
+					 void *out);
+
 void util_rebuild_ushort_elts(struct pipe_context *context,
 			      struct pipe_resource **elts,
 			      int index_bias,
 			      unsigned start, unsigned count);
 
+
+
+void util_rebuild_uint_elts_to_userptr(struct pipe_context *context,
+				       struct pipe_resource *elts,
+				       int index_bias,
+				       unsigned start, unsigned count,
+				       void *out);
+
 void util_rebuild_uint_elts(struct pipe_context *context,
 			    struct pipe_resource **elts,
 			    int index_bias,
 			    unsigned start, unsigned count);
+
 #endif
diff --git a/src/gallium/auxiliary/util/u_inlines.h b/src/gallium/auxiliary/util/u_inlines.h
index 9184b6aa4db..b4870bce981 100644
--- a/src/gallium/auxiliary/util/u_inlines.h
+++ b/src/gallium/auxiliary/util/u_inlines.h
@@ -400,6 +400,34 @@ static INLINE boolean util_get_offset(
    }
 }
 
+/**
+ * This function is used to copy an array of pipe_vertex_buffer structures,
+ * while properly referencing the pipe_vertex_buffer::buffer member.
+ *
+ * \sa util_copy_framebuffer_state
+ */
+static INLINE void util_copy_vertex_buffers(struct pipe_vertex_buffer *dst,
+                                            unsigned *dst_count,
+                                            const struct pipe_vertex_buffer *src,
+                                            unsigned src_count)
+{
+   unsigned i;
+
+   /* Reference the buffers of 'src' in 'dst'. */
+   for (i = 0; i < src_count; i++) {
+      pipe_resource_reference(&dst[i].buffer, src[i].buffer);
+   }
+   /* Unreference the rest of the buffers in 'dst'. */
+   for (; i < *dst_count; i++) {
+      pipe_resource_reference(&dst[i].buffer, NULL);
+   }
+
+   /* Update the size of 'dst' and copy over the other members
+    * of pipe_vertex_buffer. */
+   *dst_count = src_count;
+   memcpy(dst, src, src_count * sizeof(struct pipe_vertex_buffer));
+}
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/src/gallium/auxiliary/util/u_upload_mgr.c b/src/gallium/auxiliary/util/u_upload_mgr.c
index 4daa55d6638..3b3d5b418fe 100644
--- a/src/gallium/auxiliary/util/u_upload_mgr.c
+++ b/src/gallium/auxiliary/util/u_upload_mgr.c
@@ -41,22 +41,23 @@
 struct u_upload_mgr {
    struct pipe_context *pipe;
 
-   unsigned default_size;
-   unsigned alignment;
-   unsigned usage;
-
-   /* The active buffer:
-    */
-   struct pipe_resource *buffer;
-   unsigned size;
-   unsigned offset;
+   unsigned default_size;  /* Minimum size of the upload buffer, in bytes. */
+   unsigned alignment;     /* Alignment of each sub-allocation. */
+   unsigned bind;          /* Bitmask of PIPE_BIND_* flags. */
+
+   struct pipe_resource *buffer;   /* Upload buffer. */
+   struct pipe_transfer *transfer; /* Transfer object for the upload buffer. */
+   uint8_t *map;    /* Pointer to the mapped upload buffer. */
+   unsigned size;   /* Actual size of the upload buffer. */
+   unsigned offset; /* Aligned offset to the upload buffer, pointing
+                     * at the first unused byte. */
 };
 
 
 struct u_upload_mgr *u_upload_create( struct pipe_context *pipe,
                                       unsigned default_size,
                                       unsigned alignment,
-                                      unsigned usage )
+                                      unsigned bind )
 {
    struct u_upload_mgr *upload = CALLOC_STRUCT( u_upload_mgr );
    if (!upload)
@@ -65,54 +66,12 @@ struct u_upload_mgr *u_upload_create( struct pipe_context *pipe,
    upload->pipe = pipe;
    upload->default_size = default_size;
    upload->alignment = alignment;
-   upload->usage = usage;
+   upload->bind = bind;
    upload->buffer = NULL;
 
    return upload;
 }
 
-/* Slightly specialized version of buffer_write designed to maximize
- * chances of the driver consolidating successive writes into a single
- * upload.
- *
- * dirty_size may be slightly greater than size to cope with
- * alignment.  We don't want to leave holes between succesively mapped
- * regions as that may prevent the driver from consolidating uploads.
- * 
- * Note that the 'data' pointer has probably come from the application
- * and we cannot read even a byte past its end without risking
- * segfaults, or at least complaints from valgrind..
- */
-static INLINE enum pipe_error
-my_buffer_write(struct pipe_context *pipe,
-                struct pipe_resource *buf,
-                unsigned offset, unsigned size, unsigned dirty_size,
-                const void *data)
-{
-   struct pipe_transfer *transfer = NULL;
-   uint8_t *map;
-   
-   assert(offset < buf->width0);
-   assert(offset + size <= buf->width0);
-   assert(dirty_size >= size);
-   assert(size);
-
-   map = pipe_buffer_map_range(pipe, buf, offset, dirty_size,
-                               PIPE_TRANSFER_WRITE |
-                               PIPE_TRANSFER_FLUSH_EXPLICIT |
-                               PIPE_TRANSFER_DISCARD |
-                               PIPE_TRANSFER_UNSYNCHRONIZED,
-			       &transfer);
-   if (map == NULL) 
-      return PIPE_ERROR_OUT_OF_MEMORY;
-
-   memcpy(map + offset, data, size);
-   pipe_buffer_flush_mapped_range(pipe, transfer, offset, dirty_size);
-   pipe_buffer_unmap(pipe, transfer);
-
-   return PIPE_OK;
-}
-
 /* Release old buffer.
  * 
  * This must usually be called prior to firing the command stream
@@ -124,6 +83,11 @@ my_buffer_write(struct pipe_context *pipe,
  */
 void u_upload_flush( struct u_upload_mgr *upload )
 {
+   /* Unmap and unreference the upload buffer. */
+   if (upload->transfer) {
+      pipe_transfer_unmap(upload->pipe, upload->transfer);
+      upload->transfer = NULL;
+   }
    pipe_resource_reference( &upload->buffer, NULL );
    upload->size = 0;
 }
@@ -142,7 +106,7 @@ u_upload_alloc_buffer( struct u_upload_mgr *upload,
 {
    unsigned size;
 
-   /* Release old buffer, if present:
+   /* Release the old buffer, if present:
     */
    u_upload_flush( upload );
 
@@ -151,10 +115,14 @@ u_upload_alloc_buffer( struct u_upload_mgr *upload,
    size = align(MAX2(upload->default_size, min_size), 4096);
 
    upload->buffer = pipe_buffer_create( upload->pipe->screen,
-                                        upload->usage,
+                                        upload->bind,
                                         size );
    if (upload->buffer == NULL) 
       goto fail;
+
+   /* Map the new buffer. */
+   upload->map = pipe_buffer_map(upload->pipe, upload->buffer,
+                                 PIPE_TRANSFER_WRITE, &upload->transfer);
    
    upload->size = size;
 
@@ -168,38 +136,62 @@ fail:
    return PIPE_ERROR_OUT_OF_MEMORY;
 }
 
-
-enum pipe_error u_upload_data( struct u_upload_mgr *upload,
-                               unsigned size,
-                               const void *data,
-                               unsigned *out_offset,
-                               struct pipe_resource **outbuf )
+enum pipe_error u_upload_alloc( struct u_upload_mgr *upload,
+                                unsigned min_out_offset,
+                                unsigned size,
+                                unsigned *out_offset,
+                                struct pipe_resource **outbuf,
+                                boolean *flushed,
+                                void **ptr )
 {
    unsigned alloc_size = align( size, upload->alignment );
-   enum pipe_error ret = PIPE_OK;
+   unsigned alloc_offset = align(min_out_offset, upload->alignment);
+   unsigned offset;
 
-   if (upload->offset + alloc_size > upload->size) {
-      ret = u_upload_alloc_buffer( upload, alloc_size );
+   /* Make sure we have enough space in the upload buffer
+    * for the sub-allocation. */
+   if (MAX2(upload->offset, alloc_offset) + alloc_size > upload->size) {
+      enum pipe_error ret = u_upload_alloc_buffer(upload,
+                                                  alloc_offset + alloc_size);
       if (ret)
          return ret;
+
+      *flushed = TRUE;
+   } else {
+      *flushed = FALSE;
    }
 
-   /* Copy the data, using map_range if available:
-    */
-   ret = my_buffer_write( upload->pipe, 
-                          upload->buffer,
-                          upload->offset,
-                          size, 
-                          alloc_size,
-                          data );
+   offset = MAX2(upload->offset, alloc_offset);
+
+   assert(offset < upload->buffer->width0);
+   assert(offset + size <= upload->buffer->width0);
+   assert(size);
+
+   /* Emit the return values: */
+   *ptr = upload->map + offset;
+   pipe_resource_reference( outbuf, upload->buffer );
+   *out_offset = offset;
+
+   upload->offset = offset + alloc_size;
+   return PIPE_OK;
+}
+
+enum pipe_error u_upload_data( struct u_upload_mgr *upload,
+                               unsigned min_out_offset,
+                               unsigned size,
+                               const void *data,
+                               unsigned *out_offset,
+                               struct pipe_resource **outbuf,
+                               boolean *flushed )
+{
+   uint8_t *ptr;
+   enum pipe_error ret = u_upload_alloc(upload, min_out_offset, size,
+                                        out_offset, outbuf, flushed,
+                                        (void**)&ptr);
    if (ret)
       return ret;
 
-   /* Emit the return values:
-    */
-   pipe_resource_reference( outbuf, upload->buffer );
-   *out_offset = upload->offset;
-   upload->offset += alloc_size;
+   memcpy(ptr, data, size);
    return PIPE_OK;
 }
 
@@ -210,11 +202,13 @@ enum pipe_error u_upload_data( struct u_upload_mgr *upload,
  * renders or DrawElements calls.
  */
 enum pipe_error u_upload_buffer( struct u_upload_mgr *upload,
+                                 unsigned min_out_offset,
                                  unsigned offset,
                                  unsigned size,
                                  struct pipe_resource *inbuf,
                                  unsigned *out_offset,
-                                 struct pipe_resource **outbuf )
+                                 struct pipe_resource **outbuf,
+                                 boolean *flushed )
 {
    enum pipe_error ret = PIPE_OK;
    struct pipe_transfer *transfer = NULL;
@@ -233,13 +227,12 @@ enum pipe_error u_upload_buffer( struct u_upload_mgr *upload,
    if (0)
       debug_printf("upload ptr %p ofs %d sz %d\n", map, offset, size);
 
-   ret = u_upload_data( upload, 
+   ret = u_upload_data( upload,
+                        min_out_offset,
                         size,
                         map + offset,
                         out_offset,
-                        outbuf );
-   if (ret)
-      goto done;
+                        outbuf, flushed );
 
 done:
    if (map)
diff --git a/src/gallium/auxiliary/util/u_upload_mgr.h b/src/gallium/auxiliary/util/u_upload_mgr.h
index de016df02e0..c9a2ffeb572 100644
--- a/src/gallium/auxiliary/util/u_upload_mgr.h
+++ b/src/gallium/auxiliary/util/u_upload_mgr.h
@@ -32,15 +32,28 @@
 #ifndef U_UPLOAD_MGR_H
 #define U_UPLOAD_MGR_H
 
+#include "pipe/p_compiler.h"
+
 struct pipe_context;
 struct pipe_resource;
 
 
+/**
+ * Create the upload manager.
+ *
+ * \param pipe          Pipe driver.
+ * \param default_size  Minimum size of the upload buffer, in bytes.
+ * \param alignment     Alignment of each suballocation in the upload buffer.
+ * \param bind          Bitmask of PIPE_BIND_* flags.
+ */
 struct u_upload_mgr *u_upload_create( struct pipe_context *pipe,
                                       unsigned default_size,
                                       unsigned alignment,
-                                      unsigned usage );
+                                      unsigned bind );
 
+/**
+ * Destroy the upload manager.
+ */
 void u_upload_destroy( struct u_upload_mgr *upload );
 
 /* Unmap and release old buffer.
@@ -53,20 +66,55 @@ void u_upload_destroy( struct u_upload_mgr *upload );
  */
 void u_upload_flush( struct u_upload_mgr *upload );
 
+/**
+ * Sub-allocate new memory from the upload buffer.
+ *
+ * \param upload           Upload manager
+ * \param min_out_offset   Minimum offset that should be returned in out_offset.
+ * \param size             Size of the allocation.
+ * \param out_offset       Pointer to where the new buffer offset will be returned.
+ * \param outbuf           Pointer to where the upload buffer will be returned.
+ * \param flushed          Whether the upload buffer was flushed.
+ * \param ptr              Pointer to the allocated memory that is returned.
+ */
+enum pipe_error u_upload_alloc( struct u_upload_mgr *upload,
+                                unsigned min_out_offset,
+                                unsigned size,
+                                unsigned *out_offset,
+                                struct pipe_resource **outbuf,
+                                boolean *flushed,
+                                void **ptr );
 
+
+/**
+ * Allocate and write data to the upload buffer.
+ *
+ * Same as u_upload_alloc, but in addition to that, it copies "data"
+ * to the pointer returned from u_upload_alloc.
+ */
 enum pipe_error u_upload_data( struct u_upload_mgr *upload,
+                               unsigned min_out_offset,
                                unsigned size,
                                const void *data,
                                unsigned *out_offset,
-                               struct pipe_resource **outbuf );
+                               struct pipe_resource **outbuf,
+                               boolean *flushed );
 
 
+/**
+ * Allocate and copy an input buffer to the upload buffer.
+ *
+ * Same as u_upload_data, except that the input data comes from a buffer
+ * instead of a user pointer.
+ */
 enum pipe_error u_upload_buffer( struct u_upload_mgr *upload,
+                                 unsigned min_out_offset,
                                  unsigned offset,
                                  unsigned size,
                                  struct pipe_resource *inbuf,
                                  unsigned *out_offset,
-                                 struct pipe_resource **outbuf );
+                                 struct pipe_resource **outbuf,
+                                 boolean *flushed );
 
 
 
diff --git a/src/gallium/docs/source/tgsi.rst b/src/gallium/docs/source/tgsi.rst
index 7eb6bd05d7b..d986e6601e6 100644
--- a/src/gallium/docs/source/tgsi.rst
+++ b/src/gallium/docs/source/tgsi.rst
@@ -1516,6 +1516,11 @@ GL_ARB_fragment_coord_conventions extension.
 DirectX 9 uses INTEGER.
 DirectX 10 uses HALF_INTEGER.
 
+FS_COLOR0_WRITES_ALL_CBUFS
+""""""""""""""""""""""""""
+Specifies that writes to the fragment shader color 0 are replicated to all
+bound cbufs. This facilitates OpenGL's fragColor output vs fragData[0] where
+fragData is directed to a single color buffer, but fragColor is broadcast.
 
 
 Texture Sampling and Texture Formats
diff --git a/src/gallium/drivers/cell/ppu/cell_context.c b/src/gallium/drivers/cell/ppu/cell_context.c
index b6b3a700cda..f9b83c8666c 100644
--- a/src/gallium/drivers/cell/ppu/cell_context.c
+++ b/src/gallium/drivers/cell/ppu/cell_context.c
@@ -37,6 +37,7 @@
 #include "pipe/p_format.h"
 #include "util/u_memory.h"
 #include "pipe/p_screen.h"
+#include "util/u_inlines.h"
 
 #include "draw/draw_context.h"
 #include "draw/draw_private.h"
@@ -61,6 +62,11 @@ static void
 cell_destroy_context( struct pipe_context *pipe )
 {
    struct cell_context *cell = cell_context(pipe);
+   unsigned i;
+
+   for (i = 0; i < cell->num_vertex_buffers; i++) {
+      pipe_resource_reference(&cell->vertex_buffer[i].buffer, NULL);
+   }
 
    util_delete_keymap(cell->fragment_ops_cache, NULL);
 
diff --git a/src/gallium/drivers/cell/ppu/cell_state_vertex.c b/src/gallium/drivers/cell/ppu/cell_state_vertex.c
index a065d68b5a6..eb22a09a913 100644
--- a/src/gallium/drivers/cell/ppu/cell_state_vertex.c
+++ b/src/gallium/drivers/cell/ppu/cell_state_vertex.c
@@ -82,8 +82,9 @@ cell_set_vertex_buffers(struct pipe_context *pipe,
 
    assert(count <= PIPE_MAX_ATTRIBS);
 
-   memcpy(cell->vertex_buffer, buffers, count * sizeof(buffers[0]));
-   cell->num_vertex_buffers = count;
+   util_copy_vertex_buffers(cell->vertex_buffer,
+                            &cell->num_vertex_buffers,
+                            buffers, count);
 
    cell->dirty |= CELL_NEW_VERTEX;
 
diff --git a/src/gallium/drivers/failover/fo_context.c b/src/gallium/drivers/failover/fo_context.c
index e4d289c8a4d..d60718d9716 100644
--- a/src/gallium/drivers/failover/fo_context.c
+++ b/src/gallium/drivers/failover/fo_context.c
@@ -29,6 +29,7 @@
 #include "pipe/p_defines.h"
 #include "util/u_memory.h"
 #include "pipe/p_context.h"
+#include "util/u_inlines.h"
 
 #include "fo_context.h"
 #include "fo_winsys.h"
@@ -38,6 +39,11 @@
 static void failover_destroy( struct pipe_context *pipe )
 {
    struct failover_context *failover = failover_context( pipe );
+   unsigned i;
+
+   for (i = 0; i < failover->num_vertex_buffers; i++) {
+      pipe_resource_reference(&failover->vertex_buffers[i].buffer, NULL);
+   }
 
    FREE( failover );
 }
diff --git a/src/gallium/drivers/failover/fo_state.c b/src/gallium/drivers/failover/fo_state.c
index c265f381b67..af1fd953aaf 100644
--- a/src/gallium/drivers/failover/fo_state.c
+++ b/src/gallium/drivers/failover/fo_state.c
@@ -574,10 +574,10 @@ failover_set_vertex_buffers(struct pipe_context *pipe,
 {
    struct failover_context *failover = failover_context(pipe);
 
-   memcpy(failover->vertex_buffers, vertex_buffers,
-          count * sizeof(vertex_buffers[0]));
+   util_copy_vertex_buffers(failover->vertex_buffers,
+                            &failover->num_vertex_buffers,
+                            vertex_buffers, count);
    failover->dirty |= FO_NEW_VERTEX_BUFFER;
-   failover->num_vertex_buffers = count;
    failover->sw->set_vertex_buffers( failover->sw, count, vertex_buffers );
    failover->hw->set_vertex_buffers( failover->hw, count, vertex_buffers );
 }
diff --git a/src/gallium/drivers/galahad/glhd_context.c b/src/gallium/drivers/galahad/glhd_context.c
index a572ad22bd0..8cbf0b1de4a 100644
--- a/src/gallium/drivers/galahad/glhd_context.c
+++ b/src/gallium/drivers/galahad/glhd_context.c
@@ -381,6 +381,8 @@ galahad_create_vertex_elements_state(struct pipe_context *_pipe,
    struct galahad_context *glhd_pipe = galahad_context(_pipe);
    struct pipe_context *pipe = glhd_pipe->pipe;
 
+   /* XXX check if stride lines up with element size, at least for floats */
+
    return pipe->create_vertex_elements_state(pipe,
                                              num_elements,
                                              vertex_elements);
diff --git a/src/gallium/drivers/i915/i915_context.c b/src/gallium/drivers/i915/i915_context.c
index 847dd6dd47e..9be31619254 100644
--- a/src/gallium/drivers/i915/i915_context.c
+++ b/src/gallium/drivers/i915/i915_context.c
@@ -107,6 +107,10 @@ static void i915_destroy(struct pipe_context *pipe)
    if(i915->batch)
       i915->iws->batchbuffer_destroy(i915->batch);
 
+   for (i = 0; i < i915->num_vertex_buffers; i++) {
+      pipe_resource_reference(&i915->vertex_buffer[i].buffer, NULL);
+   }
+
    /* unbind framebuffer */
    for (i = 0; i < PIPE_MAX_COLOR_BUFS; i++) {
       pipe_surface_reference(&i915->framebuffer.cbufs[i], NULL);
diff --git a/src/gallium/drivers/i915/i915_fpc_translate.c b/src/gallium/drivers/i915/i915_fpc_translate.c
index 25c53210be8..9e20010c4a1 100644
--- a/src/gallium/drivers/i915/i915_fpc_translate.c
+++ b/src/gallium/drivers/i915/i915_fpc_translate.c
@@ -924,6 +924,14 @@ i915_translate_instructions(struct i915_fp_compile *p,
       tgsi_parse_token( &parse );
 
       switch( parse.FullToken.Token.Type ) {
+      case TGSI_TOKEN_TYPE_PROPERTY:
+         /*
+          * We only support one cbuf, but we still need to ignore the property
+          * correctly so we don't hit the assert at the end of the switch case.
+          */
+         assert(parse.FullToken.FullProperty.Property.PropertyName ==
+                TGSI_PROPERTY_FS_COLOR0_WRITES_ALL_CBUFS);
+         break;
       case TGSI_TOKEN_TYPE_DECLARATION:
          if (parse.FullToken.FullDeclaration.Declaration.File
                   == TGSI_FILE_CONSTANT) {
diff --git a/src/gallium/drivers/i915/i915_state.c b/src/gallium/drivers/i915/i915_state.c
index bbfcff6bc4d..f5b60ed5942 100644
--- a/src/gallium/drivers/i915/i915_state.c
+++ b/src/gallium/drivers/i915/i915_state.c
@@ -760,8 +760,9 @@ static void i915_set_vertex_buffers(struct pipe_context *pipe,
     */
    draw_flush(i915->draw);
 
-   memcpy(i915->vertex_buffer, buffers, count * sizeof(buffers[0]));
-   i915->num_vertex_buffers = count;
+   util_copy_vertex_buffers(i915->vertex_buffer,
+                            &i915->num_vertex_buffers,
+                            buffers, count);
 
    /* pass-through to draw module */
    draw_set_vertex_buffers(i915->draw, count, buffers);
diff --git a/src/gallium/drivers/i965/brw_draw_upload.c b/src/gallium/drivers/i965/brw_draw_upload.c
index ebeb1e146aa..cf9405470c8 100644
--- a/src/gallium/drivers/i965/brw_draw_upload.c
+++ b/src/gallium/drivers/i965/brw_draw_upload.c
@@ -89,13 +89,16 @@ static int brw_prepare_vertices(struct brw_context *brw)
 			  vb->buffer->width0 - vb->buffer_offset :
 			  MAX2(vb->buffer->width0 - vb->buffer_offset,
 			       vb->stride * (max_index + 1 - min_index)));
+	 boolean flushed;
 
-	 ret = u_upload_buffer( brw->vb.upload_vertex, 
+	 ret = u_upload_buffer( brw->vb.upload_vertex,
+				0,
 				vb->buffer_offset + min_index * vb->stride,
 				size,
 				vb->buffer,
 				&offset,
-				&upload_buf );
+				&upload_buf,
+				&flushed );
 	 if (ret)
 	    return ret;
 
@@ -251,13 +254,16 @@ static int brw_prepare_indices(struct brw_context *brw)
    /* Turn userbuffer into a proper hardware buffer?
     */
    if (brw_buffer_is_user_buffer(index_buffer)) {
+      boolean flushed;
 
       ret = u_upload_buffer( brw->vb.upload_index,
+                             0,
 			     index_offset,
 			     ib_size,
 			     index_buffer,
 			     &offset,
-			     &upload_buf );
+			     &upload_buf,
+			     &flushed );
       if (ret)
 	 return ret;
 
diff --git a/src/gallium/drivers/i965/brw_pipe_vertex.c b/src/gallium/drivers/i965/brw_pipe_vertex.c
index 007239efc40..e1697687ccc 100644
--- a/src/gallium/drivers/i965/brw_pipe_vertex.c
+++ b/src/gallium/drivers/i965/brw_pipe_vertex.c
@@ -248,7 +248,6 @@ static void brw_set_vertex_buffers(struct pipe_context *pipe,
                                    const struct pipe_vertex_buffer *buffers)
 {
    struct brw_context *brw = brw_context(pipe);
-   unsigned i;
 
    /* Check for no change */
    if (count == brw->curr.num_vertex_buffers &&
@@ -257,18 +256,9 @@ static void brw_set_vertex_buffers(struct pipe_context *pipe,
               count * sizeof buffers[0]) == 0)
       return;
 
-   /* Adjust refcounts */
-   for (i = 0; i < count; i++) 
-      pipe_resource_reference(&brw->curr.vertex_buffer[i].buffer, 
-                            buffers[i].buffer);
-
-   for ( ; i < brw->curr.num_vertex_buffers; i++)
-      pipe_resource_reference(&brw->curr.vertex_buffer[i].buffer,
-                            NULL);
-
-   /* Copy remaining data */
-   memcpy(brw->curr.vertex_buffer, buffers, count * sizeof buffers[0]);
-   brw->curr.num_vertex_buffers = count;
+   util_copy_vertex_buffers(brw->curr.vertex_buffer,
+                            &brw->curr.num_vertex_buffers,
+                            buffers, count);
 
    brw->state.dirty.mesa |= PIPE_NEW_VERTEX_BUFFER;
 }
@@ -318,9 +308,13 @@ brw_pipe_vertex_init( struct brw_context *brw )
 void 
 brw_pipe_vertex_cleanup( struct brw_context *brw )
 {
+   unsigned i;
 
    /* Release bound pipe vertex_buffers
     */
+   for (i = 0; i < brw->curr.num_vertex_buffers; i++) {
+      pipe_resource_reference(&brw->curr.vertex_buffer[i].buffer, NULL);
+   }
 
    /* Release some other stuff
     */
diff --git a/src/gallium/drivers/i965/intel_decode.h b/src/gallium/drivers/i965/intel_decode.h
index 7683097b869..6201a23d6a4 100644
--- a/src/gallium/drivers/i965/intel_decode.h
+++ b/src/gallium/drivers/i965/intel_decode.h
@@ -25,5 +25,7 @@
  *
  */
 
+#include "pipe/p_compiler.h"
+
 int intel_decode(const uint32_t *data, int count, uint32_t hw_offset, uint32_t devid);
 void intel_decode_context_reset(void);
diff --git a/src/gallium/drivers/i965/intel_structs.h b/src/gallium/drivers/i965/intel_structs.h
index 522e3bd92c2..ec6eec8910f 100644
--- a/src/gallium/drivers/i965/intel_structs.h
+++ b/src/gallium/drivers/i965/intel_structs.h
@@ -1,6 +1,8 @@
 #ifndef INTEL_STRUCTS_H
 #define INTEL_STRUCTS_H
 
+#include "brw_types.h"
+
 struct br0 {
    GLuint length:8;
    GLuint pad0:3;
diff --git a/src/gallium/drivers/llvmpipe/lp_bld_alpha.h b/src/gallium/drivers/llvmpipe/lp_bld_alpha.h
index 5c9392504f1..06206a24d83 100644
--- a/src/gallium/drivers/llvmpipe/lp_bld_alpha.h
+++ b/src/gallium/drivers/llvmpipe/lp_bld_alpha.h
@@ -34,10 +34,12 @@
 #ifndef LP_BLD_ALPHA_H
 #define LP_BLD_ALPHA_H
 
+#include "pipe/p_compiler.h"
 
 #include "gallivm/lp_bld.h"
 
 struct pipe_alpha_state;
+struct gallivm_state;
 struct lp_type;
 struct lp_build_mask_context;
 
diff --git a/src/gallium/drivers/llvmpipe/lp_bld_depth.h b/src/gallium/drivers/llvmpipe/lp_bld_depth.h
index 038b136a281..e01fc46ec16 100644
--- a/src/gallium/drivers/llvmpipe/lp_bld_depth.h
+++ b/src/gallium/drivers/llvmpipe/lp_bld_depth.h
@@ -36,10 +36,14 @@
 #define LP_BLD_DEPTH_H
 
 
+#include "pipe/p_compiler.h"
+#include "pipe/p_state.h"
+
 #include "gallivm/lp_bld.h"
 
  
 struct pipe_depth_state;
+struct gallivm_state;
 struct util_format_description;
 struct lp_type;
 struct lp_build_mask_context;
diff --git a/src/gallium/drivers/llvmpipe/lp_context.c b/src/gallium/drivers/llvmpipe/lp_context.c
index 2de20d6e9a3..644201ddf7c 100644
--- a/src/gallium/drivers/llvmpipe/lp_context.c
+++ b/src/gallium/drivers/llvmpipe/lp_context.c
@@ -125,6 +125,10 @@ static void llvmpipe_destroy( struct pipe_context *pipe )
       }
    }
 
+   for (i = 0; i < llvmpipe->num_vertex_buffers; i++) {
+      pipe_resource_reference(&llvmpipe->vertex_buffer[i].buffer, NULL);
+   }
+
    gallivm_destroy(llvmpipe->gallivm);
 
    align_free( llvmpipe );
diff --git a/src/gallium/drivers/llvmpipe/lp_perf.h b/src/gallium/drivers/llvmpipe/lp_perf.h
index b23a100b873..455adf7d6f8 100644
--- a/src/gallium/drivers/llvmpipe/lp_perf.h
+++ b/src/gallium/drivers/llvmpipe/lp_perf.h
@@ -33,6 +33,7 @@
 #ifndef LP_PERF_H
 #define LP_PERF_H
 
+#include "pipe/p_compiler.h"
 
 /**
  * Various counters
diff --git a/src/gallium/drivers/llvmpipe/lp_scene_queue.h b/src/gallium/drivers/llvmpipe/lp_scene_queue.h
index fd7c65a2c8b..dd9ab593b4a 100644
--- a/src/gallium/drivers/llvmpipe/lp_scene_queue.h
+++ b/src/gallium/drivers/llvmpipe/lp_scene_queue.h
@@ -29,6 +29,8 @@
 #ifndef LP_SCENE_QUEUE
 #define LP_SCENE_QUEUE
 
+#include "pipe/p_compiler.h"
+
 struct lp_scene_queue;
 struct lp_scene;
 
diff --git a/src/gallium/drivers/llvmpipe/lp_state_vertex.c b/src/gallium/drivers/llvmpipe/lp_state_vertex.c
index fb29423dd35..fffdeb6ccde 100644
--- a/src/gallium/drivers/llvmpipe/lp_state_vertex.c
+++ b/src/gallium/drivers/llvmpipe/lp_state_vertex.c
@@ -33,6 +33,7 @@
 #include "lp_state.h"
 
 #include "draw/draw_context.h"
+#include "util/u_inlines.h"
 
 
 static void *
@@ -80,8 +81,9 @@ llvmpipe_set_vertex_buffers(struct pipe_context *pipe,
 
    assert(count <= PIPE_MAX_ATTRIBS);
 
-   memcpy(llvmpipe->vertex_buffer, buffers, count * sizeof(buffers[0]));
-   llvmpipe->num_vertex_buffers = count;
+   util_copy_vertex_buffers(llvmpipe->vertex_buffer,
+                            &llvmpipe->num_vertex_buffers,
+                            buffers, count);
 
    llvmpipe->dirty |= LP_NEW_VERTEX;
 
diff --git a/src/gallium/drivers/nouveau/nouveau_screen.h b/src/gallium/drivers/nouveau/nouveau_screen.h
index 8c290273fb4..1f4e5171c01 100644
--- a/src/gallium/drivers/nouveau/nouveau_screen.h
+++ b/src/gallium/drivers/nouveau/nouveau_screen.h
@@ -66,7 +66,7 @@ void nouveau_screen_fini(struct nouveau_screen *);
 
 
 
-
+#ifndef NOUVEAU_NVC0
 static INLINE unsigned
 RING_3D(unsigned mthd, unsigned size)
 {
@@ -78,5 +78,6 @@ RING_3D_NI(unsigned mthd, unsigned size)
 {
 	return 0x40000000 | (7 << 13) | (size << 18) | mthd;
 }
+#endif
 
 #endif
diff --git a/src/gallium/drivers/nouveau/nouveau_winsys.h b/src/gallium/drivers/nouveau/nouveau_winsys.h
index 747b084bc45..8dfb84a596f 100644
--- a/src/gallium/drivers/nouveau/nouveau_winsys.h
+++ b/src/gallium/drivers/nouveau/nouveau_winsys.h
@@ -10,7 +10,9 @@
 #include "nouveau/nouveau_grobj.h"
 #include "nouveau/nouveau_notifier.h"
 #include "nouveau/nouveau_resource.h"
+#ifndef NOUVEAU_NVC0
 #include "nouveau/nv04_pushbuf.h"
+#endif
 
 #ifndef NV04_PFIFO_MAX_PACKET_LEN
 #define NV04_PFIFO_MAX_PACKET_LEN 2047
@@ -41,4 +43,7 @@ nvfx_screen_create(struct pipe_winsys *ws, struct nouveau_device *);
 extern struct pipe_screen *
 nv50_screen_create(struct pipe_winsys *ws, struct nouveau_device *);
 
+extern struct pipe_screen *
+nvc0_screen_create(struct pipe_winsys *ws, struct nouveau_device *);
+
 #endif
diff --git a/src/gallium/drivers/nouveau/nv_object.xml.h b/src/gallium/drivers/nouveau/nv_object.xml.h
index cb7653c3fe2..a5b0d0478c8 100644
--- a/src/gallium/drivers/nouveau/nv_object.xml.h
+++ b/src/gallium/drivers/nouveau/nv_object.xml.h
@@ -8,12 +8,10 @@ http://0x04.net/cgit/index.cgi/rules-ng-ng
 git clone git://0x04.net/rules-ng-ng
 
 The rules-ng-ng source files this header was generated from are:
-- nv30-40_3d.xml (  31709 bytes, from 2010-09-05 07:53:14)
-- copyright.xml  (   6503 bytes, from 2010-04-10 23:15:50)
-- nv_3ddefs.xml  (  15193 bytes, from 2010-09-05 07:50:15)
-- nv_defs.xml    (   4437 bytes, from 2010-08-05 19:38:53)
-- nv_object.xml  (  10424 bytes, from 2010-08-05 19:38:53)
-- nvchipsets.xml (   2824 bytes, from 2010-08-05 19:38:53)
+- nv_object.xml  (  11547 bytes, from 2010-10-24 15:29:34)
+- copyright.xml  (   6498 bytes, from 2010-10-03 13:18:37)
+- nvchipsets.xml (   2907 bytes, from 2010-10-15 16:28:21)
+- nv_defs.xml    (   4437 bytes, from 2010-07-06 07:43:58)
 
 Copyright (C) 2006-2010 by the following authors:
 - Artur Huillet <[email protected]> (ahuillet)
@@ -37,7 +35,7 @@ Copyright (C) 2006-2010 by the following authors:
 - Mark Carey <[email protected]> (careym)
 - Matthieu Castet <[email protected]> (mat-c)
 - nvidiaman <[email protected]> (nvidiaman)
-- Patrice Mandin <[email protected]> (pmandin, pmdata)
+- Patrice Mandin <[email protected]> (pmandin, pmdata)
 - Pekka Paalanen <[email protected]> (pq, ppaalanen)
 - Peter Popov <[email protected]> (ironpeter)
 - Richard Hughes <[email protected]> (hughsient)
@@ -180,6 +178,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 #define NV50_COMPUTE						0x000050c0
 #define NVA3_COMPUTE						0x000085c0
 #define NVC0_COMPUTE						0x000090c0
+#define NV84_CRYPT						0x000074c1
 #define NV01_SUBCHAN__SIZE					0x00002000
 #define NV01_SUBCHAN						0x00000000
 
@@ -194,9 +193,9 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 #define NV84_SUBCHAN_QUERY_GET					0x0000001c
 
-#define NV84_SUBCHAN_UNK20					0x00000020
+#define NV84_SUBCHAN_QUERY_INTR					0x00000020
 
-#define NV84_SUBCHAN_UNK24					0x00000024
+#define NV84_SUBCHAN_WRCACHE_FLUSH				0x00000024
 
 #define NV10_SUBCHAN_REF_CNT					0x00000050
 
@@ -209,7 +208,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 #define NV11_SUBCHAN_SEMAPHORE_RELEASE				0x0000006c
 
-#define NV50_SUBCHAN_UNK80					0x00000080
+#define NV40_SUBCHAN_YIELD					0x00000080
 
 #define NV01_GRAPH						0x00000000
 
@@ -227,5 +226,43 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 #define NV40_GRAPH_PM_TRIGGER					0x00000140
 
+#define NVC0_SUBCHAN__SIZE					0x00008000
+#define NVC0_SUBCHAN						0x00000000
+
+#define NVC0_SUBCHAN_OBJECT					0x00000000
+
+
+#define NVC0_SUBCHAN_QUERY_ADDRESS_HIGH				0x00000010
+
+#define NVC0_SUBCHAN_QUERY_ADDRESS_LOW				0x00000014
+
+#define NVC0_SUBCHAN_QUERY_SEQUENCE				0x00000018
+
+#define NVC0_SUBCHAN_QUERY_GET					0x0000001c
+
+#define NVC0_SUBCHAN_REF_CNT					0x00000050
+
+#define NVC0_GRAPH						0x00000000
+
+#define NVC0_GRAPH_NOP						0x00000100
+
+#define NVC0_GRAPH_NOTIFY_ADDRESS_HIGH				0x00000104
+
+#define NVC0_GRAPH_NOTIFY_ADDRESS_LOW				0x00000108
+
+#define NVC0_GRAPH_NOTIFY					0x0000010c
+#define NVC0_GRAPH_NOTIFY_WRITE					0x00000000
+#define NVC0_GRAPH_NOTIFY_WRITE_AND_AWAKEN			0x00000001
+
+#define NVC0_GRAPH_SERIALIZE					0x00000110
+
+#define NVC0_GRAPH_MACRO_UPLOAD_POS				0x00000114
+
+#define NVC0_GRAPH_MACRO_UPLOAD_DATA				0x00000118
+
+#define NVC0_GRAPH_MACRO_ID					0x0000011c
+
+#define NVC0_GRAPH_MACRO_POS					0x00000120
+
 
 #endif /* NV_OBJECT_XML */
diff --git a/src/gallium/drivers/nv50/nv50_context.c b/src/gallium/drivers/nv50/nv50_context.c
index 0874cb5e4ea..4f976161760 100644
--- a/src/gallium/drivers/nv50/nv50_context.c
+++ b/src/gallium/drivers/nv50/nv50_context.c
@@ -49,6 +49,10 @@ nv50_destroy(struct pipe_context *pipe)
 	struct nv50_context *nv50 = nv50_context(pipe);
 	int i;
 
+        for (i = 0; i < nv50->vtxbuf_nr; i++) {
+           pipe_resource_reference(&nv50->vtxbuf[i].buffer, NULL);
+        }
+
 	for (i = 0; i < 64; i++) {
 		if (!nv50->state.hw[i])
 			continue;
diff --git a/src/gallium/drivers/nv50/nv50_state.c b/src/gallium/drivers/nv50/nv50_state.c
index f42fa2d4d2b..d97566ed7cf 100644
--- a/src/gallium/drivers/nv50/nv50_state.c
+++ b/src/gallium/drivers/nv50/nv50_state.c
@@ -780,8 +780,9 @@ nv50_set_vertex_buffers(struct pipe_context *pipe, unsigned count,
 {
 	struct nv50_context *nv50 = nv50_context(pipe);
 
-	memcpy(nv50->vtxbuf, vb, sizeof(*vb) * count);
-	nv50->vtxbuf_nr = count;
+        util_copy_vertex_buffers(nv50->vtxbuf,
+                                 &nv50->vtxbuf_nr,
+                                 vb, count);
 
 	nv50->dirty |= NV50_NEW_ARRAYS;
 }
diff --git a/src/gallium/drivers/nvc0/Makefile b/src/gallium/drivers/nvc0/Makefile
new file mode 100644
index 00000000000..5c3d46d9eab
--- /dev/null
+++ b/src/gallium/drivers/nvc0/Makefile
@@ -0,0 +1,33 @@
+TOP = ../../../..
+include $(TOP)/configs/current
+
+LIBNAME = nvc0
+
+C_SOURCES = \
+	nvc0_buffer.c \
+	nvc0_context.c \
+	nvc0_draw.c \
+	nvc0_formats.c \
+	nvc0_miptree.c \
+	nvc0_resource.c \
+	nvc0_screen.c \
+	nvc0_state.c \
+	nvc0_state_validate.c \
+	nvc0_surface.c \
+	nvc0_tex.c \
+	nvc0_transfer.c \
+	nvc0_vbo.c \
+	nvc0_program.c \
+	nvc0_shader_state.c \
+	nvc0_pc.c \
+	nvc0_pc_print.c \
+	nvc0_pc_emit.c \
+	nvc0_tgsi_to_nc.c \
+	nvc0_pc_optimize.c \
+	nvc0_pc_regalloc.c \
+	nvc0_push.c \
+	nvc0_push2.c \
+	nvc0_fence.c \
+	nvc0_mm.c
+
+include ../../Makefile.template
diff --git a/src/gallium/drivers/nvc0/SConscript b/src/gallium/drivers/nvc0/SConscript
new file mode 100644
index 00000000000..46c0b5889a0
--- /dev/null
+++ b/src/gallium/drivers/nvc0/SConscript
@@ -0,0 +1,35 @@
+Import('*')
+
+env = env.Clone()
+
+nvc0 = env.ConvenienceLibrary(
+    target = 'nvc0',
+    source = [
+        'nvc0_buffer.c',
+        'nvc0_context.c',
+        'nvc0_draw.c',
+        'nvc0_formats.c',
+        'nvc0_miptree.c',
+        'nvc0_resource.c',
+        'nvc0_screen.c',
+        'nvc0_state.c',
+        'nvc0_state_validate.c',
+        'nvc0_surface.c',
+        'nvc0_tex.c',
+        'nvc0_transfer.c',
+        'nvc0_vbo.c',
+        'nvc0_program.c',
+        'nvc0_shader_state.c',
+        'nvc0_pc.c',
+        'nvc0_pc_print.c',
+        'nvc0_pc_emit.c',
+        'nvc0_tgsi_to_nc.c',
+        'nvc0_pc_optimize.c',
+        'nvc0_pc_regalloc.c',
+        'nvc0_push.c',
+        'nvc0_push2.c',
+        'nvc0_fence.c',
+        'nvc0_mm.c'
+    ])
+
+Export('nvc0')
diff --git a/src/gallium/drivers/nvc0/nv50_defs.xml.h b/src/gallium/drivers/nvc0/nv50_defs.xml.h
new file mode 100644
index 00000000000..1bf2f802b56
--- /dev/null
+++ b/src/gallium/drivers/nvc0/nv50_defs.xml.h
@@ -0,0 +1,142 @@
+#ifndef NV50_DEFS_XML
+#define NV50_DEFS_XML
+
+/* Autogenerated file, DO NOT EDIT manually!
+
+This file was generated by the rules-ng-ng headergen tool in this git repository:
+http://0x04.net/cgit/index.cgi/rules-ng-ng
+git clone git://0x04.net/rules-ng-ng
+
+The rules-ng-ng source files this header was generated from are:
+- nv50_defs.xml (   4482 bytes, from 2010-10-03 13:18:37)
+- copyright.xml (   6498 bytes, from 2010-10-03 13:18:37)
+
+Copyright (C) 2006-2010 by the following authors:
+- Artur Huillet <[email protected]> (ahuillet)
+- Ben Skeggs (darktama, darktama_)
+- B. R. <[email protected]> (koala_br)
+- Carlos Martin <[email protected]> (carlosmn)
+- Christoph Bumiller <[email protected]> (calim, chrisbmr)
+- Dawid Gajownik <[email protected]> (gajownik)
+- Dmitry Baryshkov
+- Dmitry Eremin-Solenikov <[email protected]> (lumag)
+- EdB <[email protected]> (edb_)
+- Erik Waling <[email protected]> (erikwaling)
+- Francisco Jerez <[email protected]> (curro, curro_, currojerez)
+- imirkin <[email protected]> (imirkin)
+- jb17bsome <[email protected]> (jb17bsome)
+- Jeremy Kolb <[email protected]> (kjeremy)
+- Laurent Carlier <[email protected]> (lordheavy)
+- Luca Barbieri <[email protected]> (lb, lb1)
+- Maarten Maathuis <[email protected]> (stillunknown)
+- Marcin Kościelnicki <[email protected]> (mwk, koriakin)
+- Mark Carey <[email protected]> (careym)
+- Matthieu Castet <[email protected]> (mat-c)
+- nvidiaman <[email protected]> (nvidiaman)
+- Patrice Mandin <[email protected]> (pmandin, pmdata)
+- Pekka Paalanen <[email protected]> (pq, ppaalanen)
+- Peter Popov <[email protected]> (ironpeter)
+- Richard Hughes <[email protected]> (hughsient)
+- Rudi Cilibrasi <[email protected]> (cilibrar)
+- Serge Martin
+- Simon Raffeiner
+- Stephane Loeuillet <[email protected]> (leroutier)
+- Stephane Marchesin <[email protected]> (marcheu)
+- sturmflut <[email protected]> (sturmflut)
+- Sylvain Munaut <[email protected]>
+- Victor Stinner <[email protected]> (haypo)
+- Wladmir van der Laan <[email protected]> (miathan6)
+- Younes Manton <[email protected]> (ymanton)
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice (including the
+next paragraph) shall be included in all copies or substantial
+portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/
+
+
+#define NV50_SURFACE_FORMAT_R32G32B32A32_FLOAT			0x000000c0
+#define NV50_SURFACE_FORMAT_R32G32B32A32_SINT			0x000000c1
+#define NV50_SURFACE_FORMAT_R32G32B32A32_UINT			0x000000c2
+#define NV50_SURFACE_FORMAT_R32G32B32X32_FLOAT			0x000000c3
+#define NV50_SURFACE_FORMAT_R16G16B16A16_UNORM			0x000000c6
+#define NV50_SURFACE_FORMAT_R16G16B16A16_SNORM			0x000000c7
+#define NV50_SURFACE_FORMAT_R16G16B16A16_SINT			0x000000c8
+#define NV50_SURFACE_FORMAT_R16G16B16A16_UINT			0x000000c9
+#define NV50_SURFACE_FORMAT_R16G16B16A16_FLOAT			0x000000ca
+#define NV50_SURFACE_FORMAT_R32G32_FLOAT			0x000000cb
+#define NV50_SURFACE_FORMAT_R32G32_SINT				0x000000cc
+#define NV50_SURFACE_FORMAT_R32G32_UINT				0x000000cd
+#define NV50_SURFACE_FORMAT_R16G16B16X16_FLOAT			0x000000ce
+#define NV50_SURFACE_FORMAT_A8R8G8B8_UNORM			0x000000cf
+#define NV50_SURFACE_FORMAT_A8R8G8B8_SRGB			0x000000d0
+#define NV50_SURFACE_FORMAT_A2B10G10R10_UNORM			0x000000d1
+#define NV50_SURFACE_FORMAT_A2B10G10R10_UINT			0x000000d2
+#define NV50_SURFACE_FORMAT_A8B8G8R8_UNORM			0x000000d5
+#define NV50_SURFACE_FORMAT_A8B8G8R8_SRGB			0x000000d6
+#define NV50_SURFACE_FORMAT_A8B8G8R8_SNORM			0x000000d7
+#define NV50_SURFACE_FORMAT_A8B8G8R8_SINT			0x000000d8
+#define NV50_SURFACE_FORMAT_A8B8G8R8_UINT			0x000000d9
+#define NV50_SURFACE_FORMAT_R16G16_UNORM			0x000000da
+#define NV50_SURFACE_FORMAT_R16G16_SNORM			0x000000db
+#define NV50_SURFACE_FORMAT_R16G16_SINT				0x000000dc
+#define NV50_SURFACE_FORMAT_R16G16_UINT				0x000000dd
+#define NV50_SURFACE_FORMAT_R16G16_FLOAT			0x000000de
+#define NV50_SURFACE_FORMAT_A2R10G10B10_UNORM			0x000000df
+#define NV50_SURFACE_FORMAT_B10G11R11_FLOAT			0x000000e0
+#define NV50_SURFACE_FORMAT_R32_FLOAT				0x000000e5
+#define NV50_SURFACE_FORMAT_X8R8G8B8_UNORM			0x000000e6
+#define NV50_SURFACE_FORMAT_X8R8G8B8_SRGB			0x000000e7
+#define NV50_SURFACE_FORMAT_R5G6B5_UNORM			0x000000e8
+#define NV50_SURFACE_FORMAT_A1R5G5B5_UNORM			0x000000e9
+#define NV50_SURFACE_FORMAT_R8G8_UNORM				0x000000ea
+#define NV50_SURFACE_FORMAT_R8G8_SNORM				0x000000eb
+#define NV50_SURFACE_FORMAT_R8G8_SINT				0x000000ec
+#define NV50_SURFACE_FORMAT_R8G8_UINT				0x000000ed
+#define NV50_SURFACE_FORMAT_R16_UNORM				0x000000ee
+#define NV50_SURFACE_FORMAT_R16_SNORM				0x000000ef
+#define NV50_SURFACE_FORMAT_R16_SINT				0x000000f0
+#define NV50_SURFACE_FORMAT_R16_UINT				0x000000f1
+#define NV50_SURFACE_FORMAT_R16_FLOAT				0x000000f2
+#define NV50_SURFACE_FORMAT_R8_UNORM				0x000000f3
+#define NV50_SURFACE_FORMAT_R8_SNORM				0x000000f4
+#define NV50_SURFACE_FORMAT_R8_SINT				0x000000f5
+#define NV50_SURFACE_FORMAT_R8_UINT				0x000000f6
+#define NV50_SURFACE_FORMAT_A8_UNORM				0x000000f7
+#define NV50_SURFACE_FORMAT_X1R5G5B5_UNORM			0x000000f8
+#define NV50_SURFACE_FORMAT_X8B8G8R8_UNORM			0x000000f9
+#define NV50_SURFACE_FORMAT_X8B8G8R8_SRGB			0x000000fa
+#define NV50_ZETA_FORMAT_Z32_FLOAT				0x0000000a
+#define NV50_ZETA_FORMAT_Z16_UNORM				0x00000013
+#define NV50_ZETA_FORMAT_Z24S8_UNORM				0x00000014
+#define NV50_ZETA_FORMAT_X8Z24_UNORM				0x00000015
+#define NV50_ZETA_FORMAT_S8Z24_UNORM				0x00000016
+#define NV50_ZETA_FORMAT_UNK18					0x00000018
+#define NV50_ZETA_FORMAT_Z32_FLOAT_X24S8_UNORM			0x00000019
+#define NV50_ZETA_FORMAT_UNK1D					0x0000001d
+#define NV50_ZETA_FORMAT_UNK1E					0x0000001e
+#define NV50_ZETA_FORMAT_UNK1F					0x0000001f
+#define NV50_QUERY__SIZE					0x00000010
+#define NV50_QUERY_COUNTER					0x00000000
+
+#define NV50_QUERY_RES						0x00000004
+
+#define NV50_QUERY_TIME						0x00000008
+
+
+#endif /* NV50_DEFS_XML */
diff --git a/src/gallium/drivers/nvc0/nv50_texture.xml.h b/src/gallium/drivers/nvc0/nv50_texture.xml.h
new file mode 100644
index 00000000000..9f83206516f
--- /dev/null
+++ b/src/gallium/drivers/nvc0/nv50_texture.xml.h
@@ -0,0 +1,259 @@
+#ifndef NV50_TEXTURE_XML
+#define NV50_TEXTURE_XML
+
+/* Autogenerated file, DO NOT EDIT manually!
+
+This file was generated by the rules-ng-ng headergen tool in this git repository:
+http://0x04.net/cgit/index.cgi/rules-ng-ng
+git clone git://0x04.net/rules-ng-ng
+
+The rules-ng-ng source files this header was generated from are:
+- nv50_texture.xml (   6871 bytes, from 2010-10-03 13:18:37)
+- copyright.xml    (   6498 bytes, from 2010-10-03 13:18:37)
+
+Copyright (C) 2006-2010 by the following authors:
+- Artur Huillet <[email protected]> (ahuillet)
+- Ben Skeggs (darktama, darktama_)
+- B. R. <[email protected]> (koala_br)
+- Carlos Martin <[email protected]> (carlosmn)
+- Christoph Bumiller <[email protected]> (calim, chrisbmr)
+- Dawid Gajownik <[email protected]> (gajownik)
+- Dmitry Baryshkov
+- Dmitry Eremin-Solenikov <[email protected]> (lumag)
+- EdB <[email protected]> (edb_)
+- Erik Waling <[email protected]> (erikwaling)
+- Francisco Jerez <[email protected]> (curro, curro_, currojerez)
+- imirkin <[email protected]> (imirkin)
+- jb17bsome <[email protected]> (jb17bsome)
+- Jeremy Kolb <[email protected]> (kjeremy)
+- Laurent Carlier <[email protected]> (lordheavy)
+- Luca Barbieri <[email protected]> (lb, lb1)
+- Maarten Maathuis <[email protected]> (stillunknown)
+- Marcin Kościelnicki <[email protected]> (mwk, koriakin)
+- Mark Carey <[email protected]> (careym)
+- Matthieu Castet <[email protected]> (mat-c)
+- nvidiaman <[email protected]> (nvidiaman)
+- Patrice Mandin <[email protected]> (pmandin, pmdata)
+- Pekka Paalanen <[email protected]> (pq, ppaalanen)
+- Peter Popov <[email protected]> (ironpeter)
+- Richard Hughes <[email protected]> (hughsient)
+- Rudi Cilibrasi <[email protected]> (cilibrar)
+- Serge Martin
+- Simon Raffeiner
+- Stephane Loeuillet <[email protected]> (leroutier)
+- Stephane Marchesin <[email protected]> (marcheu)
+- sturmflut <[email protected]> (sturmflut)
+- Sylvain Munaut <[email protected]>
+- Victor Stinner <[email protected]> (haypo)
+- Wladmir van der Laan <[email protected]> (miathan6)
+- Younes Manton <[email protected]> (ymanton)
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice (including the
+next paragraph) shall be included in all copies or substantial
+portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/
+
+
+#define NV50_TIC_MAP_ZERO					0x00000000
+#define NV50_TIC_MAP_C0						0x00000002
+#define NV50_TIC_MAP_C1						0x00000003
+#define NV50_TIC_MAP_C2						0x00000004
+#define NV50_TIC_MAP_C3						0x00000005
+#define NV50_TIC_MAP_ONE					0x00000007
+#define NV50_TIC_TYPE_SNORM					0x00000001
+#define NV50_TIC_TYPE_UNORM					0x00000002
+#define NV50_TIC_TYPE_SINT					0x00000003
+#define NV50_TIC_TYPE_UINT					0x00000004
+#define NV50_TIC_TYPE_SSCALED					0x00000005
+#define NV50_TIC_TYPE_USCALED					0x00000006
+#define NV50_TIC_TYPE_FLOAT					0x00000007
+#define NV50_TSC_WRAP_REPEAT					0x00000000
+#define NV50_TSC_WRAP_MIRROR_REPEAT				0x00000001
+#define NV50_TSC_WRAP_CLAMP_TO_EDGE				0x00000002
+#define NV50_TSC_WRAP_CLAMP_TO_BORDER				0x00000003
+#define NV50_TSC_WRAP_CLAMP					0x00000004
+#define NV50_TSC_WRAP_MIRROR_CLAMP_TO_EDGE			0x00000005
+#define NV50_TSC_WRAP_MIRROR_CLAMP_TO_BORDER			0x00000006
+#define NV50_TSC_WRAP_MIRROR_CLAMP				0x00000007
+#define NV50_TIC__SIZE						0x00000020
+#define NV50_TIC_0						0x00000000
+#define NV50_TIC_0_MAPA__MASK					0x38000000
+#define NV50_TIC_0_MAPA__SHIFT					27
+#define NV50_TIC_0_MAPB__MASK					0x07000000
+#define NV50_TIC_0_MAPB__SHIFT					24
+#define NV50_TIC_0_MAPG__MASK					0x00e00000
+#define NV50_TIC_0_MAPG__SHIFT					21
+#define NV50_TIC_0_MAPR__MASK					0x001c0000
+#define NV50_TIC_0_MAPR__SHIFT					18
+#define NV50_TIC_0_TYPE3__MASK					0x00038000
+#define NV50_TIC_0_TYPE3__SHIFT					15
+#define NV50_TIC_0_TYPE2__MASK					0x00007000
+#define NV50_TIC_0_TYPE2__SHIFT					12
+#define NV50_TIC_0_TYPE1__MASK					0x00000e00
+#define NV50_TIC_0_TYPE1__SHIFT					9
+#define NV50_TIC_0_TYPE0__MASK					0x000001c0
+#define NV50_TIC_0_TYPE0__SHIFT					6
+#define NV50_TIC_0_SWIZZLE__MASK				0x3ffc0000
+#define NV50_TIC_0_FMT__MASK					0x0000003f
+#define NV50_TIC_0_FMT__SHIFT					0
+#define NV50_TIC_0_FMT_32_32_32_32				0x00000001
+#define NV50_TIC_0_FMT_16_16_16_16				0x00000003
+#define NV50_TIC_0_FMT_32_32					0x00000004
+#define NV50_TIC_0_FMT_32_8					0x00000005
+#define NV50_TIC_0_FMT_8_8_8_8					0x00000008
+#define NV50_TIC_0_FMT_2_10_10_10				0x00000009
+#define NV50_TIC_0_FMT_16_16					0x0000000c
+#define NV50_TIC_0_FMT_8_24					0x0000000d
+#define NV50_TIC_0_FMT_24_8					0x0000000e
+#define NV50_TIC_0_FMT_32					0x0000000f
+#define NV50_TIC_0_FMT_4_4_4_4					0x00000012
+#define NV50_TIC_0_FMT_5_5_5_1					0x00000013
+#define NV50_TIC_0_FMT_1_5_5_5					0x00000014
+#define NV50_TIC_0_FMT_5_6_5					0x00000015
+#define NV50_TIC_0_FMT_6_5_5					0x00000016
+#define NV50_TIC_0_FMT_8_8					0x00000018
+#define NV50_TIC_0_FMT_16					0x0000001b
+#define NV50_TIC_0_FMT_8					0x0000001d
+#define NV50_TIC_0_FMT_4_4					0x0000001e
+#define NV50_TIC_0_FMT_UNK1F					0x0000001f
+#define NV50_TIC_0_FMT_E5_9_9_9					0x00000020
+#define NV50_TIC_0_FMT_10_11_11					0x00000021
+#define NV50_TIC_0_FMT_C1_C2_C1_C0				0x00000022
+#define NV50_TIC_0_FMT_C2_C1_C0_C1				0x00000023
+#define NV50_TIC_0_FMT_DXT1					0x00000024
+#define NV50_TIC_0_FMT_DXT3					0x00000025
+#define NV50_TIC_0_FMT_DXT5					0x00000026
+#define NV50_TIC_0_FMT_RGTC1					0x00000027
+#define NV50_TIC_0_FMT_RGTC2					0x00000028
+#define NV50_TIC_0_FMT_24_8_ZETA				0x00000029
+#define NV50_TIC_0_FMT_8_24_ZETA				0x0000002a
+#define NV50_TIC_0_FMT_UNK2C_ZETA				0x0000002c
+#define NV50_TIC_0_FMT_UNK2D_ZETA				0x0000002d
+#define NV50_TIC_0_FMT_UNK2E_ZETA				0x0000002e
+#define NV50_TIC_0_FMT_32_ZETA					0x0000002f
+#define NV50_TIC_0_FMT_32_8_ZETA				0x00000030
+#define NV50_TIC_0_FMT_16_ZETA				0x0000003a
+
+#define NV50_TIC_1						0x00000004
+#define NV50_TIC_1_OFFSET_LOW__MASK				0xffffffff
+#define NV50_TIC_1_OFFSET_LOW__SHIFT				0
+
+#define NV50_TIC_2						0x00000008
+#define NV50_TIC_2_OFFSET_HIGH__MASK				0x000000ff
+#define NV50_TIC_2_OFFSET_HIGH__SHIFT				0
+#define NV50_TIC_2_COLORSPACE_SRGB				0x00000400
+#define NV50_TIC_2_TARGET__MASK					0x0003c000
+#define NV50_TIC_2_TARGET__SHIFT				14
+#define NV50_TIC_2_TARGET_1D					0x00000000
+#define NV50_TIC_2_TARGET_2D					0x00004000
+#define NV50_TIC_2_TARGET_3D					0x00008000
+#define NV50_TIC_2_TARGET_CUBE					0x0000c000
+#define NV50_TIC_2_TARGET_1D_ARRAY				0x00010000
+#define NV50_TIC_2_TARGET_2D_ARRAY				0x00014000
+#define NV50_TIC_2_TARGET_BUFFER				0x00018000
+#define NV50_TIC_2_TARGET_RECT					0x0001c000
+#define NV50_TIC_2_TARGET_CUBE_ARRAY				0x00020000
+#define NV50_TIC_2_TILE_MODE_LINEAR				0x00040000
+#define NV50_TIC_2_TILE_MODE_Y__MASK				0x01c00000
+#define NV50_TIC_2_TILE_MODE_Y__SHIFT				22
+#define NV50_TIC_2_TILE_MODE_Z__MASK				0x0e000000
+#define NV50_TIC_2_TILE_MODE_Z__SHIFT				25
+#define NV50_TIC_2_2D_UNK0258__MASK				0x30000000
+#define NV50_TIC_2_2D_UNK0258__SHIFT				28
+#define NV50_TIC_2_NORMALIZED_COORDS				0x80000000
+
+#define NV50_TIC_3						0x0000000c
+#define NV50_TIC_3_PITCH__MASK					0xffffffff
+#define NV50_TIC_3_PITCH__SHIFT					0
+
+#define NV50_TIC_4						0x00000010
+#define NV50_TIC_4_WIDTH__MASK					0xffffffff
+#define NV50_TIC_4_WIDTH__SHIFT					0
+
+#define NV50_TIC_5						0x00000014
+#define NV50_TIC_5_LAST_LEVEL__MASK				0xf0000000
+#define NV50_TIC_5_LAST_LEVEL__SHIFT				28
+#define NV50_TIC_5_DEPTH__MASK					0x0fff0000
+#define NV50_TIC_5_DEPTH__SHIFT					16
+#define NV50_TIC_5_HEIGHT__MASK					0x0000ffff
+#define NV50_TIC_5_HEIGHT__SHIFT				0
+
+#define NV50_TIC_7						0x0000001c
+#define NV50_TIC_7_BASE_LEVEL__MASK				0x0000000f
+#define NV50_TIC_7_BASE_LEVEL__SHIFT				0
+#define NV50_TIC_7_MAX_LEVEL__MASK				0x000000f0
+#define NV50_TIC_7_MAX_LEVEL__SHIFT				4
+
+#define NV50_TSC__SIZE						0x00000020
+#define NV50_TSC_0						0x00000000
+#define NV50_TSC_0_WRAPS__MASK					0x00000007
+#define NV50_TSC_0_WRAPS__SHIFT					0
+#define NV50_TSC_0_WRAPT__MASK					0x00000038
+#define NV50_TSC_0_WRAPT__SHIFT					3
+#define NV50_TSC_0_WRAPR__MASK					0x000001c0
+#define NV50_TSC_0_WRAPR__SHIFT					6
+#define NV50_TSC_0_SHADOW_COMPARE_ENABLE			0x00000200
+#define NV50_TSC_0_SHADOW_COMPARE_FUNC__MASK			0x00001c00
+#define NV50_TSC_0_SHADOW_COMPARE_FUNC__SHIFT			10
+#define NV50_TSC_0_ANISOTROPY_MASK__MASK			0x00700000
+#define NV50_TSC_0_ANISOTROPY_MASK__SHIFT			20
+
+#define NV50_TSC_1						0x00000004
+#define NV50_TSC_1_UNKN_ANISO_15				0x10000000
+#define NV50_TSC_1_UNKN_ANISO_35				0x18000000
+#define NV50_TSC_1_MAGF__MASK					0x00000003
+#define NV50_TSC_1_MAGF__SHIFT					0
+#define NV50_TSC_1_MAGF_NEAREST					0x00000001
+#define NV50_TSC_1_MAGF_LINEAR					0x00000002
+#define NV50_TSC_1_MINF__MASK					0x00000030
+#define NV50_TSC_1_MINF__SHIFT					4
+#define NV50_TSC_1_MINF_NEAREST					0x00000010
+#define NV50_TSC_1_MINF_LINEAR					0x00000020
+#define NV50_TSC_1_MIPF__MASK					0x000000c0
+#define NV50_TSC_1_MIPF__SHIFT					6
+#define NV50_TSC_1_MIPF_NONE					0x00000040
+#define NV50_TSC_1_MIPF_NEAREST					0x00000080
+#define NV50_TSC_1_MIPF_LINEAR					0x000000c0
+#define NV50_TSC_1_LOD_BIAS__MASK				0x01fff000
+#define NV50_TSC_1_LOD_BIAS__SHIFT				12
+
+#define NV50_TSC_2						0x00000008
+#define NV50_TSC_2_MIN_LOD__MASK				0x00000f00
+#define NV50_TSC_2_MIN_LOD__SHIFT				8
+#define NV50_TSC_2_MAX_LOD__MASK				0x00f00000
+#define NV50_TSC_2_MAX_LOD__SHIFT				20
+
+#define NV50_TSC_4						0x00000010
+#define NV50_TSC_4_BORDER_COLOR_RED__MASK			0xffffffff
+#define NV50_TSC_4_BORDER_COLOR_RED__SHIFT			0
+
+#define NV50_TSC_5						0x00000014
+#define NV50_TSC_5_BORDER_COLOR_GREEN__MASK			0xffffffff
+#define NV50_TSC_5_BORDER_COLOR_GREEN__SHIFT			0
+
+#define NV50_TSC_6						0x00000018
+#define NV50_TSC_6_BORDER_COLOR_BLUE__MASK			0xffffffff
+#define NV50_TSC_6_BORDER_COLOR_BLUE__SHIFT			0
+
+#define NV50_TSC_7						0x0000001c
+#define NV50_TSC_7_BORDER_COLOR_ALPHA__MASK			0xffffffff
+#define NV50_TSC_7_BORDER_COLOR_ALPHA__SHIFT			0
+
+
+#endif /* NV50_TEXTURE_XML */
diff --git a/src/gallium/drivers/nvc0/nvc0_2d.xml.h b/src/gallium/drivers/nvc0/nvc0_2d.xml.h
new file mode 100644
index 00000000000..aebcd510e8e
--- /dev/null
+++ b/src/gallium/drivers/nvc0/nvc0_2d.xml.h
@@ -0,0 +1,380 @@
+#ifndef NVC0_2D_XML
+#define NVC0_2D_XML
+
+/* Autogenerated file, DO NOT EDIT manually!
+
+This file was generated by the rules-ng-ng headergen tool in this git repository:
+http://0x04.net/cgit/index.cgi/rules-ng-ng
+git clone git://0x04.net/rules-ng-ng
+
+The rules-ng-ng source files this header was generated from are:
+- nvc0_2d.xml    (   9454 bytes, from 2010-10-16 16:03:11)
+- copyright.xml  (   6498 bytes, from 2010-10-03 13:18:37)
+- nv_object.xml  (  11379 bytes, from 2010-10-16 11:43:24)
+- nvchipsets.xml (   2907 bytes, from 2010-10-15 16:28:21)
+- nv_defs.xml    (   4437 bytes, from 2010-07-06 07:43:58)
+- nv50_defs.xml  (   4482 bytes, from 2010-10-03 13:18:37)
+
+Copyright (C) 2006-2010 by the following authors:
+- Artur Huillet <[email protected]> (ahuillet)
+- Ben Skeggs (darktama, darktama_)
+- B. R. <[email protected]> (koala_br)
+- Carlos Martin <[email protected]> (carlosmn)
+- Christoph Bumiller <[email protected]> (calim, chrisbmr)
+- Dawid Gajownik <[email protected]> (gajownik)
+- Dmitry Baryshkov
+- Dmitry Eremin-Solenikov <[email protected]> (lumag)
+- EdB <[email protected]> (edb_)
+- Erik Waling <[email protected]> (erikwaling)
+- Francisco Jerez <[email protected]> (curro, curro_, currojerez)
+- imirkin <[email protected]> (imirkin)
+- jb17bsome <[email protected]> (jb17bsome)
+- Jeremy Kolb <[email protected]> (kjeremy)
+- Laurent Carlier <[email protected]> (lordheavy)
+- Luca Barbieri <[email protected]> (lb, lb1)
+- Maarten Maathuis <[email protected]> (stillunknown)
+- Marcin Kościelnicki <[email protected]> (mwk, koriakin)
+- Mark Carey <[email protected]> (careym)
+- Matthieu Castet <[email protected]> (mat-c)
+- nvidiaman <[email protected]> (nvidiaman)
+- Patrice Mandin <[email protected]> (pmandin, pmdata)
+- Pekka Paalanen <[email protected]> (pq, ppaalanen)
+- Peter Popov <[email protected]> (ironpeter)
+- Richard Hughes <[email protected]> (hughsient)
+- Rudi Cilibrasi <[email protected]> (cilibrar)
+- Serge Martin
+- Simon Raffeiner
+- Stephane Loeuillet <[email protected]> (leroutier)
+- Stephane Marchesin <[email protected]> (marcheu)
+- sturmflut <[email protected]> (sturmflut)
+- Sylvain Munaut <[email protected]>
+- Victor Stinner <[email protected]> (haypo)
+- Wladmir van der Laan <[email protected]> (miathan6)
+- Younes Manton <[email protected]> (ymanton)
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice (including the
+next paragraph) shall be included in all copies or substantial
+portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/
+
+
+
+#define NVC0_2D_DST_FORMAT					0x00000200
+
+#define NVC0_2D_DST_LINEAR					0x00000204
+
+#define NVC0_2D_DST_TILE_MODE					0x00000208
+
+#define NVC0_2D_DST_DEPTH					0x0000020c
+
+#define NVC0_2D_DST_LAYER					0x00000210
+
+#define NVC0_2D_DST_PITCH					0x00000214
+
+#define NVC0_2D_DST_WIDTH					0x00000218
+
+#define NVC0_2D_DST_HEIGHT					0x0000021c
+
+#define NVC0_2D_DST_ADDRESS_HIGH				0x00000220
+
+#define NVC0_2D_DST_ADDRESS_LOW					0x00000224
+
+#define NVC0_2D_UNK228						0x00000228
+
+#define NVC0_2D_SRC_FORMAT					0x00000230
+
+#define NVC0_2D_SRC_LINEAR					0x00000234
+
+#define NVC0_2D_SRC_TILE_MODE					0x00000238
+
+#define NVC0_2D_SRC_DEPTH					0x0000023c
+
+#define NVC0_2D_SRC_LAYER					0x00000240
+
+#define NVC0_2D_SRC_PITCH					0x00000244
+#define NVC0_2D_SRC_PITCH__MAX					0x00040000
+
+#define NVC0_2D_SRC_WIDTH					0x00000248
+#define NVC0_2D_SRC_WIDTH__MAX					0x00010000
+
+#define NVC0_2D_SRC_HEIGHT					0x0000024c
+#define NVC0_2D_SRC_HEIGHT__MAX					0x00010000
+
+#define NVC0_2D_SRC_ADDRESS_HIGH				0x00000250
+
+#define NVC0_2D_SRC_ADDRESS_LOW					0x00000254
+
+#define NVC0_2D_UNK258						0x00000258
+
+#define NVC0_2D_UNK260						0x00000260
+
+#define NVC0_2D_COND_ADDRESS_HIGH				0x00000264
+
+#define NVC0_2D_COND_ADDRESS_LOW				0x00000268
+
+#define NVC0_2D_COND_MODE					0x0000026c
+#define NVC0_2D_COND_MODE_NEVER					0x00000000
+#define NVC0_2D_COND_MODE_ALWAYS				0x00000001
+#define NVC0_2D_COND_MODE_RES_NON_ZERO				0x00000002
+#define NVC0_2D_COND_MODE_EQUAL					0x00000003
+#define NVC0_2D_COND_MODE_NOT_EQUAL				0x00000004
+
+#define NVC0_2D_CLIP_X						0x00000280
+
+#define NVC0_2D_CLIP_Y						0x00000284
+
+#define NVC0_2D_CLIP_W						0x00000288
+
+#define NVC0_2D_CLIP_H						0x0000028c
+
+#define NVC0_2D_CLIP_ENABLE					0x00000290
+
+#define NVC0_2D_COLOR_KEY_FORMAT				0x00000294
+#define NVC0_2D_COLOR_KEY_FORMAT_16BPP				0x00000000
+#define NVC0_2D_COLOR_KEY_FORMAT_15BPP				0x00000001
+#define NVC0_2D_COLOR_KEY_FORMAT_24BPP				0x00000002
+#define NVC0_2D_COLOR_KEY_FORMAT_30BPP				0x00000003
+#define NVC0_2D_COLOR_KEY_FORMAT_8BPP				0x00000004
+#define NVC0_2D_COLOR_KEY_FORMAT_16BPP2				0x00000005
+#define NVC0_2D_COLOR_KEY_FORMAT_32BPP				0x00000006
+
+#define NVC0_2D_COLOR_KEY					0x00000298
+
+#define NVC0_2D_COLOR_KEY_ENABLE				0x0000029c
+
+#define NVC0_2D_ROP						0x000002a0
+
+#define NVC0_2D_BETA1						0x000002a4
+
+#define NVC0_2D_BETA4						0x000002a8
+
+#define NVC0_2D_OPERATION					0x000002ac
+#define NVC0_2D_OPERATION_SRCCOPY_AND				0x00000000
+#define NVC0_2D_OPERATION_ROP_AND				0x00000001
+#define NVC0_2D_OPERATION_BLEND_AND				0x00000002
+#define NVC0_2D_OPERATION_SRCCOPY				0x00000003
+#define NVC0_2D_OPERATION_UNK4					0x00000004
+#define NVC0_2D_OPERATION_SRCCOPY_PREMULT			0x00000005
+#define NVC0_2D_OPERATION_BLEND_PREMULT				0x00000006
+
+#define NVC0_2D_UNK2B0						0x000002b0
+#define NVC0_2D_UNK2B0_UNK0__MASK				0x0000003f
+#define NVC0_2D_UNK2B0_UNK0__SHIFT				0
+#define NVC0_2D_UNK2B0_UNK1__MASK				0x00003f00
+#define NVC0_2D_UNK2B0_UNK1__SHIFT				8
+
+#define NVC0_2D_PATTERN_SELECT					0x000002b4
+#define NVC0_2D_PATTERN_SELECT_MONO_8X8				0x00000000
+#define NVC0_2D_PATTERN_SELECT_MONO_64X1			0x00000001
+#define NVC0_2D_PATTERN_SELECT_MONO_1X64			0x00000002
+#define NVC0_2D_PATTERN_SELECT_COLOR				0x00000003
+
+#define NVC0_2D_PATTERN_COLOR_FORMAT				0x000002e8
+#define NVC0_2D_PATTERN_COLOR_FORMAT_16BPP			0x00000000
+#define NVC0_2D_PATTERN_COLOR_FORMAT_15BPP			0x00000001
+#define NVC0_2D_PATTERN_COLOR_FORMAT_32BPP			0x00000002
+#define NVC0_2D_PATTERN_COLOR_FORMAT_8BPP			0x00000003
+#define NVC0_2D_PATTERN_COLOR_FORMAT_UNK4			0x00000004
+#define NVC0_2D_PATTERN_COLOR_FORMAT_UNK5			0x00000005
+
+#define NVC0_2D_PATTERN_MONO_FORMAT				0x000002ec
+#define NVC0_2D_PATTERN_MONO_FORMAT_CGA6			0x00000000
+#define NVC0_2D_PATTERN_MONO_FORMAT_LE				0x00000001
+
+#define NVC0_2D_PATTERN_COLOR(i0)			       (0x000002f0 + 0x4*(i0))
+#define NVC0_2D_PATTERN_COLOR__ESIZE				0x00000004
+#define NVC0_2D_PATTERN_COLOR__LEN				0x00000002
+
+#define NVC0_2D_PATTERN_BITMAP(i0)			       (0x000002f8 + 0x4*(i0))
+#define NVC0_2D_PATTERN_BITMAP__ESIZE				0x00000004
+#define NVC0_2D_PATTERN_BITMAP__LEN				0x00000002
+
+#define NVC0_2D_PATTERN_X8R8G8B8(i0)			       (0x00000300 + 0x4*(i0))
+#define NVC0_2D_PATTERN_X8R8G8B8__ESIZE				0x00000004
+#define NVC0_2D_PATTERN_X8R8G8B8__LEN				0x00000040
+#define NVC0_2D_PATTERN_X8R8G8B8_B__MASK			0x000000ff
+#define NVC0_2D_PATTERN_X8R8G8B8_B__SHIFT			0
+#define NVC0_2D_PATTERN_X8R8G8B8_G__MASK			0x0000ff00
+#define NVC0_2D_PATTERN_X8R8G8B8_G__SHIFT			8
+#define NVC0_2D_PATTERN_X8R8G8B8_R__MASK			0x00ff0000
+#define NVC0_2D_PATTERN_X8R8G8B8_R__SHIFT			16
+
+#define NVC0_2D_PATTERN_R5G6B5(i0)			       (0x00000400 + 0x4*(i0))
+#define NVC0_2D_PATTERN_R5G6B5__ESIZE				0x00000004
+#define NVC0_2D_PATTERN_R5G6B5__LEN				0x00000020
+#define NVC0_2D_PATTERN_R5G6B5_B0__MASK				0x0000001f
+#define NVC0_2D_PATTERN_R5G6B5_B0__SHIFT			0
+#define NVC0_2D_PATTERN_R5G6B5_G0__MASK				0x000007e0
+#define NVC0_2D_PATTERN_R5G6B5_G0__SHIFT			5
+#define NVC0_2D_PATTERN_R5G6B5_R0__MASK				0x0000f800
+#define NVC0_2D_PATTERN_R5G6B5_R0__SHIFT			11
+#define NVC0_2D_PATTERN_R5G6B5_B1__MASK				0x001f0000
+#define NVC0_2D_PATTERN_R5G6B5_B1__SHIFT			16
+#define NVC0_2D_PATTERN_R5G6B5_G1__MASK				0x07e00000
+#define NVC0_2D_PATTERN_R5G6B5_G1__SHIFT			21
+#define NVC0_2D_PATTERN_R5G6B5_R1__MASK				0xf8000000
+#define NVC0_2D_PATTERN_R5G6B5_R1__SHIFT			27
+
+#define NVC0_2D_PATTERN_X1R5G5B5(i0)			       (0x00000480 + 0x4*(i0))
+#define NVC0_2D_PATTERN_X1R5G5B5__ESIZE				0x00000004
+#define NVC0_2D_PATTERN_X1R5G5B5__LEN				0x00000020
+#define NVC0_2D_PATTERN_X1R5G5B5_B0__MASK			0x0000001f
+#define NVC0_2D_PATTERN_X1R5G5B5_B0__SHIFT			0
+#define NVC0_2D_PATTERN_X1R5G5B5_G0__MASK			0x000003e0
+#define NVC0_2D_PATTERN_X1R5G5B5_G0__SHIFT			5
+#define NVC0_2D_PATTERN_X1R5G5B5_R0__MASK			0x00007c00
+#define NVC0_2D_PATTERN_X1R5G5B5_R0__SHIFT			10
+#define NVC0_2D_PATTERN_X1R5G5B5_B1__MASK			0x001f0000
+#define NVC0_2D_PATTERN_X1R5G5B5_B1__SHIFT			16
+#define NVC0_2D_PATTERN_X1R5G5B5_G1__MASK			0x03e00000
+#define NVC0_2D_PATTERN_X1R5G5B5_G1__SHIFT			21
+#define NVC0_2D_PATTERN_X1R5G5B5_R1__MASK			0x7c000000
+#define NVC0_2D_PATTERN_X1R5G5B5_R1__SHIFT			26
+
+#define NVC0_2D_PATTERN_Y8(i0)				       (0x00000500 + 0x4*(i0))
+#define NVC0_2D_PATTERN_Y8__ESIZE				0x00000004
+#define NVC0_2D_PATTERN_Y8__LEN					0x00000010
+#define NVC0_2D_PATTERN_Y8_Y0__MASK				0x000000ff
+#define NVC0_2D_PATTERN_Y8_Y0__SHIFT				0
+#define NVC0_2D_PATTERN_Y8_Y1__MASK				0x0000ff00
+#define NVC0_2D_PATTERN_Y8_Y1__SHIFT				8
+#define NVC0_2D_PATTERN_Y8_Y2__MASK				0x00ff0000
+#define NVC0_2D_PATTERN_Y8_Y2__SHIFT				16
+#define NVC0_2D_PATTERN_Y8_Y3__MASK				0xff000000
+#define NVC0_2D_PATTERN_Y8_Y3__SHIFT				24
+
+#define NVC0_2D_DRAW_SHAPE					0x00000580
+#define NVC0_2D_DRAW_SHAPE_POINTS				0x00000000
+#define NVC0_2D_DRAW_SHAPE_LINES				0x00000001
+#define NVC0_2D_DRAW_SHAPE_LINE_STRIP				0x00000002
+#define NVC0_2D_DRAW_SHAPE_TRIANGLES				0x00000003
+#define NVC0_2D_DRAW_SHAPE_RECTANGLES				0x00000004
+
+#define NVC0_2D_DRAW_COLOR_FORMAT				0x00000584
+
+#define NVC0_2D_DRAW_COLOR					0x00000588
+
+#define NVC0_2D_UNK58C						0x0000058c
+#define NVC0_2D_UNK58C_0					0x00000001
+#define NVC0_2D_UNK58C_1					0x00000010
+#define NVC0_2D_UNK58C_2					0x00000100
+#define NVC0_2D_UNK58C_3					0x00001000
+
+#define NVC0_2D_DRAW_POINT16					0x000005e0
+#define NVC0_2D_DRAW_POINT16_X__MASK				0x0000ffff
+#define NVC0_2D_DRAW_POINT16_X__SHIFT				0
+#define NVC0_2D_DRAW_POINT16_Y__MASK				0xffff0000
+#define NVC0_2D_DRAW_POINT16_Y__SHIFT				16
+
+#define NVC0_2D_DRAW_POINT32_X(i0)			       (0x00000600 + 0x8*(i0))
+#define NVC0_2D_DRAW_POINT32_X__ESIZE				0x00000008
+#define NVC0_2D_DRAW_POINT32_X__LEN				0x00000040
+
+#define NVC0_2D_DRAW_POINT32_Y(i0)			       (0x00000604 + 0x8*(i0))
+#define NVC0_2D_DRAW_POINT32_Y__ESIZE				0x00000008
+#define NVC0_2D_DRAW_POINT32_Y__LEN				0x00000040
+
+#define NVC0_2D_SIFC_BITMAP_ENABLE				0x00000800
+
+#define NVC0_2D_SIFC_FORMAT					0x00000804
+
+#define NVC0_2D_SIFC_BITMAP_FORMAT				0x00000808
+#define NVC0_2D_SIFC_BITMAP_FORMAT_I1				0x00000000
+#define NVC0_2D_SIFC_BITMAP_FORMAT_I4				0x00000001
+#define NVC0_2D_SIFC_BITMAP_FORMAT_I8				0x00000002
+
+#define NVC0_2D_SIFC_BITMAP_LSB_FIRST				0x0000080c
+
+#define NVC0_2D_SIFC_BITMAP_LINE_PACK_MODE			0x00000810
+#define NVC0_2D_SIFC_BITMAP_LINE_PACK_MODE_PACKED		0x00000000
+#define NVC0_2D_SIFC_BITMAP_LINE_PACK_MODE_ALIGN_BYTE		0x00000001
+#define NVC0_2D_SIFC_BITMAP_LINE_PACK_MODE_ALIGN_WORD		0x00000002
+
+#define NVC0_2D_SIFC_BITMAP_COLOR_BIT0				0x00000814
+
+#define NVC0_2D_SIFC_BITMAP_COLOR_BIT1				0x00000818
+
+#define NVC0_2D_SIFC_BITMAP_WRITE_BIT0_ENABLE			0x0000081c
+
+#define NVC0_2D_SIFC_WIDTH					0x00000838
+
+#define NVC0_2D_SIFC_HEIGHT					0x0000083c
+
+#define NVC0_2D_SIFC_DX_DU_FRACT				0x00000840
+
+#define NVC0_2D_SIFC_DX_DU_INT					0x00000844
+
+#define NVC0_2D_SIFC_DY_DV_FRACT				0x00000848
+
+#define NVC0_2D_SIFC_DY_DV_INT					0x0000084c
+
+#define NVC0_2D_SIFC_DST_X_FRACT				0x00000850
+
+#define NVC0_2D_SIFC_DST_X_INT					0x00000854
+
+#define NVC0_2D_SIFC_DST_Y_FRACT				0x00000858
+
+#define NVC0_2D_SIFC_DST_Y_INT					0x0000085c
+
+#define NVC0_2D_SIFC_DATA					0x00000860
+
+#define NVC0_2D_UNK0870						0x00000870
+
+#define NVC0_2D_UNK0880						0x00000880
+
+#define NVC0_2D_UNK0884						0x00000884
+
+#define NVC0_2D_UNK0888						0x00000888
+
+#define NVC0_2D_BLIT_CONTROL					0x0000088c
+#define NVC0_2D_BLIT_CONTROL_ORIGIN__MASK			0x00000001
+#define NVC0_2D_BLIT_CONTROL_ORIGIN__SHIFT			0
+#define NVC0_2D_BLIT_CONTROL_ORIGIN_CENTER			0x00000000
+#define NVC0_2D_BLIT_CONTROL_ORIGIN_CORNER			0x00000001
+#define NVC0_2D_BLIT_CONTROL_FILTER__MASK			0x00000010
+#define NVC0_2D_BLIT_CONTROL_FILTER__SHIFT			4
+#define NVC0_2D_BLIT_CONTROL_FILTER_POINT_SAMPLE		0x00000000
+#define NVC0_2D_BLIT_CONTROL_FILTER_BILINEAR			0x00000010
+
+#define NVC0_2D_BLIT_DST_X					0x000008b0
+
+#define NVC0_2D_BLIT_DST_Y					0x000008b4
+
+#define NVC0_2D_BLIT_DST_W					0x000008b8
+
+#define NVC0_2D_BLIT_DST_H					0x000008bc
+
+#define NVC0_2D_BLIT_DU_DX_FRACT				0x000008c0
+
+#define NVC0_2D_BLIT_DU_DX_INT					0x000008c4
+
+#define NVC0_2D_BLIT_DV_DY_FRACT				0x000008c8
+
+#define NVC0_2D_BLIT_DV_DY_INT					0x000008cc
+
+#define NVC0_2D_BLIT_SRC_X_FRACT				0x000008d0
+
+#define NVC0_2D_BLIT_SRC_X_INT					0x000008d4
+
+#define NVC0_2D_BLIT_SRC_Y_FRACT				0x000008d8
+
+#define NVC0_2D_BLIT_SRC_Y_INT					0x000008dc
+
+
+#endif /* NVC0_2D_XML */
diff --git a/src/gallium/drivers/nvc0/nvc0_3d.xml.h b/src/gallium/drivers/nvc0/nvc0_3d.xml.h
new file mode 100644
index 00000000000..1346d999409
--- /dev/null
+++ b/src/gallium/drivers/nvc0/nvc0_3d.xml.h
@@ -0,0 +1,1101 @@
+#ifndef NVC0_3D_XML
+#define NVC0_3D_XML
+
+/* Autogenerated file, DO NOT EDIT manually!
+
+This file was generated by the rules-ng-ng headergen tool in this git repository:
+http://0x04.net/cgit/index.cgi/rules-ng-ng
+git clone git://0x04.net/rules-ng-ng
+
+The rules-ng-ng source files this header was generated from are:
+- nvc0_3d.xml    (  28058 bytes, from 2010-11-26 18:05:20)
+- copyright.xml  (   6452 bytes, from 2010-11-25 23:28:20)
+- nv_defs.xml    (   4437 bytes, from 2010-07-06 07:43:58)
+- nv_3ddefs.xml  (  16394 bytes, from 2010-10-09 08:27:14)
+- nv_object.xml  (  11547 bytes, from 2010-11-26 16:41:56)
+- nvchipsets.xml (   3074 bytes, from 2010-11-07 00:36:28)
+- nv50_defs.xml  (   4482 bytes, from 2010-10-03 13:18:37)
+
+Copyright (C) 2006-2010 by the following authors:
+- Artur Huillet <[email protected]> (ahuillet)
+- Ben Skeggs (darktama, darktama_)
+- B. R. <[email protected]> (koala_br)
+- Carlos Martin <[email protected]> (carlosmn)
+- Christoph Bumiller <[email protected]> (calim, chrisbmr)
+- Dawid Gajownik <[email protected]> (gajownik)
+- Dmitry Baryshkov
+- Dmitry Eremin-Solenikov <[email protected]> (lumag)
+- EdB <[email protected]> (edb_)
+- Erik Waling <[email protected]> (erikwaling)
+- Francisco Jerez <[email protected]> (curro)
+- imirkin <[email protected]> (imirkin)
+- jb17bsome <[email protected]> (jb17bsome)
+- Jeremy Kolb <[email protected]> (kjeremy)
+- Laurent Carlier <[email protected]> (lordheavy)
+- Luca Barbieri <[email protected]> (lb, lb1)
+- Maarten Maathuis <[email protected]> (stillunknown)
+- Marcin Kościelnicki <[email protected]> (mwk, koriakin)
+- Mark Carey <[email protected]> (careym)
+- Matthieu Castet <[email protected]> (mat-c)
+- nvidiaman <[email protected]> (nvidiaman)
+- Patrice Mandin <[email protected]> (pmandin, pmdata)
+- Pekka Paalanen <[email protected]> (pq, ppaalanen)
+- Peter Popov <[email protected]> (ironpeter)
+- Richard Hughes <[email protected]> (hughsient)
+- Rudi Cilibrasi <[email protected]> (cilibrar)
+- Serge Martin
+- Simon Raffeiner
+- Stephane Loeuillet <[email protected]> (leroutier)
+- Stephane Marchesin <[email protected]> (marcheu)
+- sturmflut <[email protected]> (sturmflut)
+- Sylvain Munaut <[email protected]>
+- Victor Stinner <[email protected]> (haypo)
+- Wladmir van der Laan <[email protected]> (miathan6)
+- Younes Manton <[email protected]> (ymanton)
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice (including the
+next paragraph) shall be included in all copies or substantial
+portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/
+
+
+
+#define NVC0_3D_NOTIFY_ADDRESS_HIGH				0x00000104
+#define NVC0_3D_NOTIFY_ADDRESS_LOW				0x00000108
+#define NVC0_3D_NOTIFY						0x0000010c
+
+#define NVC0_3D_SERIALIZE					0x00000110
+
+#define NVC0_3D_EARLY_FRAGMENT_TESTS				0x00000210
+
+#define NVC0_3D_TESS_MODE					0x00000320
+#define NVC0_3D_TESS_MODE_PRIM__MASK				0x0000000f
+#define NVC0_3D_TESS_MODE_PRIM__SHIFT				0
+#define NVC0_3D_TESS_MODE_PRIM_ISOLINES				0x00000000
+#define NVC0_3D_TESS_MODE_PRIM_TRIANGLES			0x00000001
+#define NVC0_3D_TESS_MODE_PRIM_QUADS				0x00000002
+#define NVC0_3D_TESS_MODE_SPACING__MASK				0x000000f0
+#define NVC0_3D_TESS_MODE_SPACING__SHIFT			4
+#define NVC0_3D_TESS_MODE_SPACING_EQUAL				0x00000000
+#define NVC0_3D_TESS_MODE_SPACING_FRACTIONAL_ODD		0x00000010
+#define NVC0_3D_TESS_MODE_SPACING_FRACTIONAL_EVEN		0x00000020
+#define NVC0_3D_TESS_MODE_CW					0x00000100
+#define NVC0_3D_TESS_MODE_CONNECTED				0x00000200
+
+#define NVC0_3D_TESS_LEVEL_OUTER(i0)			       (0x00000324 + 0x4*(i0))
+#define NVC0_3D_TESS_LEVEL_OUTER__ESIZE				0x00000004
+#define NVC0_3D_TESS_LEVEL_OUTER__LEN				0x00000004
+
+#define NVC0_3D_TESS_LEVEL_INNER(i0)			       (0x00000334 + 0x4*(i0))
+#define NVC0_3D_TESS_LEVEL_INNER__ESIZE				0x00000004
+#define NVC0_3D_TESS_LEVEL_INNER__LEN				0x00000002
+
+#define NVC0_3D_RASTERIZE_ENABLE				0x0000037c
+
+#define NVC0_3D_TFB(i0)					       (0x00000380 + 0x20*(i0))
+#define NVC0_3D_TFB__ESIZE					0x00000020
+#define NVC0_3D_TFB__LEN					0x00000004
+
+#define NVC0_3D_TFB_BUFFER_ENABLE(i0)			       (0x00000380 + 0x20*(i0))
+
+#define NVC0_3D_TFB_ADDRESS_HIGH(i0)			       (0x00000384 + 0x20*(i0))
+
+#define NVC0_3D_TFB_ADDRESS_LOW(i0)			       (0x00000388 + 0x20*(i0))
+
+#define NVC0_3D_TFB_BUFFER_SIZE(i0)			       (0x0000038c + 0x20*(i0))
+
+#define NVC0_3D_TFB_PRIMITIVE_ID(i0)			       (0x00000390 + 0x20*(i0))
+
+#define NVC0_3D_TFB_UNK0700(i0)				       (0x00000700 + 0x10*(i0))
+
+#define NVC0_3D_TFB_VARYING_COUNT(i0)			       (0x00000704 + 0x10*(i0))
+
+#define NVC0_3D_TFB_BUFFER_STRIDE(i0)			       (0x00000708 + 0x10*(i0))
+
+#define NVC0_3D_TFB_ENABLE					0x00000744
+
+#define NVC0_3D_LOCAL_BASE					0x0000077c
+
+#define NVC0_3D_LOCAL_ADDRESS_HIGH				0x00000790
+
+#define NVC0_3D_LOCAL_ADDRESS_LOW				0x00000794
+
+#define NVC0_3D_LOCAL_SIZE_HIGH					0x00000798
+
+#define NVC0_3D_LOCAL_SIZE_LOW					0x0000079c
+
+#define NVC0_3D_RT(i0)					       (0x00000800 + 0x20*(i0))
+#define NVC0_3D_RT__ESIZE					0x00000020
+#define NVC0_3D_RT__LEN						0x00000008
+
+#define NVC0_3D_RT_ADDRESS_HIGH(i0)			       (0x00000800 + 0x20*(i0))
+
+#define NVC0_3D_RT_ADDRESS_LOW(i0)			       (0x00000804 + 0x20*(i0))
+
+#define NVC0_3D_RT_HORIZ(i0)				       (0x00000808 + 0x20*(i0))
+
+#define NVC0_3D_RT_VERT(i0)				       (0x0000080c + 0x20*(i0))
+
+#define NVC0_3D_RT_FORMAT(i0)				       (0x00000810 + 0x20*(i0))
+
+#define NVC0_3D_RT_TILE_MODE(i0)			       (0x00000814 + 0x20*(i0))
+#define NVC0_3D_RT_TILE_MODE_UNK0				0x00000001
+#define NVC0_3D_RT_TILE_MODE_Y__MASK				0x00000070
+#define NVC0_3D_RT_TILE_MODE_Y__SHIFT				4
+#define NVC0_3D_RT_TILE_MODE_Z__MASK				0x00000700
+#define NVC0_3D_RT_TILE_MODE_Z__SHIFT				8
+
+#define NVC0_3D_RT_ARRAY_MODE(i0)			       (0x00000818 + 0x20*(i0))
+#define NVC0_3D_RT_ARRAY_MODE_LAYERS__MASK			0x0000ffff
+#define NVC0_3D_RT_ARRAY_MODE_LAYERS__SHIFT			0
+#define NVC0_3D_RT_ARRAY_MODE_VOLUME				0x00010000
+
+#define NVC0_3D_RT_LAYER_STRIDE(i0)			       (0x0000081c + 0x20*(i0))
+
+#define NVC0_3D_VIEWPORT_SCALE_X(i0)			       (0x00000a00 + 0x20*(i0))
+#define NVC0_3D_VIEWPORT_SCALE_X__ESIZE				0x00000020
+#define NVC0_3D_VIEWPORT_SCALE_X__LEN				0x00000010
+
+#define NVC0_3D_VIEWPORT_SCALE_Y(i0)			       (0x00000a04 + 0x20*(i0))
+#define NVC0_3D_VIEWPORT_SCALE_Y__ESIZE				0x00000020
+#define NVC0_3D_VIEWPORT_SCALE_Y__LEN				0x00000010
+
+#define NVC0_3D_VIEWPORT_SCALE_Z(i0)			       (0x00000a08 + 0x20*(i0))
+#define NVC0_3D_VIEWPORT_SCALE_Z__ESIZE				0x00000020
+#define NVC0_3D_VIEWPORT_SCALE_Z__LEN				0x00000010
+
+#define NVC0_3D_VIEWPORT_TRANSLATE_X(i0)		       (0x00000a0c + 0x20*(i0))
+#define NVC0_3D_VIEWPORT_TRANSLATE_X__ESIZE			0x00000020
+#define NVC0_3D_VIEWPORT_TRANSLATE_X__LEN			0x00000010
+
+#define NVC0_3D_VIEWPORT_TRANSLATE_Y(i0)		       (0x00000a10 + 0x20*(i0))
+#define NVC0_3D_VIEWPORT_TRANSLATE_Y__ESIZE			0x00000020
+#define NVC0_3D_VIEWPORT_TRANSLATE_Y__LEN			0x00000010
+
+#define NVC0_3D_VIEWPORT_TRANSLATE_Z(i0)		       (0x00000a14 + 0x20*(i0))
+#define NVC0_3D_VIEWPORT_TRANSLATE_Z__ESIZE			0x00000020
+#define NVC0_3D_VIEWPORT_TRANSLATE_Z__LEN			0x00000010
+
+#define NVC0_3D_VIEWPORT_HORIZ(i0)			       (0x00000c00 + 0x10*(i0))
+#define NVC0_3D_VIEWPORT_HORIZ__ESIZE				0x00000010
+#define NVC0_3D_VIEWPORT_HORIZ__LEN				0x00000010
+#define NVC0_3D_VIEWPORT_HORIZ_X__MASK				0x0000ffff
+#define NVC0_3D_VIEWPORT_HORIZ_X__SHIFT				0
+#define NVC0_3D_VIEWPORT_HORIZ_W__MASK				0xffff0000
+#define NVC0_3D_VIEWPORT_HORIZ_W__SHIFT				16
+
+#define NVC0_3D_VIEWPORT_VERT(i0)			       (0x00000c04 + 0x10*(i0))
+#define NVC0_3D_VIEWPORT_VERT__ESIZE				0x00000010
+#define NVC0_3D_VIEWPORT_VERT__LEN				0x00000010
+#define NVC0_3D_VIEWPORT_VERT_Y__MASK				0x0000ffff
+#define NVC0_3D_VIEWPORT_VERT_Y__SHIFT				0
+#define NVC0_3D_VIEWPORT_VERT_H__MASK				0xffff0000
+#define NVC0_3D_VIEWPORT_VERT_H__SHIFT				16
+
+#define NVC0_3D_DEPTH_RANGE_NEAR(i0)			       (0x00000c08 + 0x10*(i0))
+#define NVC0_3D_DEPTH_RANGE_NEAR__ESIZE				0x00000010
+#define NVC0_3D_DEPTH_RANGE_NEAR__LEN				0x00000010
+
+#define NVC0_3D_DEPTH_RANGE_FAR(i0)			       (0x00000c0c + 0x10*(i0))
+#define NVC0_3D_DEPTH_RANGE_FAR__ESIZE				0x00000010
+#define NVC0_3D_DEPTH_RANGE_FAR__LEN				0x00000010
+
+#define NVC0_3D_VIEWPORT_CLIP_HORIZ(i0)			       (0x00000d00 + 0x8*(i0))
+#define NVC0_3D_VIEWPORT_CLIP_HORIZ__ESIZE			0x00000008
+#define NVC0_3D_VIEWPORT_CLIP_HORIZ__LEN			0x00000008
+#define NVC0_3D_VIEWPORT_CLIP_HORIZ_MIN__MASK			0x0000ffff
+#define NVC0_3D_VIEWPORT_CLIP_HORIZ_MIN__SHIFT			0
+#define NVC0_3D_VIEWPORT_CLIP_HORIZ_MAX__MASK			0xffff0000
+#define NVC0_3D_VIEWPORT_CLIP_HORIZ_MAX__SHIFT			16
+
+#define NVC0_3D_VIEWPORT_CLIP_VERT(i0)			       (0x00000d04 + 0x8*(i0))
+#define NVC0_3D_VIEWPORT_CLIP_VERT__ESIZE			0x00000008
+#define NVC0_3D_VIEWPORT_CLIP_VERT__LEN				0x00000008
+#define NVC0_3D_VIEWPORT_CLIP_VERT_MIN__MASK			0x0000ffff
+#define NVC0_3D_VIEWPORT_CLIP_VERT_MIN__SHIFT			0
+#define NVC0_3D_VIEWPORT_CLIP_VERT_MAX__MASK			0xffff0000
+#define NVC0_3D_VIEWPORT_CLIP_VERT_MAX__SHIFT			16
+
+#define NVC0_3D_CLIPID_REGION_HORIZ(i0)			       (0x00000d40 + 0x8*(i0))
+#define NVC0_3D_CLIPID_REGION_HORIZ__ESIZE			0x00000008
+#define NVC0_3D_CLIPID_REGION_HORIZ__LEN			0x00000004
+#define NVC0_3D_CLIPID_REGION_HORIZ_X__MASK			0x0000ffff
+#define NVC0_3D_CLIPID_REGION_HORIZ_X__SHIFT			0
+#define NVC0_3D_CLIPID_REGION_HORIZ_W__MASK			0xffff0000
+#define NVC0_3D_CLIPID_REGION_HORIZ_W__SHIFT			16
+
+#define NVC0_3D_CLIPID_REGION_VERT(i0)			       (0x00000d44 + 0x8*(i0))
+#define NVC0_3D_CLIPID_REGION_VERT__ESIZE			0x00000008
+#define NVC0_3D_CLIPID_REGION_VERT__LEN				0x00000004
+#define NVC0_3D_CLIPID_REGION_VERT_Y__MASK			0x0000ffff
+#define NVC0_3D_CLIPID_REGION_VERT_Y__SHIFT			0
+#define NVC0_3D_CLIPID_REGION_VERT_H__MASK			0xffff0000
+#define NVC0_3D_CLIPID_REGION_VERT_H__SHIFT			16
+
+#define NVC0_3D_VERTEX_BUFFER_FIRST				0x00000d74
+
+#define NVC0_3D_VERTEX_BUFFER_COUNT				0x00000d78
+
+#define NVC0_3D_CLEAR_COLOR(i0)				       (0x00000d80 + 0x4*(i0))
+#define NVC0_3D_CLEAR_COLOR__ESIZE				0x00000004
+#define NVC0_3D_CLEAR_COLOR__LEN				0x00000004
+
+#define NVC0_3D_CLEAR_DEPTH					0x00000d90
+
+#define NVC0_3D_CLEAR_STENCIL					0x00000da0
+
+#define NVC0_3D_POLYGON_SMOOTH_ENABLE				0x00000db4
+
+#define NVC0_3D_POLYGON_OFFSET_POINT_ENABLE			0x00000dc0
+
+#define NVC0_3D_POLYGON_OFFSET_LINE_ENABLE			0x00000dc4
+
+#define NVC0_3D_POLYGON_OFFSET_FILL_ENABLE			0x00000dc8
+
+#define NVC0_3D_PATCH_VERTICES					0x00000dcc
+
+#define NVC0_3D_WINDOW_OFFSET_X					0x00000df8
+
+#define NVC0_3D_WINDOW_OFFSET_Y					0x00000dfc
+
+#define NVC0_3D_SCISSOR_ENABLE(i0)			       (0x00000e00 + 0x10*(i0))
+#define NVC0_3D_SCISSOR_ENABLE__ESIZE				0x00000010
+#define NVC0_3D_SCISSOR_ENABLE__LEN				0x00000010
+
+#define NVC0_3D_SCISSOR_HORIZ(i0)			       (0x00000e04 + 0x10*(i0))
+#define NVC0_3D_SCISSOR_HORIZ__ESIZE				0x00000010
+#define NVC0_3D_SCISSOR_HORIZ__LEN				0x00000010
+#define NVC0_3D_SCISSOR_HORIZ_MIN__MASK				0x0000ffff
+#define NVC0_3D_SCISSOR_HORIZ_MIN__SHIFT			0
+#define NVC0_3D_SCISSOR_HORIZ_MAX__MASK				0xffff0000
+#define NVC0_3D_SCISSOR_HORIZ_MAX__SHIFT			16
+
+#define NVC0_3D_SCISSOR_VERT(i0)			       (0x00000e08 + 0x10*(i0))
+#define NVC0_3D_SCISSOR_VERT__ESIZE				0x00000010
+#define NVC0_3D_SCISSOR_VERT__LEN				0x00000010
+#define NVC0_3D_SCISSOR_VERT_MIN__MASK				0x0000ffff
+#define NVC0_3D_SCISSOR_VERT_MIN__SHIFT				0
+#define NVC0_3D_SCISSOR_VERT_MAX__MASK				0xffff0000
+#define NVC0_3D_SCISSOR_VERT_MAX__SHIFT				16
+
+#define NVC0_3D_STENCIL_BACK_FUNC_REF				0x00000f54
+
+#define NVC0_3D_STENCIL_BACK_MASK				0x00000f58
+
+#define NVC0_3D_STENCIL_BACK_FUNC_MASK				0x00000f5c
+
+#define NVC0_3D_VERTEX_RUNOUT_ADDRESS_HIGH			0x00000f84
+
+#define NVC0_3D_VERTEX_RUNOUT_ADDRESS_LOW			0x00000f88
+
+#define NVC0_3D_DEPTH_BOUNDS(i0)			       (0x00000f9c + 0x4*(i0))
+#define NVC0_3D_DEPTH_BOUNDS__ESIZE				0x00000004
+#define NVC0_3D_DEPTH_BOUNDS__LEN				0x00000002
+
+#define NVC0_3D_MSAA_MASK(i0)				       (0x00000fbc + 0x4*(i0))
+#define NVC0_3D_MSAA_MASK__ESIZE				0x00000004
+#define NVC0_3D_MSAA_MASK__LEN					0x00000004
+
+#define NVC0_3D_CLIPID_ADDRESS_HIGH				0x00000fcc
+
+#define NVC0_3D_CLIPID_ADDRESS_LOW				0x00000fd0
+
+#define NVC0_3D_ZETA_ADDRESS_HIGH				0x00000fe0
+
+#define NVC0_3D_ZETA_ADDRESS_LOW				0x00000fe4
+
+#define NVC0_3D_ZETA_FORMAT					0x00000fe8
+
+#define NVC0_3D_ZETA_TILE_MODE					0x00000fec
+
+#define NVC0_3D_ZETA_LAYER_STRIDE				0x00000ff0
+
+#define NVC0_3D_SCREEN_SCISSOR_HORIZ				0x00000ff4
+#define NVC0_3D_SCREEN_SCISSOR_HORIZ_W__MASK			0xffff0000
+#define NVC0_3D_SCREEN_SCISSOR_HORIZ_W__SHIFT			16
+#define NVC0_3D_SCREEN_SCISSOR_HORIZ_X__MASK			0x0000ffff
+#define NVC0_3D_SCREEN_SCISSOR_HORIZ_X__SHIFT			0
+
+#define NVC0_3D_SCREEN_SCISSOR_VERT				0x00000ff8
+#define NVC0_3D_SCREEN_SCISSOR_VERT_H__MASK			0xffff0000
+#define NVC0_3D_SCREEN_SCISSOR_VERT_H__SHIFT			16
+#define NVC0_3D_SCREEN_SCISSOR_VERT_Y__MASK			0x0000ffff
+#define NVC0_3D_SCREEN_SCISSOR_VERT_Y__SHIFT			0
+
+#define NVC0_3D_VERTEX_ID					0x00001118
+
+#define NVC0_3D_VTX_ATTR_DEFINE					0x0000114c
+#define NVC0_3D_VTX_ATTR_DEFINE_ATTR__MASK			0x000000ff
+#define NVC0_3D_VTX_ATTR_DEFINE_ATTR__SHIFT			0
+#define NVC0_3D_VTX_ATTR_DEFINE_COMP__MASK			0x00000700
+#define NVC0_3D_VTX_ATTR_DEFINE_COMP__SHIFT			8
+#define NVC0_3D_VTX_ATTR_DEFINE_COMP__MIN			0x00000001
+#define NVC0_3D_VTX_ATTR_DEFINE_COMP__MAX			0x00000004
+#define NVC0_3D_VTX_ATTR_DEFINE_SIZE__MASK			0x00007000
+#define NVC0_3D_VTX_ATTR_DEFINE_SIZE__SHIFT			12
+#define NVC0_3D_VTX_ATTR_DEFINE_SIZE_8				0x00001000
+#define NVC0_3D_VTX_ATTR_DEFINE_SIZE_16				0x00002000
+#define NVC0_3D_VTX_ATTR_DEFINE_SIZE_32				0x00004000
+#define NVC0_3D_VTX_ATTR_DEFINE_TYPE__MASK			0x00070000
+#define NVC0_3D_VTX_ATTR_DEFINE_TYPE__SHIFT			16
+#define NVC0_3D_VTX_ATTR_DEFINE_TYPE_SNORM			0x00010000
+#define NVC0_3D_VTX_ATTR_DEFINE_TYPE_UNORM			0x00020000
+#define NVC0_3D_VTX_ATTR_DEFINE_TYPE_SINT			0x00030000
+#define NVC0_3D_VTX_ATTR_DEFINE_TYPE_UINT			0x00040000
+#define NVC0_3D_VTX_ATTR_DEFINE_TYPE_USCALED			0x00050000
+#define NVC0_3D_VTX_ATTR_DEFINE_TYPE_SSCALED			0x00060000
+#define NVC0_3D_VTX_ATTR_DEFINE_TYPE_FLOAT			0x00070000
+
+#define NVC0_3D_VTX_ATTR_DATA(i0)			       (0x00001150 + 0x4*(i0))
+#define NVC0_3D_VTX_ATTR_DATA__ESIZE				0x00000004
+#define NVC0_3D_VTX_ATTR_DATA__LEN				0x00000004
+
+#define NVC0_3D_VERTEX_ATTRIB_FORMAT(i0)		       (0x00001160 + 0x4*(i0))
+#define NVC0_3D_VERTEX_ATTRIB_FORMAT__ESIZE			0x00000004
+#define NVC0_3D_VERTEX_ATTRIB_FORMAT__LEN			0x00000020
+#define NVC0_3D_VERTEX_ATTRIB_FORMAT_BUFFER__MASK		0x0000003f
+#define NVC0_3D_VERTEX_ATTRIB_FORMAT_BUFFER__SHIFT		0
+#define NVC0_3D_VERTEX_ATTRIB_FORMAT_CONST			0x00000040
+#define NVC0_3D_VERTEX_ATTRIB_FORMAT_OFFSET__MASK		0x001fff80
+#define NVC0_3D_VERTEX_ATTRIB_FORMAT_OFFSET__SHIFT		7
+#define NVC0_3D_VERTEX_ATTRIB_FORMAT_SIZE__MASK			0x07e00000
+#define NVC0_3D_VERTEX_ATTRIB_FORMAT_SIZE__SHIFT		21
+#define NVC0_3D_VERTEX_ATTRIB_FORMAT_SIZE_32_32_32_32		0x00200000
+#define NVC0_3D_VERTEX_ATTRIB_FORMAT_SIZE_32_32_32		0x00400000
+#define NVC0_3D_VERTEX_ATTRIB_FORMAT_SIZE_16_16_16_16		0x00600000
+#define NVC0_3D_VERTEX_ATTRIB_FORMAT_SIZE_32_32			0x00800000
+#define NVC0_3D_VERTEX_ATTRIB_FORMAT_SIZE_16_16_16		0x00a00000
+#define NVC0_3D_VERTEX_ATTRIB_FORMAT_SIZE_8_8_8_8		0x01400000
+#define NVC0_3D_VERTEX_ATTRIB_FORMAT_SIZE_16_16			0x01e00000
+#define NVC0_3D_VERTEX_ATTRIB_FORMAT_SIZE_32			0x02400000
+#define NVC0_3D_VERTEX_ATTRIB_FORMAT_SIZE_8_8_8			0x02600000
+#define NVC0_3D_VERTEX_ATTRIB_FORMAT_SIZE_8_8			0x03000000
+#define NVC0_3D_VERTEX_ATTRIB_FORMAT_SIZE_16			0x03600000
+#define NVC0_3D_VERTEX_ATTRIB_FORMAT_SIZE_8			0x03a00000
+#define NVC0_3D_VERTEX_ATTRIB_FORMAT_SIZE_2_10_10_10		0x06000000
+#define NVC0_3D_VERTEX_ATTRIB_FORMAT_TYPE__MASK			0x78000000
+#define NVC0_3D_VERTEX_ATTRIB_FORMAT_TYPE__SHIFT		27
+#define NVC0_3D_VERTEX_ATTRIB_FORMAT_TYPE_SNORM			0x08000000
+#define NVC0_3D_VERTEX_ATTRIB_FORMAT_TYPE_UNORM			0x10000000
+#define NVC0_3D_VERTEX_ATTRIB_FORMAT_TYPE_SINT			0x18000000
+#define NVC0_3D_VERTEX_ATTRIB_FORMAT_TYPE_UINT			0x20000000
+#define NVC0_3D_VERTEX_ATTRIB_FORMAT_TYPE_USCALED		0x28000000
+#define NVC0_3D_VERTEX_ATTRIB_FORMAT_TYPE_SSCALED		0x30000000
+#define NVC0_3D_VERTEX_ATTRIB_FORMAT_TYPE_FLOAT			0x38000000
+#define NVC0_3D_VERTEX_ATTRIB_FORMAT_BGRA			0x80000000
+
+#define NVC0_3D_RT_CONTROL					0x0000121c
+#define NVC0_3D_RT_CONTROL_COUNT__MASK				0x0000000f
+#define NVC0_3D_RT_CONTROL_COUNT__SHIFT				0
+#define NVC0_3D_RT_CONTROL_MAP0__MASK				0x00000070
+#define NVC0_3D_RT_CONTROL_MAP0__SHIFT				4
+#define NVC0_3D_RT_CONTROL_MAP1__MASK				0x00000380
+#define NVC0_3D_RT_CONTROL_MAP1__SHIFT				7
+#define NVC0_3D_RT_CONTROL_MAP2__MASK				0x00001c00
+#define NVC0_3D_RT_CONTROL_MAP2__SHIFT				10
+#define NVC0_3D_RT_CONTROL_MAP3__MASK				0x0000e000
+#define NVC0_3D_RT_CONTROL_MAP3__SHIFT				13
+#define NVC0_3D_RT_CONTROL_MAP4__MASK				0x00070000
+#define NVC0_3D_RT_CONTROL_MAP4__SHIFT				16
+#define NVC0_3D_RT_CONTROL_MAP5__MASK				0x00380000
+#define NVC0_3D_RT_CONTROL_MAP5__SHIFT				19
+#define NVC0_3D_RT_CONTROL_MAP6__MASK				0x01c00000
+#define NVC0_3D_RT_CONTROL_MAP6__SHIFT				22
+#define NVC0_3D_RT_CONTROL_MAP7__MASK				0x0e000000
+#define NVC0_3D_RT_CONTROL_MAP7__SHIFT				25
+
+#define NVC0_3D_ZETA_HORIZ					0x00001228
+
+#define NVC0_3D_ZETA_VERT					0x0000122c
+
+#define NVC0_3D_ZETA_ARRAY_MODE					0x00001230
+#define NVC0_3D_ZETA_ARRAY_MODE_LAYERS__MASK			0x0000ffff
+#define NVC0_3D_ZETA_ARRAY_MODE_LAYERS__SHIFT			0
+#define NVC0_3D_ZETA_ARRAY_MODE_UNK				0x00010000
+
+#define NVC0_3D_LINKED_TSC					0x00001234
+
+#define NVC0_3D_FP_RESULT_COUNT					0x00001298
+
+#define NVC0_3D_DEPTH_TEST_ENABLE				0x000012cc
+
+#define NVC0_3D_D3D_FILL_MODE					0x000012d0
+#define NVC0_3D_D3D_FILL_MODE_POINT				0x00000001
+#define NVC0_3D_D3D_FILL_MODE_WIREFRAME				0x00000002
+#define NVC0_3D_D3D_FILL_MODE_SOLID				0x00000003
+
+#define NVC0_3D_SHADE_MODEL					0x000012d4
+#define NVC0_3D_SHADE_MODEL_FLAT				0x00001d00
+#define NVC0_3D_SHADE_MODEL_SMOOTH				0x00001d01
+
+#define NVC0_3D_BLEND_INDEPENDENT				0x000012e4
+
+#define NVC0_3D_DEPTH_WRITE_ENABLE				0x000012e8
+
+#define NVC0_3D_ALPHA_TEST_ENABLE				0x000012ec
+
+#define NVC0_3D_VB_ELEMENT_U8_SETUP				0x00001300
+#define NVC0_3D_VB_ELEMENT_U8_SETUP_OFFSET__MASK		0xc0000000
+#define NVC0_3D_VB_ELEMENT_U8_SETUP_OFFSET__SHIFT		30
+#define NVC0_3D_VB_ELEMENT_U8_SETUP_COUNT__MASK			0x3fffffff
+#define NVC0_3D_VB_ELEMENT_U8_SETUP_COUNT__SHIFT		0
+
+#define NVC0_3D_VB_ELEMENT_U8					0x00001304
+#define NVC0_3D_VB_ELEMENT_U8_I0__MASK				0x000000ff
+#define NVC0_3D_VB_ELEMENT_U8_I0__SHIFT				0
+#define NVC0_3D_VB_ELEMENT_U8_I1__MASK				0x0000ff00
+#define NVC0_3D_VB_ELEMENT_U8_I1__SHIFT				8
+#define NVC0_3D_VB_ELEMENT_U8_I2__MASK				0x00ff0000
+#define NVC0_3D_VB_ELEMENT_U8_I2__SHIFT				16
+#define NVC0_3D_VB_ELEMENT_U8_I3__MASK				0xff000000
+#define NVC0_3D_VB_ELEMENT_U8_I3__SHIFT				24
+
+#define NVC0_3D_D3D_CULL_MODE					0x00001308
+#define NVC0_3D_D3D_CULL_MODE_NONE				0x00000001
+#define NVC0_3D_D3D_CULL_MODE_FRONT				0x00000002
+#define NVC0_3D_D3D_CULL_MODE_BACK				0x00000003
+
+#define NVC0_3D_DEPTH_TEST_FUNC					0x0000130c
+#define NVC0_3D_DEPTH_TEST_FUNC_NEVER				0x00000200
+#define NVC0_3D_DEPTH_TEST_FUNC_LESS				0x00000201
+#define NVC0_3D_DEPTH_TEST_FUNC_EQUAL				0x00000202
+#define NVC0_3D_DEPTH_TEST_FUNC_LEQUAL				0x00000203
+#define NVC0_3D_DEPTH_TEST_FUNC_GREATER				0x00000204
+#define NVC0_3D_DEPTH_TEST_FUNC_NOTEQUAL			0x00000205
+#define NVC0_3D_DEPTH_TEST_FUNC_GEQUAL				0x00000206
+#define NVC0_3D_DEPTH_TEST_FUNC_ALWAYS				0x00000207
+
+#define NVC0_3D_ALPHA_TEST_REF					0x00001310
+
+#define NVC0_3D_ALPHA_TEST_FUNC					0x00001314
+#define NVC0_3D_ALPHA_TEST_FUNC_NEVER				0x00000200
+#define NVC0_3D_ALPHA_TEST_FUNC_LESS				0x00000201
+#define NVC0_3D_ALPHA_TEST_FUNC_EQUAL				0x00000202
+#define NVC0_3D_ALPHA_TEST_FUNC_LEQUAL				0x00000203
+#define NVC0_3D_ALPHA_TEST_FUNC_GREATER				0x00000204
+#define NVC0_3D_ALPHA_TEST_FUNC_NOTEQUAL			0x00000205
+#define NVC0_3D_ALPHA_TEST_FUNC_GEQUAL				0x00000206
+#define NVC0_3D_ALPHA_TEST_FUNC_ALWAYS				0x00000207
+
+#define NVC0_3D_BLEND_COLOR(i0)				       (0x0000131c + 0x4*(i0))
+#define NVC0_3D_BLEND_COLOR__ESIZE				0x00000004
+#define NVC0_3D_BLEND_COLOR__LEN				0x00000004
+
+#define NVC0_3D_TSC_FLUSH					0x00001330
+#define NVC0_3D_TSC_FLUSH_UNK0					0x00000001
+#define NVC0_3D_TSC_FLUSH_UNK1__MASK				0x03fffff0
+#define NVC0_3D_TSC_FLUSH_UNK1__SHIFT				4
+
+#define NVC0_3D_TIC_FLUSH					0x00001334
+#define NVC0_3D_TIC_FLUSH_UNK0					0x00000001
+#define NVC0_3D_TIC_FLUSH_UNK1__MASK				0x03fffff0
+#define NVC0_3D_TIC_FLUSH_UNK1__SHIFT				4
+
+#define NVC0_3D_TEX_CACHE_CTL					0x00001338
+#define NVC0_3D_TEX_CACHE_CTL_UNK1__MASK			0x00000030
+#define NVC0_3D_TEX_CACHE_CTL_UNK1__SHIFT			4
+
+#define NVC0_3D_BLEND_EQUATION_RGB				0x00001340
+#define NVC0_3D_BLEND_EQUATION_RGB_FUNC_ADD			0x00008006
+#define NVC0_3D_BLEND_EQUATION_RGB_MIN				0x00008007
+#define NVC0_3D_BLEND_EQUATION_RGB_MAX				0x00008008
+#define NVC0_3D_BLEND_EQUATION_RGB_FUNC_SUBTRACT		0x0000800a
+#define NVC0_3D_BLEND_EQUATION_RGB_FUNC_REVERSE_SUBTRACT	0x0000800b
+
+#define NVC0_3D_BLEND_FUNC_SRC_RGB				0x00001344
+
+#define NVC0_3D_BLEND_FUNC_DST_RGB				0x00001348
+
+#define NVC0_3D_BLEND_EQUATION_ALPHA				0x0000134c
+#define NVC0_3D_BLEND_EQUATION_ALPHA_FUNC_ADD			0x00008006
+#define NVC0_3D_BLEND_EQUATION_ALPHA_MIN			0x00008007
+#define NVC0_3D_BLEND_EQUATION_ALPHA_MAX			0x00008008
+#define NVC0_3D_BLEND_EQUATION_ALPHA_FUNC_SUBTRACT		0x0000800a
+#define NVC0_3D_BLEND_EQUATION_ALPHA_FUNC_REVERSE_SUBTRACT	0x0000800b
+
+#define NVC0_3D_BLEND_FUNC_SRC_ALPHA				0x00001350
+
+#define NVC0_3D_BLEND_FUNC_DST_ALPHA				0x00001358
+
+#define NVC0_3D_BLEND_ENABLE(i0)			       (0x00001360 + 0x4*(i0))
+#define NVC0_3D_BLEND_ENABLE__ESIZE				0x00000004
+#define NVC0_3D_BLEND_ENABLE__LEN				0x00000008
+
+#define NVC0_3D_STENCIL_FRONT_ENABLE				0x00001380
+
+#define NVC0_3D_STENCIL_FRONT_OP_FAIL				0x00001384
+#define NVC0_3D_STENCIL_FRONT_OP_FAIL_ZERO			0x00000000
+#define NVC0_3D_STENCIL_FRONT_OP_FAIL_INVERT			0x0000150a
+#define NVC0_3D_STENCIL_FRONT_OP_FAIL_KEEP			0x00001e00
+#define NVC0_3D_STENCIL_FRONT_OP_FAIL_REPLACE			0x00001e01
+#define NVC0_3D_STENCIL_FRONT_OP_FAIL_INCR			0x00001e02
+#define NVC0_3D_STENCIL_FRONT_OP_FAIL_DECR			0x00001e03
+#define NVC0_3D_STENCIL_FRONT_OP_FAIL_INCR_WRAP			0x00008507
+#define NVC0_3D_STENCIL_FRONT_OP_FAIL_DECR_WRAP			0x00008508
+
+#define NVC0_3D_STENCIL_FRONT_OP_ZFAIL				0x00001388
+#define NVC0_3D_STENCIL_FRONT_OP_ZFAIL_ZERO			0x00000000
+#define NVC0_3D_STENCIL_FRONT_OP_ZFAIL_INVERT			0x0000150a
+#define NVC0_3D_STENCIL_FRONT_OP_ZFAIL_KEEP			0x00001e00
+#define NVC0_3D_STENCIL_FRONT_OP_ZFAIL_REPLACE			0x00001e01
+#define NVC0_3D_STENCIL_FRONT_OP_ZFAIL_INCR			0x00001e02
+#define NVC0_3D_STENCIL_FRONT_OP_ZFAIL_DECR			0x00001e03
+#define NVC0_3D_STENCIL_FRONT_OP_ZFAIL_INCR_WRAP		0x00008507
+#define NVC0_3D_STENCIL_FRONT_OP_ZFAIL_DECR_WRAP		0x00008508
+
+#define NVC0_3D_STENCIL_FRONT_OP_ZPASS				0x0000138c
+#define NVC0_3D_STENCIL_FRONT_OP_ZPASS_ZERO			0x00000000
+#define NVC0_3D_STENCIL_FRONT_OP_ZPASS_INVERT			0x0000150a
+#define NVC0_3D_STENCIL_FRONT_OP_ZPASS_KEEP			0x00001e00
+#define NVC0_3D_STENCIL_FRONT_OP_ZPASS_REPLACE			0x00001e01
+#define NVC0_3D_STENCIL_FRONT_OP_ZPASS_INCR			0x00001e02
+#define NVC0_3D_STENCIL_FRONT_OP_ZPASS_DECR			0x00001e03
+#define NVC0_3D_STENCIL_FRONT_OP_ZPASS_INCR_WRAP		0x00008507
+#define NVC0_3D_STENCIL_FRONT_OP_ZPASS_DECR_WRAP		0x00008508
+
+#define NVC0_3D_STENCIL_FRONT_FUNC_FUNC				0x00001390
+#define NVC0_3D_STENCIL_FRONT_FUNC_FUNC_NEVER			0x00000200
+#define NVC0_3D_STENCIL_FRONT_FUNC_FUNC_LESS			0x00000201
+#define NVC0_3D_STENCIL_FRONT_FUNC_FUNC_EQUAL			0x00000202
+#define NVC0_3D_STENCIL_FRONT_FUNC_FUNC_LEQUAL			0x00000203
+#define NVC0_3D_STENCIL_FRONT_FUNC_FUNC_GREATER			0x00000204
+#define NVC0_3D_STENCIL_FRONT_FUNC_FUNC_NOTEQUAL		0x00000205
+#define NVC0_3D_STENCIL_FRONT_FUNC_FUNC_GEQUAL			0x00000206
+#define NVC0_3D_STENCIL_FRONT_FUNC_FUNC_ALWAYS			0x00000207
+
+#define NVC0_3D_STENCIL_FRONT_FUNC_REF				0x00001394
+
+#define NVC0_3D_STENCIL_FRONT_MASK				0x00001398
+
+#define NVC0_3D_STENCIL_FRONT_FUNC_MASK				0x0000139c
+
+#define NVC0_3D_FRAG_COLOR_CLAMP_EN				0x000013a8
+#define NVC0_3D_FRAG_COLOR_CLAMP_EN_0				0x00000001
+#define NVC0_3D_FRAG_COLOR_CLAMP_EN_1				0x00000010
+#define NVC0_3D_FRAG_COLOR_CLAMP_EN_2				0x00000100
+#define NVC0_3D_FRAG_COLOR_CLAMP_EN_3				0x00001000
+#define NVC0_3D_FRAG_COLOR_CLAMP_EN_4				0x00010000
+#define NVC0_3D_FRAG_COLOR_CLAMP_EN_5				0x00100000
+#define NVC0_3D_FRAG_COLOR_CLAMP_EN_6				0x01000000
+#define NVC0_3D_FRAG_COLOR_CLAMP_EN_7				0x10000000
+
+#define NVC0_3D_Y_ORIGIN_BOTTOM					0x000013ac
+
+#define NVC0_3D_LINE_WIDTH					0x000013b0
+
+#define NVC0_3D_GP_VERTEX_OUTPUT_COUNT				0x00001420
+#define NVC0_3D_GP_VERTEX_OUTPUT_COUNT__MIN			0x00000001
+#define NVC0_3D_GP_VERTEX_OUTPUT_COUNT__MAX			0x00000400
+
+#define NVC0_3D_FENCE_UNK					0x0000142c
+
+#define NVC0_3D_VB_ELEMENT_BASE					0x00001434
+
+#define NVC0_3D_VB_INSTANCE_BASE				0x00001438
+
+#define NVC0_3D_CODE_CB_FLUSH					0x00001440
+
+#define NVC0_3D_CLIPID_HEIGHT					0x00001504
+#define NVC0_3D_CLIPID_HEIGHT__MAX				0x00002000
+
+#define NVC0_3D_VP_CLIP_DISTANCE_ENABLE				0x00001510
+#define NVC0_3D_VP_CLIP_DISTANCE_ENABLE_0			0x00000001
+#define NVC0_3D_VP_CLIP_DISTANCE_ENABLE_1			0x00000002
+#define NVC0_3D_VP_CLIP_DISTANCE_ENABLE_2			0x00000004
+#define NVC0_3D_VP_CLIP_DISTANCE_ENABLE_3			0x00000008
+#define NVC0_3D_VP_CLIP_DISTANCE_ENABLE_4			0x00000010
+#define NVC0_3D_VP_CLIP_DISTANCE_ENABLE_5			0x00000020
+#define NVC0_3D_VP_CLIP_DISTANCE_ENABLE_6			0x00000040
+#define NVC0_3D_VP_CLIP_DISTANCE_ENABLE_7			0x00000080
+
+#define NVC0_3D_SAMPLECNT_ENABLE				0x00001514
+
+#define NVC0_3D_POINT_SIZE					0x00001518
+
+#define NVC0_3D_POINT_SPRITE_ENABLE				0x00001520
+
+#define NVC0_3D_SAMPLECNT_RESET					0x00001530
+
+#define NVC0_3D_MULTISAMPLE_ZETA_ENABLE				0x00001534
+
+#define NVC0_3D_ZETA_ENABLE					0x00001538
+
+#define NVC0_3D_MULTISAMPLE_CTRL				0x0000153c
+#define NVC0_3D_MULTISAMPLE_CTRL_ALPHA_TO_COVERAGE		0x00000001
+#define NVC0_3D_MULTISAMPLE_CTRL_ALPHA_TO_ONE			0x00000010
+
+#define NVC0_3D_COND_ADDRESS_HIGH				0x00001550
+
+#define NVC0_3D_COND_ADDRESS_LOW				0x00001554
+
+#define NVC0_3D_COND_MODE					0x00001558
+#define NVC0_3D_COND_MODE_NEVER					0x00000000
+#define NVC0_3D_COND_MODE_ALWAYS				0x00000001
+#define NVC0_3D_COND_MODE_RES_NON_ZERO				0x00000002
+#define NVC0_3D_COND_MODE_EQUAL					0x00000003
+#define NVC0_3D_COND_MODE_NOT_EQUAL				0x00000004
+
+#define NVC0_3D_TSC_ADDRESS_HIGH				0x0000155c
+
+#define NVC0_3D_TSC_ADDRESS_LOW					0x00001560
+#define NVC0_3D_TSC_ADDRESS_LOW__ALIGN				0x00000020
+
+#define NVC0_3D_TSC_LIMIT					0x00001564
+#define NVC0_3D_TSC_LIMIT__MAX					0x00001fff
+
+#define NVC0_3D_POLYGON_OFFSET_FACTOR				0x0000156c
+
+#define NVC0_3D_LINE_SMOOTH_ENABLE				0x00001570
+
+#define NVC0_3D_TIC_ADDRESS_HIGH				0x00001574
+
+#define NVC0_3D_TIC_ADDRESS_LOW					0x00001578
+
+#define NVC0_3D_TIC_LIMIT					0x0000157c
+
+#define NVC0_3D_STENCIL_TWO_SIDE_ENABLE				0x00001594
+
+#define NVC0_3D_STENCIL_BACK_OP_FAIL				0x00001598
+#define NVC0_3D_STENCIL_BACK_OP_FAIL_ZERO			0x00000000
+#define NVC0_3D_STENCIL_BACK_OP_FAIL_INVERT			0x0000150a
+#define NVC0_3D_STENCIL_BACK_OP_FAIL_KEEP			0x00001e00
+#define NVC0_3D_STENCIL_BACK_OP_FAIL_REPLACE			0x00001e01
+#define NVC0_3D_STENCIL_BACK_OP_FAIL_INCR			0x00001e02
+#define NVC0_3D_STENCIL_BACK_OP_FAIL_DECR			0x00001e03
+#define NVC0_3D_STENCIL_BACK_OP_FAIL_INCR_WRAP			0x00008507
+#define NVC0_3D_STENCIL_BACK_OP_FAIL_DECR_WRAP			0x00008508
+
+#define NVC0_3D_STENCIL_BACK_OP_ZFAIL				0x0000159c
+#define NVC0_3D_STENCIL_BACK_OP_ZFAIL_ZERO			0x00000000
+#define NVC0_3D_STENCIL_BACK_OP_ZFAIL_INVERT			0x0000150a
+#define NVC0_3D_STENCIL_BACK_OP_ZFAIL_KEEP			0x00001e00
+#define NVC0_3D_STENCIL_BACK_OP_ZFAIL_REPLACE			0x00001e01
+#define NVC0_3D_STENCIL_BACK_OP_ZFAIL_INCR			0x00001e02
+#define NVC0_3D_STENCIL_BACK_OP_ZFAIL_DECR			0x00001e03
+#define NVC0_3D_STENCIL_BACK_OP_ZFAIL_INCR_WRAP			0x00008507
+#define NVC0_3D_STENCIL_BACK_OP_ZFAIL_DECR_WRAP			0x00008508
+
+#define NVC0_3D_STENCIL_BACK_OP_ZPASS				0x000015a0
+#define NVC0_3D_STENCIL_BACK_OP_ZPASS_ZERO			0x00000000
+#define NVC0_3D_STENCIL_BACK_OP_ZPASS_INVERT			0x0000150a
+#define NVC0_3D_STENCIL_BACK_OP_ZPASS_KEEP			0x00001e00
+#define NVC0_3D_STENCIL_BACK_OP_ZPASS_REPLACE			0x00001e01
+#define NVC0_3D_STENCIL_BACK_OP_ZPASS_INCR			0x00001e02
+#define NVC0_3D_STENCIL_BACK_OP_ZPASS_DECR			0x00001e03
+#define NVC0_3D_STENCIL_BACK_OP_ZPASS_INCR_WRAP			0x00008507
+#define NVC0_3D_STENCIL_BACK_OP_ZPASS_DECR_WRAP			0x00008508
+
+#define NVC0_3D_STENCIL_BACK_FUNC_FUNC				0x000015a4
+#define NVC0_3D_STENCIL_BACK_FUNC_FUNC_NEVER			0x00000200
+#define NVC0_3D_STENCIL_BACK_FUNC_FUNC_LESS			0x00000201
+#define NVC0_3D_STENCIL_BACK_FUNC_FUNC_EQUAL			0x00000202
+#define NVC0_3D_STENCIL_BACK_FUNC_FUNC_LEQUAL			0x00000203
+#define NVC0_3D_STENCIL_BACK_FUNC_FUNC_GREATER			0x00000204
+#define NVC0_3D_STENCIL_BACK_FUNC_FUNC_NOTEQUAL			0x00000205
+#define NVC0_3D_STENCIL_BACK_FUNC_FUNC_GEQUAL			0x00000206
+#define NVC0_3D_STENCIL_BACK_FUNC_FUNC_ALWAYS			0x00000207
+
+#define NVC0_3D_MULTISAMPLE_COLOR_ENABLE			0x000015b4
+
+#define NVC0_3D_FRAMEBUFFER_SRGB				0x000015b8
+
+#define NVC0_3D_POLYGON_OFFSET_UNITS				0x000015bc
+
+#define NVC0_3D_GP_BUILTIN_RESULT_EN				0x000015cc
+#define NVC0_3D_GP_BUILTIN_RESULT_EN_LAYER			0x00010000
+
+#define NVC0_3D_MULTISAMPLE_MODE				0x000015d0
+#define NVC0_3D_MULTISAMPLE_MODE_1X				0x00000000
+#define NVC0_3D_MULTISAMPLE_MODE_2XMS				0x00000001
+#define NVC0_3D_MULTISAMPLE_MODE_4XMS				0x00000002
+#define NVC0_3D_MULTISAMPLE_MODE_8XMS				0x00000003
+#define NVC0_3D_MULTISAMPLE_MODE_4XMS_4XCS			0x00000008
+#define NVC0_3D_MULTISAMPLE_MODE_4XMS_12XCS			0x00000009
+#define NVC0_3D_MULTISAMPLE_MODE_8XMS_8XCS			0x0000000a
+
+#define NVC0_3D_VERTEX_BEGIN_D3D				0x000015d4
+#define NVC0_3D_VERTEX_BEGIN_D3D_PRIMITIVE__MASK		0x0fffffff
+#define NVC0_3D_VERTEX_BEGIN_D3D_PRIMITIVE__SHIFT		0
+#define NVC0_3D_VERTEX_BEGIN_D3D_PRIMITIVE_POINTS		0x00000001
+#define NVC0_3D_VERTEX_BEGIN_D3D_PRIMITIVE_LINES		0x00000002
+#define NVC0_3D_VERTEX_BEGIN_D3D_PRIMITIVE_LINE_STRIP		0x00000003
+#define NVC0_3D_VERTEX_BEGIN_D3D_PRIMITIVE_TRIANGLES		0x00000004
+#define NVC0_3D_VERTEX_BEGIN_D3D_PRIMITIVE_TRIANGLE_STRIP	0x00000005
+#define NVC0_3D_VERTEX_BEGIN_D3D_PRIMITIVE_LINES_ADJACENCY	0x0000000a
+#define NVC0_3D_VERTEX_BEGIN_D3D_PRIMITIVE_LINE_STRIP_ADJACENCY	0x0000000b
+#define NVC0_3D_VERTEX_BEGIN_D3D_PRIMITIVE_TRIANGLES_ADJACENCY	0x0000000c
+#define NVC0_3D_VERTEX_BEGIN_D3D_PRIMITIVE_TRIANGLE_STRIP_ADJACENCY	0x0000000d
+#define NVC0_3D_VERTEX_BEGIN_D3D_INSTANCE_NEXT			0x10000000
+
+#define NVC0_3D_VERTEX_END_D3D					0x000015d8
+#define NVC0_3D_VERTEX_END_D3D_UNK0				0x00000001
+#define NVC0_3D_VERTEX_END_D3D_UNK1				0x00000002
+
+#define NVC0_3D_EDGEFLAG_ENABLE					0x000015e4
+
+#define NVC0_3D_VB_ELEMENT_U32					0x000015e8
+
+#define NVC0_3D_VB_ELEMENT_U16_SETUP				0x000015ec
+#define NVC0_3D_VB_ELEMENT_U16_SETUP_OFFSET__MASK		0xc0000000
+#define NVC0_3D_VB_ELEMENT_U16_SETUP_OFFSET__SHIFT		30
+#define NVC0_3D_VB_ELEMENT_U16_SETUP_COUNT__MASK		0x3fffffff
+#define NVC0_3D_VB_ELEMENT_U16_SETUP_COUNT__SHIFT		0
+
+#define NVC0_3D_VB_ELEMENT_U16					0x000015f0
+#define NVC0_3D_VB_ELEMENT_U16_I0__MASK				0x0000ffff
+#define NVC0_3D_VB_ELEMENT_U16_I0__SHIFT			0
+#define NVC0_3D_VB_ELEMENT_U16_I1__MASK				0xffff0000
+#define NVC0_3D_VB_ELEMENT_U16_I1__SHIFT			16
+
+#define NVC0_3D_VERTEX_BASE_HIGH				0x000015f4
+
+#define NVC0_3D_VERTEX_BASE_LOW					0x000015f8
+
+#define NVC0_3D_POINT_COORD_REPLACE				0x00001604
+#define NVC0_3D_POINT_COORD_REPLACE_BITS__MASK			0x00001fff
+#define NVC0_3D_POINT_COORD_REPLACE_BITS__SHIFT			0
+
+#define NVC0_3D_CODE_ADDRESS_HIGH				0x00001608
+
+#define NVC0_3D_CODE_ADDRESS_LOW				0x0000160c
+
+#define NVC0_3D_VERTEX_END_GL					0x00001614
+#define NVC0_3D_VERTEX_END_GL_UNK0				0x00000001
+#define NVC0_3D_VERTEX_END_GL_UNK1				0x00000002
+
+#define NVC0_3D_VERTEX_BEGIN_GL					0x00001618
+#define NVC0_3D_VERTEX_BEGIN_GL_PRIMITIVE__MASK			0x0fffffff
+#define NVC0_3D_VERTEX_BEGIN_GL_PRIMITIVE__SHIFT		0
+#define NVC0_3D_VERTEX_BEGIN_GL_PRIMITIVE_POINTS		0x00000000
+#define NVC0_3D_VERTEX_BEGIN_GL_PRIMITIVE_LINES			0x00000001
+#define NVC0_3D_VERTEX_BEGIN_GL_PRIMITIVE_LINE_LOOP		0x00000002
+#define NVC0_3D_VERTEX_BEGIN_GL_PRIMITIVE_LINE_STRIP		0x00000003
+#define NVC0_3D_VERTEX_BEGIN_GL_PRIMITIVE_TRIANGLES		0x00000004
+#define NVC0_3D_VERTEX_BEGIN_GL_PRIMITIVE_TRIANGLE_STRIP	0x00000005
+#define NVC0_3D_VERTEX_BEGIN_GL_PRIMITIVE_TRIANGLE_FAN		0x00000006
+#define NVC0_3D_VERTEX_BEGIN_GL_PRIMITIVE_QUADS			0x00000007
+#define NVC0_3D_VERTEX_BEGIN_GL_PRIMITIVE_QUAD_STRIP		0x00000008
+#define NVC0_3D_VERTEX_BEGIN_GL_PRIMITIVE_POLYGON		0x00000009
+#define NVC0_3D_VERTEX_BEGIN_GL_PRIMITIVE_LINES_ADJACENCY	0x0000000a
+#define NVC0_3D_VERTEX_BEGIN_GL_PRIMITIVE_LINE_STRIP_ADJACENCY	0x0000000b
+#define NVC0_3D_VERTEX_BEGIN_GL_PRIMITIVE_TRIANGLES_ADJACENCY	0x0000000c
+#define NVC0_3D_VERTEX_BEGIN_GL_PRIMITIVE_TRIANGLE_STRIP_ADJACENCY	0x0000000d
+#define NVC0_3D_VERTEX_BEGIN_GL_PRIMITIVE_PATCHES		0x0000000e
+#define NVC0_3D_VERTEX_BEGIN_GL_INSTANCE_NEXT			0x10000000
+
+#define NVC0_3D_VERTEX_DATA					0x00001640
+
+#define NVC0_3D_PRIM_RESTART_ENABLE				0x00001644
+
+#define NVC0_3D_PRIM_RESTART_INDEX				0x00001648
+
+#define NVC0_3D_VP_GP_BUILTIN_ATTR_EN				0x0000164c
+#define NVC0_3D_VP_GP_BUILTIN_ATTR_EN_VERTEX_ID			0x00000001
+#define NVC0_3D_VP_GP_BUILTIN_ATTR_EN_INSTANCE_ID		0x00000010
+#define NVC0_3D_VP_GP_BUILTIN_ATTR_EN_PRIMITIVE_ID		0x00000100
+#define NVC0_3D_VP_GP_BUILTIN_ATTR_EN_UNK12			0x00001000
+
+#define NVC0_3D_POINT_SMOOTH_ENABLE				0x00001658
+
+#define NVC0_3D_POINT_SPRITE_CTRL				0x00001660
+
+#define NVC0_3D_TEX_MISC					0x00001664
+#define NVC0_3D_TEX_MISC_SEAMLESS_CUBE_MAP			0x00000004
+
+#define NVC0_3D_LINE_STIPPLE_ENABLE				0x0000166c
+
+#define NVC0_3D_LINE_STIPPLE_PATTERN				0x00001680
+
+#define NVC0_3D_PROVOKING_VERTEX_LAST				0x00001684
+
+#define NVC0_3D_VERTEX_TWO_SIDE_ENABLE				0x00001688
+
+#define NVC0_3D_POLYGON_STIPPLE_ENABLE				0x0000168c
+
+#define NVC0_3D_POLYGON_STIPPLE_PATTERN(i0)		       (0x00001700 + 0x4*(i0))
+#define NVC0_3D_POLYGON_STIPPLE_PATTERN__ESIZE			0x00000004
+#define NVC0_3D_POLYGON_STIPPLE_PATTERN__LEN			0x00000020
+
+#define NVC0_3D_STRMOUT_UNK1780(i0)			       (0x00001780 + 0x4*(i0))
+#define NVC0_3D_STRMOUT_UNK1780__ESIZE				0x00000004
+#define NVC0_3D_STRMOUT_UNK1780__LEN				0x00000004
+
+#define NVC0_3D_UNK17BC_ADDRESS_HIGH				0x000017bc
+
+#define NVC0_3D_UNK17BC_ADDRESS_LOW				0x000017c0
+
+#define NVC0_3D_UNK17BC_LIMIT					0x000017c4
+
+#define NVC0_3D_INDEX_ARRAY_START_HIGH				0x000017c8
+
+#define NVC0_3D_INDEX_ARRAY_START_LOW				0x000017cc
+
+#define NVC0_3D_INDEX_ARRAY_LIMIT_HIGH				0x000017d0
+
+#define NVC0_3D_INDEX_ARRAY_LIMIT_LOW				0x000017d4
+
+#define NVC0_3D_INDEX_LOG2_SIZE					0x000017d8
+
+#define NVC0_3D_INDEX_BATCH_FIRST				0x000017dc
+
+#define NVC0_3D_INDEX_BATCH_COUNT				0x000017e0
+
+#define NVC0_3D_VERTEX_ARRAY_PER_INSTANCE(i0)		       (0x00001880 + 0x4*(i0))
+#define NVC0_3D_VERTEX_ARRAY_PER_INSTANCE__ESIZE		0x00000004
+#define NVC0_3D_VERTEX_ARRAY_PER_INSTANCE__LEN			0x00000020
+
+#define NVC0_3D_VP_POINT_SIZE_EN				0x00001910
+
+#define NVC0_3D_CULL_FACE_ENABLE				0x00001918
+
+#define NVC0_3D_FRONT_FACE					0x0000191c
+#define NVC0_3D_FRONT_FACE_CW					0x00000900
+#define NVC0_3D_FRONT_FACE_CCW					0x00000901
+
+#define NVC0_3D_CULL_FACE					0x00001920
+#define NVC0_3D_CULL_FACE_FRONT					0x00000404
+#define NVC0_3D_CULL_FACE_BACK					0x00000405
+#define NVC0_3D_CULL_FACE_FRONT_AND_BACK			0x00000408
+
+#define NVC0_3D_VIEWPORT_TRANSFORM_EN				0x0000192c
+
+#define NVC0_3D_VIEW_VOLUME_CLIP_CTRL				0x0000193c
+#define NVC0_3D_VIEW_VOLUME_CLIP_CTRL_UNK0			0x00000001
+#define NVC0_3D_VIEW_VOLUME_CLIP_CTRL_UNK1			0x00000002
+#define NVC0_3D_VIEW_VOLUME_CLIP_CTRL_UNK2			0x00000004
+#define NVC0_3D_VIEW_VOLUME_CLIP_CTRL_UNK3			0x00000008
+#define NVC0_3D_VIEW_VOLUME_CLIP_CTRL_UNK4			0x00000010
+#define NVC0_3D_VIEW_VOLUME_CLIP_CTRL_UNK7			0x00000080
+#define NVC0_3D_VIEW_VOLUME_CLIP_CTRL_UNK10			0x00000400
+#define NVC0_3D_VIEW_VOLUME_CLIP_CTRL_UNK11			0x00000800
+#define NVC0_3D_VIEW_VOLUME_CLIP_CTRL_UNK12			0x00001000
+#define NVC0_3D_VIEW_VOLUME_CLIP_CTRL_UNK13			0x00002000
+
+#define NVC0_3D_VIEWPORT_CLIP_RECTS_EN				0x0000194c
+
+#define NVC0_3D_VIEWPORT_CLIP_MODE				0x00001950
+#define NVC0_3D_VIEWPORT_CLIP_MODE_INSIDE_ANY			0x00000000
+#define NVC0_3D_VIEWPORT_CLIP_MODE_OUTSIDE_ALL			0x00000001
+#define NVC0_3D_VIEWPORT_CLIP_MODE_NEVER			0x00000002
+
+#define NVC0_3D_FP_ZORDER_CTRL					0x0000196c
+#define NVC0_3D_FP_ZORDER_CTRL_0				0x00000001
+#define NVC0_3D_FP_ZORDER_CTRL_1				0x00000010
+
+#define NVC0_3D_CLIPID_ENABLE					0x0000197c
+
+#define NVC0_3D_CLIPID_WIDTH					0x00001980
+#define NVC0_3D_CLIPID_WIDTH__MAX				0x00002000
+#define NVC0_3D_CLIPID_WIDTH__ALIGN				0x00000040
+
+#define NVC0_3D_CLIPID_ID					0x00001984
+
+#define NVC0_3D_FP_CONTROL					0x000019a8
+#define NVC0_3D_FP_CONTROL_MULTIPLE_RESULTS			0x00000001
+#define NVC0_3D_FP_CONTROL_EXPORTS_Z				0x00000100
+#define NVC0_3D_FP_CONTROL_USES_KIL				0x00100000
+
+#define NVC0_3D_DEPTH_BOUNDS_EN					0x000019bc
+
+#define NVC0_3D_LOGIC_OP_ENABLE					0x000019c4
+
+#define NVC0_3D_LOGIC_OP					0x000019c8
+#define NVC0_3D_LOGIC_OP_CLEAR					0x00001500
+#define NVC0_3D_LOGIC_OP_AND					0x00001501
+#define NVC0_3D_LOGIC_OP_AND_REVERSE				0x00001502
+#define NVC0_3D_LOGIC_OP_COPY					0x00001503
+#define NVC0_3D_LOGIC_OP_AND_INVERTED				0x00001504
+#define NVC0_3D_LOGIC_OP_NOOP					0x00001505
+#define NVC0_3D_LOGIC_OP_XOR					0x00001506
+#define NVC0_3D_LOGIC_OP_OR					0x00001507
+#define NVC0_3D_LOGIC_OP_NOR					0x00001508
+#define NVC0_3D_LOGIC_OP_EQUIV					0x00001509
+#define NVC0_3D_LOGIC_OP_INVERT					0x0000150a
+#define NVC0_3D_LOGIC_OP_OR_REVERSE				0x0000150b
+#define NVC0_3D_LOGIC_OP_COPY_INVERTED				0x0000150c
+#define NVC0_3D_LOGIC_OP_OR_INVERTED				0x0000150d
+#define NVC0_3D_LOGIC_OP_NAND					0x0000150e
+#define NVC0_3D_LOGIC_OP_SET					0x0000150f
+
+#define NVC0_3D_CLEAR_BUFFERS					0x000019d0
+#define NVC0_3D_CLEAR_BUFFERS_Z					0x00000001
+#define NVC0_3D_CLEAR_BUFFERS_S					0x00000002
+#define NVC0_3D_CLEAR_BUFFERS_R					0x00000004
+#define NVC0_3D_CLEAR_BUFFERS_G					0x00000008
+#define NVC0_3D_CLEAR_BUFFERS_B					0x00000010
+#define NVC0_3D_CLEAR_BUFFERS_A					0x00000020
+#define NVC0_3D_CLEAR_BUFFERS_RT__MASK				0x000003c0
+#define NVC0_3D_CLEAR_BUFFERS_RT__SHIFT				6
+#define NVC0_3D_CLEAR_BUFFERS_LAYER__MASK			0x001ffc00
+#define NVC0_3D_CLEAR_BUFFERS_LAYER__SHIFT			10
+
+#define NVC0_3D_COLOR_MASK(i0)				       (0x00001a00 + 0x4*(i0))
+#define NVC0_3D_COLOR_MASK__ESIZE				0x00000004
+#define NVC0_3D_COLOR_MASK__LEN					0x00000008
+#define NVC0_3D_COLOR_MASK_R					0x0000000f
+#define NVC0_3D_COLOR_MASK_G					0x000000f0
+#define NVC0_3D_COLOR_MASK_B					0x00000f00
+#define NVC0_3D_COLOR_MASK_A					0x0000f000
+
+#define NVC0_3D_QUERY_ADDRESS_HIGH				0x00001b00
+
+#define NVC0_3D_QUERY_ADDRESS_LOW				0x00001b04
+
+#define NVC0_3D_QUERY_SEQUENCE					0x00001b08
+
+#define NVC0_3D_QUERY_GET					0x00001b0c
+#define NVC0_3D_QUERY_GET_FENCE					0x1000f010
+#define NVC0_3D_QUERY_GET_SAMPLE_COUNT				0x0100f002
+#define NVC0_3D_QUERY_GET_TFB					0x05805002
+#define NVC0_3D_QUERY_GET_GENERATED_PRIMS			0x06805002
+#define NVC0_3D_QUERY_GET_UNK00005002				0x00005002
+
+#define NVC0_3D_VERTEX_ARRAY_FETCH(i0)			       (0x00001c00 + 0x10*(i0))
+#define NVC0_3D_VERTEX_ARRAY_FETCH__ESIZE			0x00000010
+#define NVC0_3D_VERTEX_ARRAY_FETCH__LEN				0x00000020
+#define NVC0_3D_VERTEX_ARRAY_FETCH_STRIDE__MASK			0x00000fff
+#define NVC0_3D_VERTEX_ARRAY_FETCH_STRIDE__SHIFT		0
+#define NVC0_3D_VERTEX_ARRAY_FETCH_ENABLE			0x00001000
+
+#define NVC0_3D_VERTEX_ARRAY_DIVISOR(i0)		       (0x00001c0c + 0x10*(i0))
+#define NVC0_3D_VERTEX_ARRAY_DIVISOR__ESIZE			0x00000010
+#define NVC0_3D_VERTEX_ARRAY_DIVISOR__LEN			0x00000020
+
+#define NVC0_3D_IBLEND(i0)				       (0x00001e00 + 0x20*(i0))
+#define NVC0_3D_IBLEND__ESIZE					0x00000020
+#define NVC0_3D_IBLEND__LEN					0x00000008
+
+#define NVC0_3D_IBLEND_EQUATION_RGB(i0)			       (0x00001e04 + 0x20*(i0))
+#define NVC0_3D_IBLEND_EQUATION_RGB_FUNC_ADD			0x00008006
+#define NVC0_3D_IBLEND_EQUATION_RGB_MIN				0x00008007
+#define NVC0_3D_IBLEND_EQUATION_RGB_MAX				0x00008008
+#define NVC0_3D_IBLEND_EQUATION_RGB_FUNC_SUBTRACT		0x0000800a
+#define NVC0_3D_IBLEND_EQUATION_RGB_FUNC_REVERSE_SUBTRACT	0x0000800b
+
+#define NVC0_3D_IBLEND_FUNC_SRC_RGB(i0)			       (0x00001e08 + 0x20*(i0))
+
+#define NVC0_3D_IBLEND_FUNC_DST_RGB(i0)			       (0x00001e0c + 0x20*(i0))
+
+#define NVC0_3D_IBLEND_EQUATION_ALPHA(i0)		       (0x00001e10 + 0x20*(i0))
+#define NVC0_3D_IBLEND_EQUATION_ALPHA_FUNC_ADD			0x00008006
+#define NVC0_3D_IBLEND_EQUATION_ALPHA_MIN			0x00008007
+#define NVC0_3D_IBLEND_EQUATION_ALPHA_MAX			0x00008008
+#define NVC0_3D_IBLEND_EQUATION_ALPHA_FUNC_SUBTRACT		0x0000800a
+#define NVC0_3D_IBLEND_EQUATION_ALPHA_FUNC_REVERSE_SUBTRACT	0x0000800b
+
+#define NVC0_3D_IBLEND_FUNC_SRC_ALPHA(i0)		       (0x00001e14 + 0x20*(i0))
+
+#define NVC0_3D_IBLEND_FUNC_DST_ALPHA(i0)		       (0x00001e18 + 0x20*(i0))
+
+#define NVC0_3D_SP(i0)					       (0x00002000 + 0x40*(i0))
+#define NVC0_3D_SP__ESIZE					0x00000040
+#define NVC0_3D_SP__LEN						0x00000006
+
+#define NVC0_3D_SP_SELECT(i0)				       (0x00002000 + 0x40*(i0))
+#define NVC0_3D_SP_SELECT_ENABLE				0x00000001
+#define NVC0_3D_SP_SELECT_PROGRAM__MASK				0x00000070
+#define NVC0_3D_SP_SELECT_PROGRAM__SHIFT			4
+#define NVC0_3D_SP_SELECT_PROGRAM_VP_A				0x00000000
+#define NVC0_3D_SP_SELECT_PROGRAM_VP_B				0x00000010
+#define NVC0_3D_SP_SELECT_PROGRAM_TCP				0x00000020
+#define NVC0_3D_SP_SELECT_PROGRAM_TEP				0x00000030
+#define NVC0_3D_SP_SELECT_PROGRAM_GP				0x00000040
+#define NVC0_3D_SP_SELECT_PROGRAM_FP				0x00000050
+
+#define NVC0_3D_SP_START_ID(i0)				       (0x00002004 + 0x40*(i0))
+
+#define NVC0_3D_SP_GPR_ALLOC(i0)			       (0x0000200c + 0x40*(i0))
+
+#define NVC0_3D_TEX_LIMITS(i0)				       (0x00002200 + 0x10*(i0))
+#define NVC0_3D_TEX_LIMITS__ESIZE				0x00000010
+#define NVC0_3D_TEX_LIMITS__LEN					0x00000005
+
+#define NVC0_3D_CB_SIZE						0x00002380
+
+#define NVC0_3D_CB_ADDRESS_HIGH					0x00002384
+
+#define NVC0_3D_CB_ADDRESS_LOW					0x00002388
+
+#define NVC0_3D_CB_POS						0x0000238c
+
+#define NVC0_3D_CB_DATA(i0)				       (0x00002390 + 0x4*(i0))
+#define NVC0_3D_CB_DATA__ESIZE					0x00000004
+#define NVC0_3D_CB_DATA__LEN					0x00000010
+
+#define NVC0_3D_BIND_TSC(i0)				       (0x00002400 + 0x20*(i0))
+#define NVC0_3D_BIND_TSC__ESIZE					0x00000020
+#define NVC0_3D_BIND_TSC__LEN					0x00000005
+#define NVC0_3D_BIND_TSC_ACTIVE					0x00000001
+#define NVC0_3D_BIND_TSC_SAMPLER__MASK				0x00000ff0
+#define NVC0_3D_BIND_TSC_SAMPLER__SHIFT				4
+#define NVC0_3D_BIND_TSC_TSC__MASK				0x01fff000
+#define NVC0_3D_BIND_TSC_TSC__SHIFT				12
+
+#define NVC0_3D_BIND_TIC(i0)				       (0x00002404 + 0x20*(i0))
+#define NVC0_3D_BIND_TIC__ESIZE					0x00000020
+#define NVC0_3D_BIND_TIC__LEN					0x00000005
+#define NVC0_3D_BIND_TIC_ACTIVE					0x00000001
+#define NVC0_3D_BIND_TIC_TEXTURE__MASK				0x000001fe
+#define NVC0_3D_BIND_TIC_TEXTURE__SHIFT				1
+#define NVC0_3D_BIND_TIC_TIC__MASK				0x7ffffe00
+#define NVC0_3D_BIND_TIC_TIC__SHIFT				9
+
+#define NVC0_3D_CB_BIND(i0)				       (0x00002410 + 0x20*(i0))
+#define NVC0_3D_CB_BIND__ESIZE					0x00000020
+#define NVC0_3D_CB_BIND__LEN					0x00000005
+#define NVC0_3D_CB_BIND_VALID					0x00000001
+#define NVC0_3D_CB_BIND_INDEX__MASK				0x000000f0
+#define NVC0_3D_CB_BIND_INDEX__SHIFT				4
+
+#define NVC0_3D_VERT_COLOR_CLAMP_EN				0x00002600
+
+#define NVC0_3D_TFB_VARYING_LOCS(i0)			       (0x00002800 + 0x4*(i0))
+#define NVC0_3D_TFB_VARYING_LOCS__ESIZE				0x00000004
+#define NVC0_3D_TFB_VARYING_LOCS__LEN				0x00000080
+
+#define NVC0_3D_COLOR_MASK_BROADCAST				0x00003808
+
+#define NVC0_3D_VERTEX_ARRAY_SELECT				0x00003820
+
+#define NVC0_3D_VERTEX_ARRAY_LIMIT_HIGH				0x00003824
+
+#define NVC0_3D_VERTEX_ARRAY_LIMIT_LOW				0x00003828
+
+#define NVC0_3D_VERTEX_ARRAY_START_HIGH				0x0000382c
+
+#define NVC0_3D_VERTEX_ARRAY_START_LOW				0x00003830
+
+#define NVC0_3D_BLEND_ENABLES					0x00003858
+
+#define NVC0_3D_POLYGON_MODE_FRONT				0x00003868
+#define NVC0_3D_POLYGON_MODE_FRONT_POINT			0x00001b00
+#define NVC0_3D_POLYGON_MODE_FRONT_LINE				0x00001b01
+#define NVC0_3D_POLYGON_MODE_FRONT_FILL				0x00001b02
+
+#define NVC0_3D_POLYGON_MODE_BACK				0x00003870
+#define NVC0_3D_POLYGON_MODE_BACK_POINT				0x00001b00
+#define NVC0_3D_POLYGON_MODE_BACK_LINE				0x00001b01
+#define NVC0_3D_POLYGON_MODE_BACK_FILL				0x00001b02
+
+#define NVC0_3D_GP_SELECT					0x00003878
+
+#define NVC0_3D_TEP_SELECT					0x00003880
+
+
+#endif /* NVC0_3D_XML */
diff --git a/src/gallium/drivers/nvc0/nvc0_3ddefs.xml.h b/src/gallium/drivers/nvc0/nvc0_3ddefs.xml.h
new file mode 100644
index 00000000000..84b152213a2
--- /dev/null
+++ b/src/gallium/drivers/nvc0/nvc0_3ddefs.xml.h
@@ -0,0 +1,98 @@
+#ifndef NV_3DDEFS_XML
+#define NV_3DDEFS_XML
+
+/* Autogenerated file, DO NOT EDIT manually!
+
+This file was generated by the rules-ng-ng headergen tool in this git repository:
+http://0x04.net/cgit/index.cgi/rules-ng-ng
+git clone git://0x04.net/rules-ng-ng
+
+The rules-ng-ng source files this header was generated from are:
+- nvc0_3d.xml    (  26312 bytes, from 2010-10-08 10:10:01)
+- copyright.xml  (   6498 bytes, from 2010-10-03 13:18:37)
+- nv_defs.xml    (   4437 bytes, from 2010-07-06 07:43:58)
+- nv_3ddefs.xml  (  16397 bytes, from 2010-10-08 13:30:38)
+- nv_object.xml  (  11249 bytes, from 2010-10-07 15:31:28)
+- nvchipsets.xml (   2824 bytes, from 2010-07-07 13:41:20)
+- nv50_defs.xml  (   4482 bytes, from 2010-10-03 13:18:37)
+
+Copyright (C) 2006-2010 by the following authors:
+- Artur Huillet <[email protected]> (ahuillet)
+- Ben Skeggs (darktama, darktama_)
+- B. R. <[email protected]> (koala_br)
+- Carlos Martin <[email protected]> (carlosmn)
+- Christoph Bumiller <[email protected]> (calim, chrisbmr)
+- Dawid Gajownik <[email protected]> (gajownik)
+- Dmitry Baryshkov
+- Dmitry Eremin-Solenikov <[email protected]> (lumag)
+- EdB <[email protected]> (edb_)
+- Erik Waling <[email protected]> (erikwaling)
+- Francisco Jerez <[email protected]> (curro, curro_, currojerez)
+- imirkin <[email protected]> (imirkin)
+- jb17bsome <[email protected]> (jb17bsome)
+- Jeremy Kolb <[email protected]> (kjeremy)
+- Laurent Carlier <[email protected]> (lordheavy)
+- Luca Barbieri <[email protected]> (lb, lb1)
+- Maarten Maathuis <[email protected]> (stillunknown)
+- Marcin Kościelnicki <[email protected]> (mwk, koriakin)
+- Mark Carey <[email protected]> (careym)
+- Matthieu Castet <[email protected]> (mat-c)
+- nvidiaman <[email protected]> (nvidiaman)
+- Patrice Mandin <[email protected]> (pmandin, pmdata)
+- Pekka Paalanen <[email protected]> (pq, ppaalanen)
+- Peter Popov <[email protected]> (ironpeter)
+- Richard Hughes <[email protected]> (hughsient)
+- Rudi Cilibrasi <[email protected]> (cilibrar)
+- Serge Martin
+- Simon Raffeiner
+- Stephane Loeuillet <[email protected]> (leroutier)
+- Stephane Marchesin <[email protected]> (marcheu)
+- sturmflut <[email protected]> (sturmflut)
+- Sylvain Munaut <[email protected]>
+- Victor Stinner <[email protected]> (haypo)
+- Wladmir van der Laan <[email protected]> (miathan6)
+- Younes Manton <[email protected]> (ymanton)
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice (including the
+next paragraph) shall be included in all copies or substantial
+portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/
+
+
+#define NV50_3D_BLEND_FACTOR_ZERO				0x00004000
+#define NV50_3D_BLEND_FACTOR_ONE				0x00004001
+#define NV50_3D_BLEND_FACTOR_SRC_COLOR				0x00004300
+#define NV50_3D_BLEND_FACTOR_ONE_MINUS_SRC_COLOR		0x00004301
+#define NV50_3D_BLEND_FACTOR_SRC_ALPHA				0x00004302
+#define NV50_3D_BLEND_FACTOR_ONE_MINUS_SRC_ALPHA		0x00004303
+#define NV50_3D_BLEND_FACTOR_DST_ALPHA				0x00004304
+#define NV50_3D_BLEND_FACTOR_ONE_MINUS_DST_ALPHA		0x00004305
+#define NV50_3D_BLEND_FACTOR_DST_COLOR				0x00004306
+#define NV50_3D_BLEND_FACTOR_ONE_MINUS_DST_COLOR		0x00004307
+#define NV50_3D_BLEND_FACTOR_SRC_ALPHA_SATURATE			0x00004308
+#define NV50_3D_BLEND_FACTOR_CONSTANT_COLOR			0x0000c001
+#define NV50_3D_BLEND_FACTOR_ONE_MINUS_CONSTANT_COLOR		0x0000c002
+#define NV50_3D_BLEND_FACTOR_CONSTANT_ALPHA			0x0000c003
+#define NV50_3D_BLEND_FACTOR_ONE_MINUS_CONSTANT_ALPHA		0x0000c004
+#define NV50_3D_BLEND_FACTOR_SRC1_COLOR				0x0000c900
+#define NV50_3D_BLEND_FACTOR_ONE_MINUS_SRC1_COLOR		0x0000c901
+#define NV50_3D_BLEND_FACTOR_SRC1_ALPHA				0x0000c902
+#define NV50_3D_BLEND_FACTOR_ONE_MINUS_SRC1_ALPHA		0x0000c903
+
+#endif /* NV_3DDEFS_XML */
diff --git a/src/gallium/drivers/nvc0/nvc0_buffer.c b/src/gallium/drivers/nvc0/nvc0_buffer.c
new file mode 100644
index 00000000000..06841bb19b8
--- /dev/null
+++ b/src/gallium/drivers/nvc0/nvc0_buffer.c
@@ -0,0 +1,468 @@
+
+#include "util/u_inlines.h"
+#include "util/u_memory.h"
+#include "util/u_math.h"
+
+#define NOUVEAU_NVC0
+#include "nouveau/nouveau_screen.h"
+#include "nouveau/nouveau_winsys.h"
+#undef NOUVEAU_NVC0
+
+#include "nvc0_context.h"
+#include "nvc0_resource.h"
+
+struct nvc0_transfer {
+   struct pipe_transfer base;
+};
+
+static INLINE struct nvc0_transfer *
+nvc0_transfer(struct pipe_transfer *transfer)
+{
+   return (struct nvc0_transfer *)transfer;
+}
+
+static INLINE boolean
+nvc0_buffer_allocate(struct nvc0_screen *screen, struct nvc0_resource *buf,
+                     unsigned domain)
+{
+   if (domain == NOUVEAU_BO_VRAM) {
+      buf->mm = nvc0_mm_allocate(screen->mm_VRAM, buf->base.width0, &buf->bo,
+                                 &buf->offset);
+      if (!buf->bo)
+         return nvc0_buffer_allocate(screen, buf, NOUVEAU_BO_GART);
+   } else
+   if (domain == NOUVEAU_BO_GART) {
+      buf->mm = nvc0_mm_allocate(screen->mm_GART, buf->base.width0, &buf->bo,
+                                 &buf->offset);
+      if (!buf->bo)
+         return FALSE;
+   }
+   if (domain != NOUVEAU_BO_GART) {
+      if (!buf->data) {
+         buf->data = MALLOC(buf->base.width0);
+         if (!buf->data)
+            return FALSE;
+      }
+   }
+   buf->domain = domain;
+   return TRUE;
+}
+
+static INLINE void
+release_allocation(struct nvc0_mm_allocation **mm, struct nvc0_fence *fence)
+{
+   if (fence && fence->state != NVC0_FENCE_STATE_SIGNALLED) {
+      (*mm)->next = fence->buffers;
+      fence->buffers = (*mm);
+   } else {
+      nvc0_mm_free(*mm);
+   }
+   (*mm) = NULL;
+}
+
+static void
+nvc0_buffer_destroy(struct pipe_screen *pscreen,
+                    struct pipe_resource *presource)
+{
+   struct nvc0_resource *res = nvc0_resource(presource);
+
+   nouveau_bo_ref(NULL, &res->bo);
+
+   if (res->mm)
+      release_allocation(&res->mm, res->fence);
+
+   if (res->data && !(res->status & NVC0_BUFFER_STATUS_USER_MEMORY))
+      FREE(res->data);
+
+   FREE(res);
+}
+
+/* Maybe just migrate to GART right away if we actually need to do this. */
+boolean
+nvc0_buffer_download(struct nvc0_context *nvc0, struct nvc0_resource *buf,
+                     unsigned start, unsigned size)
+{
+   struct nvc0_mm_allocation *mm;
+   struct nouveau_bo *bounce = NULL;
+   uint32_t offset;
+
+   assert(buf->domain == NOUVEAU_BO_VRAM);
+
+   mm = nvc0_mm_allocate(nvc0->screen->mm_GART, size, &bounce, &offset);
+   if (!bounce)
+      return FALSE;
+
+   nvc0_m2mf_copy_linear(nvc0, bounce, offset, NOUVEAU_BO_GART,
+                         buf->bo, buf->offset + start, NOUVEAU_BO_VRAM,
+                         size);
+
+   if (nouveau_bo_map_range(bounce, offset, size, NOUVEAU_BO_RD))
+      return FALSE;
+   memcpy(buf->data + start, bounce->map, size);
+   nouveau_bo_unmap(bounce);
+
+   buf->status &= ~NVC0_BUFFER_STATUS_DIRTY;
+
+   nouveau_bo_ref(NULL, &bounce);
+   if (mm)
+      nvc0_mm_free(mm);
+   return TRUE;
+}
+
+static boolean
+nvc0_buffer_upload(struct nvc0_context *nvc0, struct nvc0_resource *buf,
+                   unsigned start, unsigned size)
+{
+   struct nvc0_mm_allocation *mm;
+   struct nouveau_bo *bounce = NULL;
+   uint32_t offset;
+
+   mm = nvc0_mm_allocate(nvc0->screen->mm_GART, size, &bounce, &offset);
+   if (!bounce)
+      return FALSE;
+
+   nouveau_bo_map_range(bounce, offset, size,
+                        NOUVEAU_BO_WR | NOUVEAU_BO_NOSYNC);
+   memcpy(bounce->map, buf->data + start, size);
+   nouveau_bo_unmap(bounce);
+
+   nvc0_m2mf_copy_linear(nvc0, buf->bo, buf->offset + start, NOUVEAU_BO_VRAM,
+                         bounce, offset, NOUVEAU_BO_GART, size);
+
+   nouveau_bo_ref(NULL, &bounce);
+   if (mm)
+      release_allocation(&mm, nvc0->screen->fence.current);
+
+   if (start == 0 && size == buf->base.width0)
+      buf->status &= ~NVC0_BUFFER_STATUS_DIRTY;
+   return TRUE;
+}
+
+static struct pipe_transfer *
+nvc0_buffer_transfer_get(struct pipe_context *pipe,
+                         struct pipe_resource *resource,
+                         unsigned level,
+                         unsigned usage,
+                         const struct pipe_box *box)
+{
+   struct nvc0_resource *buf = nvc0_resource(resource);
+   struct nvc0_transfer *xfr = CALLOC_STRUCT(nvc0_transfer);
+   if (!xfr)
+      return NULL;
+
+   xfr->base.resource = resource;
+   xfr->base.box.x = box->x;
+   xfr->base.box.width = box->width;
+   xfr->base.usage = usage;
+
+   if (buf->domain == NOUVEAU_BO_VRAM) {
+      if (usage & PIPE_TRANSFER_READ) {
+         if (buf->status & NVC0_BUFFER_STATUS_DIRTY)
+            nvc0_buffer_download(nvc0_context(pipe), buf, 0, buf->base.width0);
+      }
+   }
+
+   return &xfr->base;
+}
+
+static void
+nvc0_buffer_transfer_destroy(struct pipe_context *pipe,
+                             struct pipe_transfer *transfer)
+{
+   struct nvc0_resource *buf = nvc0_resource(transfer->resource);
+   struct nvc0_transfer *xfr = nvc0_transfer(transfer);
+
+   if (xfr->base.usage & PIPE_TRANSFER_WRITE) {
+      /* writing is worse */
+      nvc0_buffer_adjust_score(nvc0_context(pipe), buf, -5000);
+
+      if (buf->domain == NOUVEAU_BO_VRAM) {
+         nvc0_buffer_upload(nvc0_context(pipe), buf,
+                            transfer->box.x, transfer->box.width);
+      }
+
+      if (buf->domain != 0 && (buf->base.bind & (PIPE_BIND_VERTEX_BUFFER |
+                                                 PIPE_BIND_INDEX_BUFFER)))
+         nvc0_context(pipe)->vbo_dirty = TRUE;
+   }
+
+   FREE(xfr);
+}
+
+static INLINE boolean
+nvc0_buffer_sync(struct nvc0_resource *buf, unsigned rw)
+{
+   if (rw == PIPE_TRANSFER_READ) {
+      if (!buf->fence_wr)
+         return TRUE;
+      if (!nvc0_fence_wait(buf->fence_wr))
+         return FALSE;
+   } else {
+      if (!buf->fence)
+         return TRUE;
+      if (!nvc0_fence_wait(buf->fence))
+         return FALSE;
+
+      nvc0_fence_reference(&buf->fence, NULL);
+   }
+   nvc0_fence_reference(&buf->fence_wr, NULL);
+
+   return TRUE;
+}
+
+static INLINE boolean
+nvc0_buffer_busy(struct nvc0_resource *buf, unsigned rw)
+{
+   if (rw == PIPE_TRANSFER_READ)
+      return (buf->fence_wr && !nvc0_fence_signalled(buf->fence_wr));
+   else
+      return (buf->fence && !nvc0_fence_signalled(buf->fence));
+}
+
+static void *
+nvc0_buffer_transfer_map(struct pipe_context *pipe,
+                         struct pipe_transfer *transfer)
+{
+   struct nvc0_transfer *xfr = nvc0_transfer(transfer);
+   struct nvc0_resource *buf = nvc0_resource(transfer->resource);
+   struct nouveau_bo *bo = buf->bo;
+   uint8_t *map;
+   int ret;
+   uint32_t offset = xfr->base.box.x;
+   uint32_t flags;
+
+   nvc0_buffer_adjust_score(nvc0_context(pipe), buf, -250);
+
+   if (buf->domain != NOUVEAU_BO_GART)
+      return buf->data + offset;
+
+   if (buf->mm)
+      flags = NOUVEAU_BO_NOSYNC | NOUVEAU_BO_RDWR;
+   else
+      flags = nouveau_screen_transfer_flags(xfr->base.usage);
+
+   offset += buf->offset;
+
+   ret = nouveau_bo_map_range(buf->bo, offset, xfr->base.box.width, flags);
+   if (ret)
+      return NULL;
+   map = bo->map;
+
+   /* Unmap right now. Since multiple buffers can share a single nouveau_bo,
+    * not doing so might make future maps fail or trigger "reloc while mapped"
+    * errors. For now, mappings to userspace are guaranteed to be persistent.
+    */
+   nouveau_bo_unmap(bo);
+
+   if (buf->mm) {
+      if (xfr->base.usage & PIPE_TRANSFER_DONTBLOCK) {
+         if (nvc0_buffer_busy(buf, xfr->base.usage & PIPE_TRANSFER_READ_WRITE))
+            return NULL;
+      } else
+      if (!(xfr->base.usage & PIPE_TRANSFER_UNSYNCHRONIZED)) {
+         nvc0_buffer_sync(buf, xfr->base.usage & PIPE_TRANSFER_READ_WRITE);
+      }
+   }
+   return map;
+}
+
+
+
+static void
+nvc0_buffer_transfer_flush_region(struct pipe_context *pipe,
+                                  struct pipe_transfer *transfer,
+                                  const struct pipe_box *box)
+{
+   struct nvc0_resource *res = nvc0_resource(transfer->resource);
+   struct nouveau_bo *bo = res->bo;
+   unsigned offset = res->offset + transfer->box.x + box->x;
+
+   /* not using non-snoop system memory yet, no need for cflush */
+   if (1)
+      return;
+
+   /* XXX: maybe need to upload for VRAM buffers here */
+
+   nouveau_screen_bo_map_flush_range(pipe->screen, bo, offset, box->width);
+}
+
+static void
+nvc0_buffer_transfer_unmap(struct pipe_context *pipe,
+                           struct pipe_transfer *transfer)
+{
+   /* we've called nouveau_bo_unmap right after map */
+}
+
+const struct u_resource_vtbl nvc0_buffer_vtbl =
+{
+   u_default_resource_get_handle,     /* get_handle */
+   nvc0_buffer_destroy,               /* resource_destroy */
+   NULL,                              /* is_resource_referenced */
+   nvc0_buffer_transfer_get,          /* get_transfer */
+   nvc0_buffer_transfer_destroy,      /* transfer_destroy */
+   nvc0_buffer_transfer_map,          /* transfer_map */
+   nvc0_buffer_transfer_flush_region, /* transfer_flush_region */
+   nvc0_buffer_transfer_unmap,        /* transfer_unmap */
+   u_default_transfer_inline_write    /* transfer_inline_write */
+};
+
+struct pipe_resource *
+nvc0_buffer_create(struct pipe_screen *pscreen,
+                   const struct pipe_resource *templ)
+{
+   struct nvc0_screen *screen = nvc0_screen(pscreen);
+   struct nvc0_resource *buffer;
+   boolean ret;
+
+   buffer = CALLOC_STRUCT(nvc0_resource);
+   if (!buffer)
+      return NULL;
+
+   buffer->base = *templ;
+   buffer->vtbl = &nvc0_buffer_vtbl;
+   pipe_reference_init(&buffer->base.reference, 1);
+   buffer->base.screen = pscreen;
+
+   if (buffer->base.bind & PIPE_BIND_CONSTANT_BUFFER)
+      ret = nvc0_buffer_allocate(screen, buffer, 0);
+   else
+      ret = nvc0_buffer_allocate(screen, buffer, NOUVEAU_BO_GART);
+
+   if (ret == FALSE)
+      goto fail;
+
+   return &buffer->base;
+
+fail:
+   FREE(buffer);
+   return NULL;
+}
+
+
+struct pipe_resource *
+nvc0_user_buffer_create(struct pipe_screen *pscreen,
+                        void *ptr,
+                        unsigned bytes,
+                        unsigned bind)
+{
+   struct nvc0_resource *buffer;
+
+   buffer = CALLOC_STRUCT(nvc0_resource);
+   if (!buffer)
+      return NULL;
+
+   pipe_reference_init(&buffer->base.reference, 1);
+   buffer->vtbl = &nvc0_buffer_vtbl;
+   buffer->base.screen = pscreen;
+   buffer->base.format = PIPE_FORMAT_R8_UNORM;
+   buffer->base.usage = PIPE_USAGE_IMMUTABLE;
+   buffer->base.bind = bind;
+   buffer->base.width0 = bytes;
+   buffer->base.height0 = 1;
+   buffer->base.depth0 = 1;
+
+   buffer->data = ptr;
+   buffer->status = NVC0_BUFFER_STATUS_USER_MEMORY;
+
+   return &buffer->base;
+}
+
+static INLINE boolean
+nvc0_buffer_fetch_data(struct nvc0_resource *buf,
+                       struct nouveau_bo *bo, unsigned offset, unsigned size)
+{
+   if (!buf->data) {
+      buf->data = MALLOC(size);
+      if (!buf->data)
+         return FALSE;
+   }
+   if (nouveau_bo_map_range(bo, offset, size, NOUVEAU_BO_RD))
+      return FALSE;
+   memcpy(buf->data, bo->map, size);
+   nouveau_bo_unmap(bo);
+
+   return TRUE;
+}
+
+/* Migrate a linear buffer (vertex, index, constants) USER -> GART -> VRAM. */
+boolean
+nvc0_buffer_migrate(struct nvc0_context *nvc0,
+                    struct nvc0_resource *buf, const unsigned new_domain)
+{
+   struct nvc0_screen *screen = nvc0_screen(buf->base.screen);
+   struct nouveau_bo *bo;
+   const unsigned old_domain = buf->domain;
+   unsigned size = buf->base.width0;
+   unsigned offset;
+   int ret;
+
+   assert(new_domain != old_domain);
+
+   if (new_domain == NOUVEAU_BO_GART && old_domain == 0) {
+      if (!nvc0_buffer_allocate(screen, buf, new_domain))
+         return FALSE;
+      ret = nouveau_bo_map_range(buf->bo, buf->offset, size, NOUVEAU_BO_WR |
+                                 NOUVEAU_BO_NOSYNC);
+      if (ret)
+         return ret;
+      memcpy(buf->bo->map, buf->data, size);
+      nouveau_bo_unmap(buf->bo);
+      FREE(buf->data);
+   } else
+   if (old_domain != 0 && new_domain != 0) {
+      struct nvc0_mm_allocation *mm = buf->mm;
+
+      if (new_domain == NOUVEAU_BO_VRAM) {
+         /* keep a system memory copy of our data in case we hit a fallback */
+         if (!nvc0_buffer_fetch_data(buf, buf->bo, buf->offset, size))
+            return FALSE;
+         debug_printf("migrating %u KiB to VRAM\n", size / 1024);
+      }
+
+      offset = buf->offset;
+      bo = buf->bo;
+      buf->bo = NULL;
+      buf->mm = NULL;
+      nvc0_buffer_allocate(screen, buf, new_domain);
+
+      nvc0_m2mf_copy_linear(nvc0, buf->bo, buf->offset, new_domain,
+                            bo, offset, old_domain, buf->base.width0);
+
+      nouveau_bo_ref(NULL, &bo);
+      if (mm)
+         release_allocation(&mm, screen->fence.current);
+   } else
+   if (new_domain == NOUVEAU_BO_VRAM && old_domain == 0) {
+      if (!nvc0_buffer_allocate(screen, buf, NOUVEAU_BO_VRAM))
+         return FALSE;
+      if (!nvc0_buffer_upload(nvc0, buf, 0, buf->base.width0))
+         return FALSE;
+   } else
+      return FALSE;
+
+   assert(buf->domain == new_domain);
+   return TRUE;
+}
+
+/* Migrate data from glVertexAttribPointer(non-VBO) user buffers to GART.
+ * MUST NOT FLUSH THE PUSH BUFFER, we could be in the middle of a method.
+ */
+boolean
+nvc0_migrate_vertices(struct nvc0_resource *buf, unsigned base, unsigned size)
+{
+   struct nvc0_screen *screen = nvc0_screen(buf->base.screen);
+   int ret;
+
+   assert(buf->data && !buf->domain);
+
+   if (!nvc0_buffer_allocate(screen, buf, NOUVEAU_BO_GART))
+      return FALSE;
+   ret = nouveau_bo_map_range(buf->bo, base + buf->offset, size,
+                              NOUVEAU_BO_WR | NOUVEAU_BO_NOSYNC);
+   if (ret)
+      return FALSE;
+   memcpy(buf->bo->map, buf->data + base, size);
+   nouveau_bo_unmap(buf->bo);
+
+   return TRUE;
+}
diff --git a/src/gallium/drivers/nvc0/nvc0_context.c b/src/gallium/drivers/nvc0/nvc0_context.c
new file mode 100644
index 00000000000..b2b4fd62eeb
--- /dev/null
+++ b/src/gallium/drivers/nvc0/nvc0_context.c
@@ -0,0 +1,163 @@
+/*
+ * Copyright 2010 Christoph Bumiller
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF
+ * OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "draw/draw_context.h"
+#include "pipe/p_defines.h"
+
+#include "nvc0_context.h"
+#include "nvc0_screen.h"
+#include "nvc0_resource.h"
+
+#include "nouveau/nouveau_reloc.h"
+
+static void
+nvc0_flush(struct pipe_context *pipe, unsigned flags,
+           struct pipe_fence_handle **fence)
+{
+   struct nvc0_context *nvc0 = nvc0_context(pipe);
+   struct nouveau_channel *chan = nvc0->screen->base.channel;
+
+   if (flags & PIPE_FLUSH_TEXTURE_CACHE) {
+      BEGIN_RING(chan, RING_3D(SERIALIZE), 1);
+      OUT_RING  (chan, 0);
+      BEGIN_RING(chan, RING_3D(TEX_CACHE_CTL), 1);
+      OUT_RING  (chan, 0x00);
+   }
+
+   if (fence) {
+      nvc0_screen_fence_new(nvc0->screen, (struct nvc0_fence **)fence, TRUE);
+   }
+
+   if (flags & (PIPE_FLUSH_SWAPBUFFERS | PIPE_FLUSH_FRAME)) {
+      FIRE_RING(chan);
+
+      nvc0_screen_fence_next(nvc0->screen);
+   }
+}
+
+static void
+nvc0_destroy(struct pipe_context *pipe)
+{
+   struct nvc0_context *nvc0 = nvc0_context(pipe);
+
+   draw_destroy(nvc0->draw);
+
+   if (nvc0->screen->cur_ctx == nvc0)
+      nvc0->screen->cur_ctx = NULL;
+
+   FREE(nvc0);
+}
+
+struct pipe_context *
+nvc0_create(struct pipe_screen *pscreen, void *priv)
+{
+   struct pipe_winsys *pipe_winsys = pscreen->winsys;
+   struct nvc0_screen *screen = nvc0_screen(pscreen);
+   struct nvc0_context *nvc0;
+
+   nvc0 = CALLOC_STRUCT(nvc0_context);
+   if (!nvc0)
+      return NULL;
+   nvc0->screen = screen;
+
+   nvc0->pipe.winsys = pipe_winsys;
+   nvc0->pipe.screen = pscreen;
+   nvc0->pipe.priv = priv;
+
+   nvc0->pipe.destroy = nvc0_destroy;
+
+   nvc0->pipe.draw_vbo = nvc0_draw_vbo;
+   nvc0->pipe.clear = nvc0_clear;
+
+   nvc0->pipe.flush = nvc0_flush;
+
+   screen->base.channel->user_private = nvc0;
+
+   nvc0_init_surface_functions(nvc0);
+   nvc0_init_state_functions(nvc0);
+   nvc0_init_resource_functions(&nvc0->pipe);
+
+   nvc0->draw = draw_create(&nvc0->pipe);
+   assert(nvc0->draw);
+   draw_set_rasterize_stage(nvc0->draw, nvc0_draw_render_stage(nvc0));
+
+   return &nvc0->pipe;
+}
+
+struct resident {
+   struct nvc0_resource *res;
+   uint32_t flags;
+};
+
+void
+nvc0_bufctx_add_resident(struct nvc0_context *nvc0, int ctx,
+                         struct nvc0_resource *resource, uint32_t flags)
+{
+   struct resident rsd = { resource, flags };
+
+   if (!resource->bo)
+      return;
+
+   /* We don't need to reference the resource here, it will be referenced
+    * in the context/state, and bufctx will be reset when state changes.
+    */
+   util_dynarray_append(&nvc0->residents[ctx], struct resident, rsd);
+}
+
+void
+nvc0_bufctx_del_resident(struct nvc0_context *nvc0, int ctx,
+                         struct nvc0_resource *resource)
+{
+   struct resident *rsd, *top;
+   unsigned i;
+
+   for (i = 0; i < nvc0->residents[ctx].size / sizeof(struct resident); ++i) {
+      rsd = util_dynarray_element(&nvc0->residents[ctx], struct resident, i);
+
+      if (rsd->res == resource) {
+         top = util_dynarray_pop_ptr(&nvc0->residents[ctx], struct resident);
+         if (rsd != top)
+            *rsd = *top;
+         break;
+      }
+   }
+}
+
+void
+nvc0_bufctx_emit_relocs(struct nvc0_context *nvc0)
+{
+   struct resident *rsd;
+   struct util_dynarray *array;
+   unsigned ctx, i;
+
+   for (ctx = 0; ctx < NVC0_BUFCTX_COUNT; ++ctx) {
+      array = &nvc0->residents[ctx];
+
+      for (i = 0; i < array->size / sizeof(struct resident); ++i) {
+         rsd = util_dynarray_element(array, struct resident, i);
+
+         nvc0_resource_validate(rsd->res, rsd->flags);
+      }
+   }
+
+   nvc0_screen_make_buffers_resident(nvc0->screen);
+}
diff --git a/src/gallium/drivers/nvc0/nvc0_context.h b/src/gallium/drivers/nvc0/nvc0_context.h
new file mode 100644
index 00000000000..c181f15dbd8
--- /dev/null
+++ b/src/gallium/drivers/nvc0/nvc0_context.h
@@ -0,0 +1,223 @@
+#ifndef __NVC0_CONTEXT_H__
+#define __NVC0_CONTEXT_H__
+
+#include <stdio.h>
+#include "pipe/p_context.h"
+#include "pipe/p_defines.h"
+#include "pipe/p_state.h"
+
+#include "util/u_memory.h"
+#include "util/u_math.h"
+#include "util/u_inlines.h"
+#include "util/u_dynarray.h"
+
+#include "draw/draw_vertex.h"
+
+#include "nvc0_winsys.h"
+#include "nvc0_stateobj.h"
+#include "nvc0_screen.h"
+#include "nvc0_program.h"
+#include "nvc0_resource.h"
+
+#include "nvc0_3ddefs.xml.h"
+#include "nvc0_3d.xml.h"
+#include "nvc0_2d.xml.h"
+#include "nvc0_m2mf.xml.h"
+
+#define NOUVEAU_ERR(fmt, args...) \
+   fprintf(stderr, "%s:%d -  "fmt, __FUNCTION__, __LINE__, ##args);
+
+#ifdef NOUVEAU_DEBUG
+# define NOUVEAU_DBG(args...) printf(args);
+#else
+# define NOUVEAU_DBG(args...)
+#endif
+
+#define NVC0_NEW_BLEND        (1 << 0)
+#define NVC0_NEW_RASTERIZER   (1 << 1)
+#define NVC0_NEW_ZSA          (1 << 2)
+#define NVC0_NEW_VERTPROG     (1 << 3)
+#define NVC0_NEW_TCTLPROG     (1 << 4)
+#define NVC0_NEW_TEVLPROG     (1 << 5)
+#define NVC0_NEW_GMTYPROG     (1 << 6)
+#define NVC0_NEW_FRAGPROG     (1 << 7)
+#define NVC0_NEW_BLEND_COLOUR (1 << 8)
+#define NVC0_NEW_STENCIL_REF  (1 << 9)
+#define NVC0_NEW_CLIP         (1 << 10)
+#define NVC0_NEW_SAMPLE_MASK  (1 << 11)
+#define NVC0_NEW_FRAMEBUFFER  (1 << 12)
+#define NVC0_NEW_STIPPLE      (1 << 13)
+#define NVC0_NEW_SCISSOR      (1 << 14)
+#define NVC0_NEW_VIEWPORT     (1 << 15)
+#define NVC0_NEW_ARRAYS       (1 << 16)
+#define NVC0_NEW_VERTEX       (1 << 17)
+#define NVC0_NEW_CONSTBUF     (1 << 18)
+#define NVC0_NEW_TEXTURES     (1 << 19)
+#define NVC0_NEW_SAMPLERS     (1 << 20)
+
+#define NVC0_BUFCTX_CONSTANT 0
+#define NVC0_BUFCTX_FRAME    1
+#define NVC0_BUFCTX_VERTEX   2
+#define NVC0_BUFCTX_TEXTURES 3
+#define NVC0_BUFCTX_COUNT    4
+
+struct nvc0_context {
+   struct pipe_context pipe;
+
+   struct nvc0_screen *screen;
+
+   struct util_dynarray residents[NVC0_BUFCTX_COUNT];
+
+   uint32_t dirty;
+
+   struct {
+      uint32_t instance_bits;
+      uint32_t instance_base;
+      int32_t index_bias;
+      boolean prim_restart;
+      uint8_t num_vtxbufs;
+      uint8_t num_vtxelts;
+      uint8_t num_textures[5];
+      uint8_t num_samplers[5];
+      uint16_t scissor;
+      uint32_t uniform_buffer_bound[5];
+   } state;
+
+   struct nvc0_blend_stateobj *blend;
+   struct nvc0_rasterizer_stateobj *rast;
+   struct nvc0_zsa_stateobj *zsa;
+   struct nvc0_vertex_stateobj *vertex;
+
+   struct nvc0_program *vertprog;
+   struct nvc0_program *tctlprog;
+   struct nvc0_program *tevlprog;
+   struct nvc0_program *gmtyprog;
+   struct nvc0_program *fragprog;
+
+   struct pipe_resource *constbuf[5][16];
+   uint16_t constbuf_dirty[5];
+
+   struct pipe_vertex_buffer vtxbuf[PIPE_MAX_ATTRIBS];
+   unsigned num_vtxbufs;
+   struct pipe_index_buffer idxbuf;
+   uint32_t vbo_fifo;
+   unsigned vbo_min_index; /* from pipe_draw_info, for vertex upload */
+   unsigned vbo_max_index;
+
+   struct pipe_sampler_view *textures[5][PIPE_MAX_SAMPLERS];
+   unsigned num_textures[5];
+   struct nvc0_tsc_entry *samplers[5][PIPE_MAX_SAMPLERS];
+   unsigned num_samplers[5];
+
+   struct pipe_framebuffer_state framebuffer;
+   struct pipe_blend_color blend_colour;
+   struct pipe_stencil_ref stencil_ref;
+   struct pipe_poly_stipple stipple;
+   struct pipe_scissor_state scissor;
+   struct pipe_viewport_state viewport;
+   struct pipe_clip_state clip;
+
+   unsigned sample_mask;
+
+   boolean vbo_dirty;
+   boolean vbo_push_hint;
+
+   struct draw_context *draw;
+};
+
+static INLINE struct nvc0_context *
+nvc0_context(struct pipe_context *pipe)
+{
+   return (struct nvc0_context *)pipe;
+}
+
+struct nvc0_surface {
+   struct pipe_surface base;
+   uint32_t offset;
+   uint32_t width;
+   uint16_t height;
+   uint16_t depth;
+};
+
+static INLINE struct nvc0_surface *
+nvc0_surface(struct pipe_surface *ps)
+{
+   return (struct nvc0_surface *)ps;
+}
+
+/* nvc0_context.c */
+struct pipe_context *nvc0_create(struct pipe_screen *, void *);
+
+void nvc0_bufctx_emit_relocs(struct nvc0_context *);
+void nvc0_bufctx_add_resident(struct nvc0_context *, int ctx,
+                              struct nvc0_resource *, uint32_t flags);
+void nvc0_bufctx_del_resident(struct nvc0_context *, int ctx,
+                              struct nvc0_resource *);
+static INLINE void
+nvc0_bufctx_reset(struct nvc0_context *nvc0, int ctx)
+{
+   util_dynarray_resize(&nvc0->residents[ctx], 0);
+}
+
+/* nvc0_draw.c */
+extern struct draw_stage *nvc0_draw_render_stage(struct nvc0_context *);
+
+/* nvc0_program.c */
+boolean nvc0_program_translate(struct nvc0_program *);
+void nvc0_program_destroy(struct nvc0_context *, struct nvc0_program *);
+
+/* nvc0_shader_state.c */
+void nvc0_vertprog_validate(struct nvc0_context *);
+void nvc0_tctlprog_validate(struct nvc0_context *);
+void nvc0_tevlprog_validate(struct nvc0_context *);
+void nvc0_gmtyprog_validate(struct nvc0_context *);
+void nvc0_fragprog_validate(struct nvc0_context *);
+
+/* nvc0_state.c */
+extern void nvc0_init_state_functions(struct nvc0_context *);
+
+/* nvc0_state_validate.c */
+extern boolean nvc0_state_validate(struct nvc0_context *);
+
+/* nvc0_surface.c */
+extern void nvc0_clear(struct pipe_context *, unsigned buffers,
+                       const float *rgba, double depth, unsigned stencil);
+extern void nvc0_init_surface_functions(struct nvc0_context *);
+
+/* nvc0_tex.c */
+void nvc0_validate_textures(struct nvc0_context *);
+void nvc0_validate_samplers(struct nvc0_context *);
+
+struct pipe_sampler_view *
+nvc0_create_sampler_view(struct pipe_context *,
+                         struct pipe_resource *,
+                         const struct pipe_sampler_view *);
+
+/* nvc0_transfer.c */
+void
+nvc0_m2mf_push_linear(struct nvc0_context *nvc0,
+		      struct nouveau_bo *dst, unsigned domain, int offset,
+		      unsigned size, void *data);
+void
+nvc0_m2mf_copy_linear(struct nvc0_context *nvc0,
+		      struct nouveau_bo *dst, unsigned dstoff, unsigned dstdom,
+		      struct nouveau_bo *src, unsigned srcoff, unsigned srcdom,
+		      unsigned size);
+
+/* nvc0_vbo.c */
+void nvc0_draw_vbo(struct pipe_context *, const struct pipe_draw_info *);
+
+void *
+nvc0_vertex_state_create(struct pipe_context *pipe,
+                         unsigned num_elements,
+                         const struct pipe_vertex_element *elements);
+void
+nvc0_vertex_state_delete(struct pipe_context *pipe, void *hwcso);
+
+void nvc0_vertex_arrays_validate(struct nvc0_context *nvc0);
+
+/* nvc0_push.c */
+void nvc0_push_vbo(struct nvc0_context *, const struct pipe_draw_info *);
+void nvc0_push_vbo2(struct nvc0_context *, const struct pipe_draw_info *);
+
+#endif
diff --git a/src/gallium/drivers/nvc0/nvc0_draw.c b/src/gallium/drivers/nvc0/nvc0_draw.c
new file mode 100644
index 00000000000..ac7e9f66a11
--- /dev/null
+++ b/src/gallium/drivers/nvc0/nvc0_draw.c
@@ -0,0 +1,88 @@
+/*
+ * Copyright 2008 Ben Skeggs
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF
+ * OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "draw/draw_pipe.h"
+
+#include "nvc0_context.h"
+
+struct nvc0_render_stage {
+   struct draw_stage stage;
+   struct nvc0_context *nvc0;
+};
+
+static INLINE struct nvc0_render_stage *
+nvc0_render_stage(struct draw_stage *stage)
+{
+   return (struct nvc0_render_stage *)stage;
+}
+
+static void
+nvc0_render_point(struct draw_stage *stage, struct prim_header *prim)
+{
+   NOUVEAU_ERR("\n");
+}
+
+static void
+nvc0_render_line(struct draw_stage *stage, struct prim_header *prim)
+{
+   NOUVEAU_ERR("\n");
+}
+
+static void
+nvc0_render_tri(struct draw_stage *stage, struct prim_header *prim)
+{
+   NOUVEAU_ERR("\n");
+}
+
+static void
+nvc0_render_flush(struct draw_stage *stage, unsigned flags)
+{
+}
+
+static void
+nvc0_render_reset_stipple_counter(struct draw_stage *stage)
+{
+   NOUVEAU_ERR("\n");
+}
+
+static void
+nvc0_render_destroy(struct draw_stage *stage)
+{
+   FREE(stage);
+}
+
+struct draw_stage *
+nvc0_draw_render_stage(struct nvc0_context *nvc0)
+{
+   struct nvc0_render_stage *rs = CALLOC_STRUCT(nvc0_render_stage);
+
+   rs->nvc0 = nvc0;
+   rs->stage.draw = nvc0->draw;
+   rs->stage.destroy = nvc0_render_destroy;
+   rs->stage.point = nvc0_render_point;
+   rs->stage.line = nvc0_render_line;
+   rs->stage.tri = nvc0_render_tri;
+   rs->stage.flush = nvc0_render_flush;
+   rs->stage.reset_stipple_counter = nvc0_render_reset_stipple_counter;
+
+   return &rs->stage;
+}
diff --git a/src/gallium/drivers/nvc0/nvc0_fence.c b/src/gallium/drivers/nvc0/nvc0_fence.c
new file mode 100644
index 00000000000..7c214ca9a75
--- /dev/null
+++ b/src/gallium/drivers/nvc0/nvc0_fence.c
@@ -0,0 +1,202 @@
+/*
+ * Copyright 2010 Christoph Bumiller
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF
+ * OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "nvc0_fence.h"
+#include "nvc0_context.h"
+#include "nvc0_screen.h"
+
+#ifdef PIPE_OS_UNIX
+#include <sched.h>
+#endif
+
+boolean
+nvc0_screen_fence_new(struct nvc0_screen *screen, struct nvc0_fence **fence,
+                      boolean emit)
+{
+   *fence = CALLOC_STRUCT(nvc0_fence);
+   if (!*fence)
+      return FALSE;
+
+   (*fence)->screen = screen;
+   (*fence)->ref = 1;
+
+   if (emit)
+      nvc0_fence_emit(*fence);
+
+   return TRUE;
+}
+
+void
+nvc0_fence_emit(struct nvc0_fence *fence)
+{
+   struct nvc0_screen *screen = fence->screen;
+   struct nouveau_channel *chan = screen->base.channel;
+
+   fence->sequence = ++screen->fence.sequence;
+
+   assert(fence->state == NVC0_FENCE_STATE_AVAILABLE);
+
+   BEGIN_RING(chan, RING_3D(QUERY_ADDRESS_HIGH), 4);
+   OUT_RELOCh(chan, screen->fence.bo, 0, NOUVEAU_BO_WR);
+   OUT_RELOCl(chan, screen->fence.bo, 0, NOUVEAU_BO_WR);
+   OUT_RING  (chan, fence->sequence);
+   OUT_RING  (chan, NVC0_3D_QUERY_GET_FENCE);
+
+   ++fence->ref;
+
+   if (screen->fence.tail)
+      screen->fence.tail->next = fence;
+   else
+      screen->fence.head = fence;
+
+   screen->fence.tail = fence;
+
+   fence->state = NVC0_FENCE_STATE_EMITTED;
+}
+
+static void
+nvc0_fence_trigger_release_buffers(struct nvc0_fence *fence);
+
+void
+nvc0_fence_del(struct nvc0_fence *fence)
+{
+   struct nvc0_fence *it;
+   struct nvc0_screen *screen = fence->screen;
+
+   if (fence->state == NVC0_FENCE_STATE_EMITTED) {
+      if (fence == screen->fence.head) {
+         screen->fence.head = fence->next;
+         if (!screen->fence.head)
+            screen->fence.tail = NULL;
+      } else {
+         for (it = screen->fence.head; it && it->next != fence; it = it->next);
+         it->next = fence->next;
+         if (screen->fence.tail == fence)
+            screen->fence.tail = it;
+      }
+   }
+
+   if (fence->buffers) {
+      debug_printf("WARNING: deleting fence with buffers "
+                   "still hooked to it !\n");
+      nvc0_fence_trigger_release_buffers(fence);
+   }
+
+   FREE(fence);
+}
+
+static void
+nvc0_fence_trigger_release_buffers(struct nvc0_fence *fence)
+{
+   struct nvc0_mm_allocation *alloc = fence->buffers;
+
+   while (alloc) {
+      struct nvc0_mm_allocation *next = alloc->next;
+      nvc0_mm_free(alloc);
+      alloc = next;
+   };
+   fence->buffers = NULL;
+}
+
+static void
+nvc0_screen_fence_update(struct nvc0_screen *screen)
+{
+   struct nvc0_fence *fence;
+   struct nvc0_fence *next = NULL;
+   uint32_t sequence = screen->fence.map[0];
+
+   if (screen->fence.sequence_ack == sequence)
+      return;
+   screen->fence.sequence_ack = sequence;
+
+   for (fence = screen->fence.head; fence; fence = next) {
+      next = fence->next;
+      sequence = fence->sequence;
+
+      fence->state = NVC0_FENCE_STATE_SIGNALLED;
+
+      if (fence->buffers)
+         nvc0_fence_trigger_release_buffers(fence);
+
+      nvc0_fence_reference(&fence, NULL);
+
+      if (sequence == screen->fence.sequence_ack)
+         break;
+   }
+   screen->fence.head = next;
+   if (!next)
+      screen->fence.tail = NULL;
+}
+
+#define NVC0_FENCE_MAX_SPINS (1 << 17)
+
+boolean
+nvc0_fence_signalled(struct nvc0_fence *fence)
+{
+   struct nvc0_screen *screen = fence->screen;
+
+   if (fence->state == NVC0_FENCE_STATE_EMITTED)
+      nvc0_screen_fence_update(screen);
+
+   return fence->state == NVC0_FENCE_STATE_SIGNALLED;
+}
+
+boolean
+nvc0_fence_wait(struct nvc0_fence *fence)
+{
+   struct nvc0_screen *screen = fence->screen;
+   int spins = 0;
+
+   if (fence->state == NVC0_FENCE_STATE_AVAILABLE) {
+      nvc0_fence_emit(fence);
+
+      FIRE_RING(screen->base.channel);
+
+      if (fence == screen->fence.current)
+         nvc0_screen_fence_new(screen, &screen->fence.current, FALSE);
+   }
+
+   do {
+      nvc0_screen_fence_update(screen);
+
+      if (fence->state == NVC0_FENCE_STATE_SIGNALLED)
+         return TRUE;
+      spins++;
+#ifdef PIPE_OS_UNIX
+      if (!(spins % 8)) /* donate a few cycles */
+         sched_yield();
+#endif
+   } while (spins < NVC0_FENCE_MAX_SPINS);
+
+   if (spins > 9000)
+      NOUVEAU_ERR("fence %x: been spinning too long\n", fence->sequence);
+
+   return FALSE;
+}
+
+void
+nvc0_screen_fence_next(struct nvc0_screen *screen)
+{
+   nvc0_fence_emit(screen->fence.current);
+   nvc0_screen_fence_new(screen, &screen->fence.current, FALSE);
+   nvc0_screen_fence_update(screen);
+}
diff --git a/src/gallium/drivers/nvc0/nvc0_fence.h b/src/gallium/drivers/nvc0/nvc0_fence.h
new file mode 100644
index 00000000000..e63c164bda4
--- /dev/null
+++ b/src/gallium/drivers/nvc0/nvc0_fence.h
@@ -0,0 +1,48 @@
+
+#ifndef __NVC0_FENCE_H__
+#define __NVC0_FENCE_H__
+
+#include "util/u_inlines.h"
+#include "util/u_double_list.h"
+
+#define NVC0_FENCE_STATE_AVAILABLE 0
+#define NVC0_FENCE_STATE_EMITTED   1
+#define NVC0_FENCE_STATE_SIGNALLED 2
+
+struct nvc0_mm_allocation;
+
+struct nvc0_fence {
+   struct nvc0_fence *next;
+   struct nvc0_screen *screen;
+   int state;
+   int ref;
+   uint32_t sequence;
+   struct nvc0_mm_allocation *buffers;
+};
+
+void nvc0_fence_emit(struct nvc0_fence *);
+void nvc0_fence_del(struct nvc0_fence *);
+
+boolean nvc0_fence_wait(struct nvc0_fence *);
+boolean nvc0_fence_signalled(struct nvc0_fence *);
+
+static INLINE void
+nvc0_fence_reference(struct nvc0_fence **ref, struct nvc0_fence *fence)
+{
+   if (*ref) {
+      if (--(*ref)->ref == 0)
+         nvc0_fence_del(*ref);
+   }
+   if (fence)
+      ++fence->ref;
+
+   *ref = fence;
+}
+
+static INLINE struct nvc0_fence *
+nvc0_fence(struct pipe_fence_handle *fence)
+{
+   return (struct nvc0_fence *)fence;
+}
+
+#endif // __NVC0_FENCE_H__
diff --git a/src/gallium/drivers/nvc0/nvc0_formats.c b/src/gallium/drivers/nvc0/nvc0_formats.c
new file mode 100644
index 00000000000..5d023573818
--- /dev/null
+++ b/src/gallium/drivers/nvc0/nvc0_formats.c
@@ -0,0 +1,462 @@
+/*
+ * Copyright 2010 Christoph Bumiller
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF
+ * OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "nvc0_screen.h"
+#include "nv50_texture.xml.h"
+#include "nvc0_3d.xml.h"
+#include "nv50_defs.xml.h"
+#include "nv50_texture.xml.h"
+#include "pipe/p_defines.h"
+
+#define A_(cr, cg, cb, ca, t0, t1, t2, t3, sz, r)     \
+   (NV50_TIC_MAP_##cr << NV50_TIC_0_MAPR__SHIFT) |    \
+   (NV50_TIC_TYPE_##t0 << NV50_TIC_0_TYPE0__SHIFT) |  \
+   (NV50_TIC_MAP_##cg << NV50_TIC_0_MAPG__SHIFT) |    \
+   (NV50_TIC_TYPE_##t1 << NV50_TIC_0_TYPE1__SHIFT) |  \
+   (NV50_TIC_MAP_##cb << NV50_TIC_0_MAPB__SHIFT) |    \
+   (NV50_TIC_TYPE_##t2 << NV50_TIC_0_TYPE2__SHIFT) |  \
+   (NV50_TIC_MAP_##ca << NV50_TIC_0_MAPA__SHIFT) |    \
+   (NV50_TIC_TYPE_##t3 << NV50_TIC_0_TYPE3__SHIFT) |  \
+   NV50_TIC_0_FMT_##sz,                               \
+   NVC0_3D_VERTEX_ATTRIB_FORMAT_SIZE_##sz |           \
+   NVC0_3D_VERTEX_ATTRIB_FORMAT_TYPE_##t0 |           \
+   (r << 31)
+
+#define B_(cr, cg, cb, ca, t0, t1, t2, t3, sz, r)     \
+   (NV50_TIC_MAP_##cr << NV50_TIC_0_MAPR__SHIFT) |    \
+   (NV50_TIC_TYPE_##t0 << NV50_TIC_0_TYPE0__SHIFT) |  \
+   (NV50_TIC_MAP_##cg << NV50_TIC_0_MAPG__SHIFT) |    \
+   (NV50_TIC_TYPE_##t1 << NV50_TIC_0_TYPE1__SHIFT) |  \
+   (NV50_TIC_MAP_##cb << NV50_TIC_0_MAPB__SHIFT) |    \
+   (NV50_TIC_TYPE_##t2 << NV50_TIC_0_TYPE2__SHIFT) |  \
+   (NV50_TIC_MAP_##ca << NV50_TIC_0_MAPA__SHIFT) |    \
+   (NV50_TIC_TYPE_##t3 << NV50_TIC_0_TYPE3__SHIFT) |  \
+   NV50_TIC_0_FMT_##sz, 0
+
+#define VERTEX_BUFFER PIPE_BIND_VERTEX_BUFFER
+#define SAMPLER_VIEW  PIPE_BIND_SAMPLER_VIEW
+#define RENDER_TARGET PIPE_BIND_RENDER_TARGET
+#define DEPTH_STENCIL PIPE_BIND_DEPTH_STENCIL
+#define SCANOUT       PIPE_BIND_SCANOUT
+
+/* for vertex buffers: */
+#define NV50_TIC_0_FMT_8_8_8    NV50_TIC_0_FMT_8_8_8_8
+#define NV50_TIC_0_FMT_16_16_16 NV50_TIC_0_FMT_16_16_16_16
+#define NV50_TIC_0_FMT_32_32_32 NV50_TIC_0_FMT_32_32_32_32
+
+const struct nvc0_format nvc0_format_table[PIPE_FORMAT_COUNT] =
+{
+   /* COMMON FORMATS */
+
+   [PIPE_FORMAT_B8G8R8A8_UNORM] = { NV50_SURFACE_FORMAT_A8R8G8B8_UNORM,
+    A_(C2, C1, C0, C3, UNORM, UNORM, UNORM, UNORM, 8_8_8_8, 1),
+    VERTEX_BUFFER | SAMPLER_VIEW | RENDER_TARGET | SCANOUT },
+
+   [PIPE_FORMAT_B8G8R8X8_UNORM] = { NV50_SURFACE_FORMAT_X8R8G8B8_UNORM,
+    A_(C2, C1, C0, ONE, UNORM, UNORM, UNORM, UNORM, 8_8_8_8, 1),
+    VERTEX_BUFFER | SAMPLER_VIEW | RENDER_TARGET | SCANOUT },
+
+   [PIPE_FORMAT_B8G8R8A8_SRGB] = { NV50_SURFACE_FORMAT_A8R8G8B8_SRGB,
+    A_(C2, C1, C0, C3, UNORM, UNORM, UNORM, UNORM, 8_8_8_8, 1),
+    VERTEX_BUFFER | SAMPLER_VIEW | RENDER_TARGET },
+
+   [PIPE_FORMAT_B8G8R8X8_SRGB] = { NV50_SURFACE_FORMAT_X8R8G8B8_SRGB,
+    A_(C2, C1, C0, ONE, UNORM, UNORM, UNORM, UNORM, 8_8_8_8, 1),
+    VERTEX_BUFFER | SAMPLER_VIEW | RENDER_TARGET },
+
+   [PIPE_FORMAT_B5G6R5_UNORM] = { NV50_SURFACE_FORMAT_R5G6B5_UNORM,
+    B_(C2, C1, C0, ONE, UNORM, UNORM, UNORM, UNORM, 5_6_5, 1),
+    SAMPLER_VIEW | RENDER_TARGET | SCANOUT },
+
+   [PIPE_FORMAT_B5G5R5A1_UNORM] = { NV50_SURFACE_FORMAT_A1R5G5B5_UNORM,
+    B_(C2, C1, C0, C3, UNORM, UNORM, UNORM, UNORM, 1_5_5_5, 1),
+    SAMPLER_VIEW | RENDER_TARGET | SCANOUT },
+
+   [PIPE_FORMAT_B4G4R4A4_UNORM] = { NV50_SURFACE_FORMAT_R16_UNORM,
+    B_(C2, C1, C0, C3, UNORM, UNORM, UNORM, UNORM, 4_4_4_4, 1),
+    SAMPLER_VIEW },
+
+   [PIPE_FORMAT_R10G10B10A2_UNORM] = { NV50_SURFACE_FORMAT_A2B10G10R10_UNORM,
+    A_(C0, C1, C2, C3, UNORM, UNORM, UNORM, UNORM, 2_10_10_10, 0),
+    SAMPLER_VIEW | RENDER_TARGET | VERTEX_BUFFER | SCANOUT },
+
+   [PIPE_FORMAT_B10G10R10A2_UNORM] = { NV50_SURFACE_FORMAT_A2R10G10B10_UNORM,
+    A_(C2, C1, C0, C3, UNORM, UNORM, UNORM, UNORM, 2_10_10_10, 1),
+    SAMPLER_VIEW | RENDER_TARGET | VERTEX_BUFFER },
+
+   /* DEPTH/STENCIL FORMATS */
+
+   [PIPE_FORMAT_Z16_UNORM] = { NV50_ZETA_FORMAT_Z16_UNORM,
+    B_(C0, C0, C0, ONE, UNORM, UINT, UINT, UINT, 16_ZETA, 0),
+    SAMPLER_VIEW | DEPTH_STENCIL },
+
+   [PIPE_FORMAT_Z24_UNORM_S8_USCALED] = { NV50_ZETA_FORMAT_S8Z24_UNORM,
+    B_(C0, C0, C0, ONE, UNORM, UINT, UINT, UINT, 8_24, 0),
+    SAMPLER_VIEW | DEPTH_STENCIL },
+
+   [PIPE_FORMAT_Z24X8_UNORM] = { NV50_ZETA_FORMAT_X8Z24_UNORM,
+    B_(C0, C0, C0, ONE, UNORM, UINT, UINT, UINT, 8_24, 0),
+    SAMPLER_VIEW | DEPTH_STENCIL },
+
+   [PIPE_FORMAT_S8_USCALED_Z24_UNORM] = { NV50_ZETA_FORMAT_S8Z24_UNORM,
+    B_(C1, C1, C1, ONE, UINT, UNORM, UINT, UINT, 24_8, 0),
+    SAMPLER_VIEW | DEPTH_STENCIL },
+
+   [PIPE_FORMAT_Z32_FLOAT] = { NV50_ZETA_FORMAT_Z32_FLOAT,
+    B_(C0, C0, C0, ONE, FLOAT, UINT, UINT, UINT, 32_ZETA, 0),
+    SAMPLER_VIEW | DEPTH_STENCIL },
+
+   [PIPE_FORMAT_Z32_FLOAT_S8X24_USCALED] = {
+    NV50_ZETA_FORMAT_Z32_FLOAT_X24S8_UNORM,
+    B_(C0, C0, C0, ONE, FLOAT, UINT, UINT, UINT, 32_8, 0),
+    SAMPLER_VIEW | DEPTH_STENCIL },
+
+   /* LUMINANCE, ALPHA, INTENSITY */
+
+   [PIPE_FORMAT_L8_UNORM] = { NV50_SURFACE_FORMAT_R8_UNORM,
+    A_(C0, C0, C0, ONE, UNORM, UNORM, UNORM, UNORM, 8, 0),
+    SAMPLER_VIEW },
+
+   [PIPE_FORMAT_L8_SRGB] = { NV50_SURFACE_FORMAT_R8_UNORM,
+    A_(C0, C0, C0, ONE, UNORM, UNORM, UNORM, UNORM, 8, 0),
+    SAMPLER_VIEW },
+
+   [PIPE_FORMAT_I8_UNORM] = { NV50_SURFACE_FORMAT_R8_UNORM,
+    A_(C0, C0, C0, C0, UNORM, UNORM, UNORM, UNORM, 8, 0),
+    SAMPLER_VIEW },
+
+   [PIPE_FORMAT_A8_UNORM] = { NV50_SURFACE_FORMAT_A8_UNORM,
+    A_(ZERO, ZERO, ZERO, C0, UNORM, UNORM, UNORM, UNORM, 8, 0),
+    SAMPLER_VIEW | RENDER_TARGET },
+
+   [PIPE_FORMAT_L8A8_UNORM] = { NV50_SURFACE_FORMAT_R16_UNORM,
+    A_(C0, C0, C0, C1, UNORM, UNORM, UNORM, UNORM, 8_8, 0),
+    SAMPLER_VIEW },
+
+   [PIPE_FORMAT_L8A8_SRGB] = { 0,
+    A_(C0, C0, C0, C1, UNORM, UNORM, UNORM, UNORM, 8_8, 0),
+    SAMPLER_VIEW },
+
+   /* DXT, RGTC */
+
+   [PIPE_FORMAT_DXT1_RGB] = { 0,
+    B_(C0, C1, C2, ONE, UNORM, UNORM, UNORM, UNORM, DXT1, 0),
+    SAMPLER_VIEW },
+
+   [PIPE_FORMAT_DXT1_RGBA] = { 0,
+    B_(C0, C1, C2, C3, UNORM, UNORM, UNORM, UNORM, DXT1, 0),
+    SAMPLER_VIEW },
+
+   [PIPE_FORMAT_DXT3_RGBA] = { 0,
+    B_(C0, C1, C2, C3, UNORM, UNORM, UNORM, UNORM, DXT3, 0),
+    SAMPLER_VIEW },
+
+   [PIPE_FORMAT_DXT5_RGBA] = { 0,
+    B_(C0, C1, C2, C3, UNORM, UNORM, UNORM, UNORM, DXT5, 0),
+    SAMPLER_VIEW },
+
+   [PIPE_FORMAT_RGTC1_UNORM] = { 0,
+    B_(C0, ZERO, ZERO, ONE, UNORM, UNORM, UNORM, UNORM, RGTC1, 0),
+    SAMPLER_VIEW },
+
+   [PIPE_FORMAT_RGTC1_SNORM] = { 0,
+    B_(C0, ZERO, ZERO, ONE, SNORM, SNORM, SNORM, SNORM, RGTC1, 0),
+    SAMPLER_VIEW },
+
+   [PIPE_FORMAT_RGTC2_UNORM] = { 0,
+    B_(C0, C1, ZERO, ONE, UNORM, UNORM, UNORM, UNORM, RGTC2, 0),
+    SAMPLER_VIEW },
+
+   [PIPE_FORMAT_RGTC2_SNORM] = { 0,
+    B_(C0, C1, ZERO, ONE, SNORM, SNORM, SNORM, SNORM, RGTC2, 0),
+    SAMPLER_VIEW },
+
+   /* FLOAT 16 */
+
+   [PIPE_FORMAT_R16G16B16A16_FLOAT] = { NV50_SURFACE_FORMAT_R16G16B16A16_FLOAT,
+    A_(C0, C1, C2, C3, FLOAT, FLOAT, FLOAT, FLOAT, 16_16_16_16, 0),
+    VERTEX_BUFFER | SAMPLER_VIEW | RENDER_TARGET },
+
+   [PIPE_FORMAT_R16G16B16_FLOAT] = { NV50_SURFACE_FORMAT_R16G16B16X16_FLOAT,
+    A_(C0, C1, C2, ONE, FLOAT, FLOAT, FLOAT, FLOAT, 16_16_16, 0),
+    VERTEX_BUFFER | SAMPLER_VIEW | RENDER_TARGET },
+
+   [PIPE_FORMAT_R16G16_FLOAT] = { NV50_SURFACE_FORMAT_R16G16_FLOAT,
+    A_(C0, C1, ZERO, ONE, FLOAT, FLOAT, FLOAT, FLOAT, 16_16, 0),
+    VERTEX_BUFFER | SAMPLER_VIEW | RENDER_TARGET },
+
+   [PIPE_FORMAT_R16_FLOAT] = { NV50_SURFACE_FORMAT_R16_FLOAT,
+    A_(C0, ZERO, ZERO, ONE, FLOAT, FLOAT, FLOAT, FLOAT, 16, 0),
+    VERTEX_BUFFER | SAMPLER_VIEW | RENDER_TARGET },
+
+   /* FLOAT 32 */
+
+   [PIPE_FORMAT_R32G32B32A32_FLOAT] = { NV50_SURFACE_FORMAT_R32G32B32A32_FLOAT,
+    A_(C0, C1, C2, C3, FLOAT, FLOAT, FLOAT, FLOAT, 32_32_32_32, 0),
+    VERTEX_BUFFER | SAMPLER_VIEW | RENDER_TARGET },
+
+   [PIPE_FORMAT_R32G32B32_FLOAT] = { NV50_SURFACE_FORMAT_R32G32B32X32_FLOAT,
+    A_(C0, C1, C2, ONE, FLOAT, FLOAT, FLOAT, FLOAT, 32_32_32, 0),
+    VERTEX_BUFFER | SAMPLER_VIEW | RENDER_TARGET },
+
+   [PIPE_FORMAT_R32G32_FLOAT] = { NV50_SURFACE_FORMAT_R32G32_FLOAT,
+    A_(C0, C1, ZERO, ONE, FLOAT, FLOAT, FLOAT, FLOAT, 32_32, 0),
+    VERTEX_BUFFER | SAMPLER_VIEW | RENDER_TARGET },
+
+   [PIPE_FORMAT_R32_FLOAT] = { NV50_SURFACE_FORMAT_R32_FLOAT,
+    A_(C0, ZERO, ZERO, ONE, FLOAT, FLOAT, FLOAT, FLOAT, 32, 0),
+    VERTEX_BUFFER | SAMPLER_VIEW | RENDER_TARGET },
+
+   /* ODD FORMATS */
+
+   [PIPE_FORMAT_R11G11B10_FLOAT] = { NV50_SURFACE_FORMAT_B10G11R11_FLOAT,
+    B_(C0, C1, C2, ONE, FLOAT, FLOAT, FLOAT, FLOAT, 10_11_11, 0),
+    SAMPLER_VIEW | RENDER_TARGET },
+
+   [PIPE_FORMAT_R9G9B9E5_FLOAT] = { 0,
+    B_(C0, C1, C2, ONE, FLOAT, FLOAT, FLOAT, FLOAT, E5_9_9_9, 0),
+    SAMPLER_VIEW },
+
+   /* SNORM 32 */
+
+   [PIPE_FORMAT_R32G32B32A32_SNORM] = { 0,
+    A_(C0, C1, C2, C3, FLOAT, FLOAT, FLOAT, FLOAT, 32_32_32_32, 0),
+    VERTEX_BUFFER | SAMPLER_VIEW },
+
+   [PIPE_FORMAT_R32G32B32_SNORM] = { 0,
+    A_(C0, C1, C2, ONE, SNORM, SNORM, SNORM, SNORM, 32_32_32, 0),
+    VERTEX_BUFFER | SAMPLER_VIEW },
+
+   [PIPE_FORMAT_R32G32_SNORM] = { 0,
+    A_(C0, C1, ZERO, ONE, SNORM, SNORM, SNORM, SNORM, 32_32, 0),
+    VERTEX_BUFFER | SAMPLER_VIEW },
+
+   [PIPE_FORMAT_R32_SNORM] = { 0,
+    A_(C0, ZERO, ZERO, ONE, SNORM, SNORM, SNORM, SNORM, 32, 0),
+    VERTEX_BUFFER | SAMPLER_VIEW },
+
+   /* UNORM 32 */
+
+   [PIPE_FORMAT_R32G32B32A32_UNORM] = { 0,
+    A_(C0, C1, C2, C3, FLOAT, FLOAT, FLOAT, FLOAT, 32_32_32_32, 0),
+    VERTEX_BUFFER | SAMPLER_VIEW },
+
+   [PIPE_FORMAT_R32G32B32_UNORM] = { 0,
+    A_(C0, C1, C2, ONE, UNORM, UNORM, UNORM, UNORM, 32_32_32, 0),
+    VERTEX_BUFFER | SAMPLER_VIEW },
+
+   [PIPE_FORMAT_R32G32_UNORM] = { 0,
+    A_(C0, C1, ZERO, ONE, UNORM, UNORM, UNORM, UNORM, 32_32, 0),
+    VERTEX_BUFFER | SAMPLER_VIEW },
+
+   [PIPE_FORMAT_R32_UNORM] = { 0,
+    A_(C0, ZERO, ZERO, ONE, UNORM, UNORM, UNORM, UNORM, 32, 0),
+    VERTEX_BUFFER | SAMPLER_VIEW },
+
+   /* SNORM 16 */
+
+   [PIPE_FORMAT_R16G16B16A16_SNORM] = { NV50_SURFACE_FORMAT_R16G16B16A16_SNORM,
+    A_(C0, C1, C2, C3, SNORM, SNORM, SNORM, SNORM, 16_16_16_16, 0),
+    VERTEX_BUFFER | SAMPLER_VIEW | RENDER_TARGET },
+
+   [PIPE_FORMAT_R16G16B16_SNORM] = { 0,
+    A_(C0, C1, C2, ONE, SNORM, SNORM, SNORM, SNORM, 16_16_16, 0),
+    VERTEX_BUFFER | SAMPLER_VIEW },
+
+   [PIPE_FORMAT_R16G16_SNORM] = { NV50_SURFACE_FORMAT_R16G16_SNORM,
+    A_(C0, C1, C2, C3, SNORM, SNORM, SNORM, SNORM, 16_16, 0),
+    VERTEX_BUFFER | SAMPLER_VIEW | RENDER_TARGET },
+
+   [PIPE_FORMAT_R16_SNORM] = { NV50_SURFACE_FORMAT_R16_SNORM,
+    A_(C0, ZERO, ZERO, ONE, SNORM, SNORM, SNORM, SNORM, 16, 0),
+    VERTEX_BUFFER | SAMPLER_VIEW | RENDER_TARGET },
+
+   /* UNORM 16 */
+
+   [PIPE_FORMAT_R16G16B16A16_UNORM] = { NV50_SURFACE_FORMAT_R16G16B16A16_UNORM,
+    A_(C0, C1, C2, C3, UNORM, UNORM, UNORM, UNORM, 16_16_16_16, 0),
+    VERTEX_BUFFER | SAMPLER_VIEW | RENDER_TARGET },
+
+   [PIPE_FORMAT_R16G16B16_UNORM] = { 0,
+    A_(C0, C1, C2, ONE, UNORM, UNORM, UNORM, UNORM, 16_16_16, 0),
+    VERTEX_BUFFER | SAMPLER_VIEW },
+
+   [PIPE_FORMAT_R16G16_UNORM] = { NV50_SURFACE_FORMAT_R16G16_UNORM,
+    A_(C0, C1, C2, C3, UNORM, UNORM, UNORM, UNORM, 16_16, 0),
+    VERTEX_BUFFER | SAMPLER_VIEW | RENDER_TARGET },
+
+   [PIPE_FORMAT_R16_UNORM] = { NV50_SURFACE_FORMAT_R16_UNORM,
+    A_(C0, ZERO, ZERO, ONE, UNORM, UNORM, UNORM, UNORM, 16, 0),
+    VERTEX_BUFFER | SAMPLER_VIEW | RENDER_TARGET },
+
+   /* SNORM 8 */
+
+   [PIPE_FORMAT_R8G8B8A8_SNORM] = { NV50_SURFACE_FORMAT_A8B8G8R8_SNORM,
+    A_(C0, C1, C2, C3, SNORM, SNORM, SNORM, SNORM, 8_8_8_8, 0),
+    VERTEX_BUFFER | SAMPLER_VIEW | RENDER_TARGET },
+
+   [PIPE_FORMAT_R8G8B8_SNORM] = { 0,
+    A_(C0, C1, C2, ONE, SNORM, SNORM, SNORM, SNORM, 8_8_8, 0),
+    VERTEX_BUFFER | SAMPLER_VIEW },
+
+   [PIPE_FORMAT_R8G8_SNORM] = { NV50_SURFACE_FORMAT_R8G8_SNORM,
+    A_(C0, C1, ZERO, ONE, SNORM, SNORM, SNORM, SNORM, 8_8, 0),
+    VERTEX_BUFFER | SAMPLER_VIEW | RENDER_TARGET },
+
+   [PIPE_FORMAT_R8_SNORM] = { NV50_SURFACE_FORMAT_R8_SNORM,
+    A_(C0, ZERO, ZERO, ONE, SNORM, SNORM, SNORM, SNORM, 8, 0),
+    VERTEX_BUFFER | SAMPLER_VIEW | RENDER_TARGET },
+
+   /* UNORM 8 */
+
+   [PIPE_FORMAT_R8G8B8A8_UNORM] = { NV50_SURFACE_FORMAT_A8B8G8R8_UNORM,
+    A_(C0, C1, C2, C3, UNORM, UNORM, UNORM, UNORM, 8_8_8_8, 0),
+    VERTEX_BUFFER | SAMPLER_VIEW | RENDER_TARGET },
+
+   [PIPE_FORMAT_R8G8B8A8_SRGB] = { NV50_SURFACE_FORMAT_A8B8G8R8_SRGB,
+    A_(C0, C1, C2, C3, UNORM, UNORM, UNORM, UNORM, 8_8_8_8, 0),
+    SAMPLER_VIEW | RENDER_TARGET },
+
+   [PIPE_FORMAT_R8G8B8_UNORM] = { NV50_SURFACE_FORMAT_X8B8G8R8_UNORM,
+    A_(C0, C1, C2, ONE, UNORM, UNORM, UNORM, UNORM, 8_8_8, 0),
+    VERTEX_BUFFER | SAMPLER_VIEW | RENDER_TARGET },
+
+   [PIPE_FORMAT_R8G8B8_SRGB] = { NV50_SURFACE_FORMAT_X8B8G8R8_SRGB,
+    A_(C0, C1, C2, ONE, UNORM, UNORM, UNORM, UNORM, 8_8_8, 0),
+    SAMPLER_VIEW | RENDER_TARGET },
+
+   [PIPE_FORMAT_R8G8_UNORM] = { NV50_SURFACE_FORMAT_R8G8_UNORM,
+    A_(C0, C1, ZERO, ONE, UNORM, UNORM, UNORM, UNORM, 8_8, 0),
+    VERTEX_BUFFER | SAMPLER_VIEW | RENDER_TARGET },
+
+   [PIPE_FORMAT_R8_UNORM] = { NV50_SURFACE_FORMAT_R8_UNORM,
+    A_(C0, ZERO, ZERO, ONE, UNORM, UNORM, UNORM, UNORM, 8, 0),
+    VERTEX_BUFFER | SAMPLER_VIEW | RENDER_TARGET },
+
+   /* SSCALED 32 */
+
+   [PIPE_FORMAT_R32G32B32A32_SSCALED] = { NV50_SURFACE_FORMAT_R32G32B32A32_SINT,
+    A_(C0, C1, C2, C3, SSCALED, SSCALED, SSCALED, SSCALED, 32_32_32_32, 0),
+    VERTEX_BUFFER | SAMPLER_VIEW },
+
+   [PIPE_FORMAT_R32G32B32_SSCALED] = { 0,
+    A_(C0, C1, C2, ONE, SSCALED, SSCALED, SSCALED, SSCALED, 32_32_32, 0),
+    VERTEX_BUFFER | SAMPLER_VIEW },
+
+   [PIPE_FORMAT_R32G32_SSCALED] = { NV50_SURFACE_FORMAT_R32G32_SINT,
+    A_(C0, C1, ZERO, ONE, SSCALED, SSCALED, SSCALED, SSCALED, 32_32, 0),
+    VERTEX_BUFFER | SAMPLER_VIEW },
+
+   [PIPE_FORMAT_R32_SSCALED] = { 0,
+    A_(C0, ZERO, ZERO, ONE, SSCALED, SSCALED, SSCALED, SSCALED, 32, 0),
+    VERTEX_BUFFER | SAMPLER_VIEW },
+
+   /* USCALED 32 */
+
+   [PIPE_FORMAT_R32G32B32A32_USCALED] = { NV50_SURFACE_FORMAT_R32G32B32A32_UINT,
+    A_(C0, C1, C2, C3, USCALED, USCALED, USCALED, USCALED, 32_32_32_32, 0),
+    VERTEX_BUFFER | SAMPLER_VIEW },
+
+   [PIPE_FORMAT_R32G32B32_USCALED] = { 0,
+    A_(C0, C1, C2, ONE, USCALED, USCALED, USCALED, USCALED, 32_32_32, 0),
+    VERTEX_BUFFER | SAMPLER_VIEW },
+
+   [PIPE_FORMAT_R32G32_USCALED] = { NV50_SURFACE_FORMAT_R32G32_UINT,
+    A_(C0, C1, ZERO, ONE, USCALED, USCALED, USCALED, USCALED, 32_32, 0),
+    VERTEX_BUFFER | SAMPLER_VIEW },
+
+   [PIPE_FORMAT_R32_USCALED] = { 0,
+    A_(C0, ZERO, ZERO, ONE, USCALED, USCALED, USCALED, USCALED, 32, 0),
+    VERTEX_BUFFER | SAMPLER_VIEW },
+
+   /* SSCALED 16 */
+
+   [PIPE_FORMAT_R16G16B16A16_SSCALED] = { NV50_SURFACE_FORMAT_R16G16B16A16_SINT,
+    A_(C0, C1, C2, C3, SSCALED, SSCALED, SSCALED, SSCALED, 16_16_16_16, 0),
+    VERTEX_BUFFER | SAMPLER_VIEW },
+
+   [PIPE_FORMAT_R16G16B16_SSCALED] = { 0,
+    A_(C0, C1, C2, ONE, SSCALED, SSCALED, SSCALED, SSCALED, 16_16_16, 0),
+    VERTEX_BUFFER | SAMPLER_VIEW },
+
+   [PIPE_FORMAT_R16G16_SSCALED] = { NV50_SURFACE_FORMAT_R16G16_SINT,
+    A_(C0, C1, ZERO, ONE, SSCALED, SSCALED, SSCALED, SSCALED, 16_16, 0),
+    VERTEX_BUFFER | SAMPLER_VIEW },
+
+   [PIPE_FORMAT_R16_SSCALED] = { NV50_SURFACE_FORMAT_R16_SINT,
+    A_(C0, ZERO, ZERO, ONE, SSCALED, SSCALED, SSCALED, SSCALED, 16, 0),
+    VERTEX_BUFFER | SAMPLER_VIEW },
+
+   /* USCALED 16 */
+
+   [PIPE_FORMAT_R16G16B16A16_USCALED] = { NV50_SURFACE_FORMAT_R16G16B16A16_UINT,
+    A_(C0, C1, C2, C3, USCALED, USCALED, USCALED, USCALED, 16_16_16_16, 0),
+    VERTEX_BUFFER | SAMPLER_VIEW },
+
+   [PIPE_FORMAT_R16G16B16_USCALED] = { 0,
+    A_(C0, C1, C2, ONE, USCALED, USCALED, USCALED, USCALED, 16_16_16, 0),
+    VERTEX_BUFFER | SAMPLER_VIEW },
+
+   [PIPE_FORMAT_R16G16_USCALED] = { NV50_SURFACE_FORMAT_R16G16_UINT,
+    A_(C0, C1, ZERO, ONE, USCALED, USCALED, USCALED, USCALED, 16_16, 0),
+    VERTEX_BUFFER | SAMPLER_VIEW },
+
+   [PIPE_FORMAT_R16_USCALED] = { NV50_SURFACE_FORMAT_R16_UINT,
+    A_(C0, ZERO, ZERO, ONE, USCALED, USCALED, USCALED, USCALED, 16, 0),
+    VERTEX_BUFFER | SAMPLER_VIEW },
+
+   /* SSCALED 8 */
+
+   [PIPE_FORMAT_R8G8B8A8_SSCALED] = { NV50_SURFACE_FORMAT_A8B8G8R8_SINT,
+    A_(C0, C1, C2, C3, SSCALED, SSCALED, SSCALED, SSCALED, 8_8_8_8, 0),
+    VERTEX_BUFFER | SAMPLER_VIEW },
+
+   [PIPE_FORMAT_R8G8B8_SSCALED] = { 0,
+    A_(C0, C1, C2, ONE, SSCALED, SSCALED, SSCALED, SSCALED, 8_8_8, 0),
+    VERTEX_BUFFER | SAMPLER_VIEW },
+
+   [PIPE_FORMAT_R8G8_SSCALED] = { NV50_SURFACE_FORMAT_R8G8_SINT,
+    A_(C0, C1, ZERO, ONE, SSCALED, SSCALED, SSCALED, SSCALED, 8_8, 0),
+    VERTEX_BUFFER | SAMPLER_VIEW },
+
+   [PIPE_FORMAT_R8_SSCALED] = { NV50_SURFACE_FORMAT_R8_SINT,
+    A_(C0, ZERO, ZERO, ONE, SSCALED, SSCALED, SSCALED, SSCALED, 8, 0),
+    VERTEX_BUFFER | SAMPLER_VIEW },
+
+   /* USCALED 8 */
+
+   [PIPE_FORMAT_R8G8B8A8_USCALED] = { NV50_SURFACE_FORMAT_A8B8G8R8_UINT,
+    A_(C0, C1, C2, C3, USCALED, USCALED, USCALED, USCALED, 8_8_8_8, 0),
+    VERTEX_BUFFER | SAMPLER_VIEW },
+
+   [PIPE_FORMAT_R8G8B8_USCALED] = { 0,
+    A_(C0, C1, C2, ONE, USCALED, USCALED, USCALED, USCALED, 8_8_8, 0),
+    VERTEX_BUFFER | SAMPLER_VIEW },
+
+   [PIPE_FORMAT_R8G8_USCALED] = { NV50_SURFACE_FORMAT_R8G8_UINT,
+    A_(C0, C1, ZERO, ONE, USCALED, USCALED, USCALED, USCALED, 8_8, 0),
+    VERTEX_BUFFER | SAMPLER_VIEW },
+
+   [PIPE_FORMAT_R8_USCALED] = { NV50_SURFACE_FORMAT_R8_UINT,
+    A_(C0, ZERO, ZERO, ONE, USCALED, USCALED, USCALED, USCALED, 8, 0),
+    VERTEX_BUFFER | SAMPLER_VIEW },
+};
diff --git a/src/gallium/drivers/nvc0/nvc0_graph_macros.h b/src/gallium/drivers/nvc0/nvc0_graph_macros.h
new file mode 100644
index 00000000000..8da963a4c5b
--- /dev/null
+++ b/src/gallium/drivers/nvc0/nvc0_graph_macros.h
@@ -0,0 +1,235 @@
+
+#ifndef __NVC0_PGRAPH_MACROS_H__
+#define __NVC0_PGRAPH_MACROS_H__
+
+/* extrinsrt r1, r2, src, size, dst: replace bits [dst:dst+size) in r1
+ *  with bits [src:src+size) in r2
+ *
+ * bra(n)z annul: no delay slot
+ */
+
+/* The comments above the macros describe what they *should* be doing,
+ * but we use less functionality for now.
+ */
+
+/*
+ * for (i = 0; i < 8; ++i)
+ *    [NVC0_3D_BLEND_ENABLE(i)] = BIT(i of arg);
+ *
+ * [3428] = arg;
+ *
+ * if (arg == 0 || [NVC0_3D_MULTISAMPLE_ENABLE] == 0)
+ *    [0d9c] = 0;
+ * else
+ *    [0d9c] = [342c];
+ */
+static const uint32_t nvc0_9097_blend_enables[] =
+{
+   0x05360021, /* 0x00: maddr [NVC0_3D_BLEND_ENABLE(0), increment = 4] */
+   0x00404042, /* 0x01: send extrinsrt 0 $r1 0 0x1 0 */
+   0x00424042, /* 0x02: send extrinsrt 0 $r1 0x1 0x1 0 */
+   0x00444042, /* 0x03: send extrinsrt 0 $r1 0x2 0x1 0 */
+   0x00464042, /* 0x04: send extrinsrt 0 $r1 0x3 0x1 0 */
+   0x00484042, /* 0x05: send extrinsrt 0 $r1 0x4 0x1 0 */
+   0x004a4042, /* 0x06: send extrinsrt 0 $r1 0x5 0x1 0 */
+   0x004c40c2, /* 0x07: exit send extrinsrt 0 $r1 0x6 0x1 0 */
+   0x004e4042, /* 0x08: send extrinsrt 0 $r1 0x7 0x1 0 */
+};
+
+/*
+ * uint64 limit = (parm(0) << 32) | parm(1);
+ * uint64 start = (parm(2) << 32);
+ *
+ * if (limit) {
+ *    start |= parm(3);
+ *    --limit;
+ * } else {
+ *    start |= 1;
+ * }
+ *
+ * [0x1c04 + (arg & 0xf) * 16 + 0] = (start >> 32) & 0xff;
+ * [0x1c04 + (arg & 0xf) * 16 + 4] = start & 0xffffffff;
+ * [0x1f00 + (arg & 0xf) * 8 + 0] = (limit >> 32) & 0xff;
+ * [0x1f00 + (arg & 0xf) * 8 + 4] = limit & 0xffffffff;
+ */
+static const uint32_t nvc0_9097_vertex_array_select[] =
+{
+   0x00000201, /* 0x00: parm $r2 */
+   0x00000301, /* 0x01: parm $r3 */
+   0x00000401, /* 0x02: parm $r4 */
+   0x00000501, /* 0x03: parm $r5 */
+   0x11004612, /* 0x04: mov $r6 extrinsrt 0 $r1 0 4 2 */
+   0x09004712, /* 0x05: mov $r7 extrinsrt 0 $r1 0 4 1 */
+   0x05c07621, /* 0x06: maddr $r6 add $6 0x1701 */
+   0x00002041, /* 0x07: send $r4 */
+   0x00002841, /* 0x08: send $r5 */
+   0x05f03f21, /* 0x09: maddr $r7 add $7 0x17c0 */
+   0x000010c1, /* 0x0a: exit send $r2 */
+   0x00001841, /* 0x0b: send $r3 */
+};
+
+static const uint32_t nvc0_9097_color_mask_brdc[] =
+{
+   0x05a00021, /* maddr [NVC0_3D_COLOR_MASK(0), increment = 4] */
+   0x00000841, /* send $r1 */
+   0x00000841, /* send $r1 */
+   0x00000841, /* send $r1 */
+   0x00000841, /* send $r1 */
+   0x00000841, /* send $r1 */
+   0x00000841, /* send $r1 */
+   0x000008c1, /* exit send $r1 */
+   0x00000841, /* send $r1 */
+};
+
+/*
+ * [GL_POLYGON_MODE_FRONT] = arg;
+ *
+ * if (BIT(31 of [0x3410]))
+ *    [1a24] = 0x7353;
+ *
+ * if ([NVC0_3D_SP_SELECT(3)] == 0x31 || [NVC0_3D_SP_SELECT(4)] == 0x41)
+ *    [02ec] = 0;
+ * else
+ * if ([GL_POLYGON_MODE_BACK] == GL_LINE || arg == GL_LINE)
+ *    [02ec] = BYTE(1 of [0x3410]) << 4;
+ * else
+ *    [02ec] = BYTE(0 of [0x3410]) << 4;
+ */
+static const uint32_t nvc0_9097_poly_mode_front[] =
+{
+   0x00db0215, /* 0x00: read $r2 [NVC0_3D_POLYGON_MODE_BACK] */
+   0x020c0315, /* 0x01: read $r3 [NVC0_3D_SP_SELECT(3)] */
+   0x00128f10, /* 0x02: mov $r7 or $r1 $r2 */
+   0x02100415, /* 0x03: read $r4 [NVC0_3D_SP_SELECT(4)] */
+   0x00004211, /* 0x04: mov $r2 0x1 */
+   0x00180611, /* 0x05: mov $r6 0x60 */
+   0x0014bf10, /* 0x06: mov $r7 and $r7 $r2 */
+   0x0000f807, /* 0x07: braz $r7 0xa */
+   0x00dac021, /* 0x08: maddr 0x36b */
+   0x00800611, /* 0x09: mov $r6 0x200 */
+   0x00131f10, /* 0x0a: mov $r7 or $r3 $r4 */
+   0x0014bf10, /* 0x0b: mov $r7 and $r7 $r2 */
+   0x0000f807, /* 0x0c: braz $r7 0xf */
+   0x00000841, /* 0x0d: send $r1 */
+   0x00000611, /* 0x0e: mov $r6 0 */
+   0x002ec0a1, /* 0x0f: exit maddr [02ec] */
+   0x00003041  /* 0x10: send $r6 */
+};
+
+/*
+ * [GL_POLYGON_MODE_BACK] = arg;
+ *
+ * if (BIT(31 of [0x3410]))
+ *    [1a24] = 0x7353;
+ *
+ * if ([NVC0_3D_SP_SELECT(3)] == 0x31 || [NVC0_3D_SP_SELECT(4)] == 0x41)
+ *    [02ec] = 0;
+ * else
+ * if ([GL_POLYGON_MODE_FRONT] == GL_LINE || arg == GL_LINE)
+ *    [02ec] = BYTE(1 of [0x3410]) << 4;
+ * else
+ *    [02ec] = BYTE(0 of [0x3410]) << 4;
+ */
+/* NOTE: 0x3410 = 0x80002006 by default,
+ *  POLYGON_MODE == GL_LINE check replaced by (MODE & 1)
+ *  SP_SELECT(i) == (i << 4) | 1 check replaced by SP_SELECT(i) & 1
+ */
+static const uint32_t nvc0_9097_poly_mode_back[] =
+{
+   0x00dac215, /* 0x00: read $r2 [NVC0_3D_POLYGON_MODE_FRONT] */
+   0x020c0315, /* 0x01: read $r3 [NVC0_3D_SP_SELECT(3)] */
+   0x00128f10, /* 0x02: mov $r7 or $r1 $r2 */
+   0x02100415, /* 0x03: read $r4 [NVC0_3D_SP_SELECT(4)] */
+   0x00004211, /* 0x04: mov $r2 0x1 */
+   0x00180611, /* 0x05: mov $r6 0x60 */
+   0x0014bf10, /* 0x06: mov $r7 and $r7 $r2 */
+   0x0000f807, /* 0x07: braz $r7 0xa */
+   0x00dac021, /* 0x08: maddr 0x36b */
+   0x00800611, /* 0x09: mov $r6 0x200 */
+   0x00131f10, /* 0x0a: mov $r7 or $r3 $r4 */
+   0x0014bf10, /* 0x0b: mov $r7 and $r7 $r2 */
+   0x0000f807, /* 0x0c: braz $r7 0xf */
+   0x00000841, /* 0x0d: send $r1 */
+   0x00000611, /* 0x0e: mov $r6 0 */
+   0x002ec0a1, /* 0x0f: exit maddr [02ec] */
+   0x00003041  /* 0x10: send $r6 */
+};
+
+/*
+ * [NVC0_3D_SP_SELECT(4)] = arg
+ *
+ * if BIT(31 of [0x3410]) == 0
+ *    [1a24] = 0x7353;
+ *
+ * if ([NVC0_3D_SP_SELECT(3)] == 0x31 || arg == 0x41)
+ *    [02ec] = 0
+ * else
+ * if (any POLYGON MODE == LINE)
+ *    [02ec] = BYTE(1 of [3410]) << 4;
+ * else
+ *    [02ec] = BYTE(0 of [3410]) << 4; // 02ec valid bits are 0xff1
+ */
+static const uint32_t nvc0_9097_gp_select[] = /* 0x0f */
+{
+   0x00dac215, /* 0x00: read $r2 0x36b */
+   0x00db0315, /* 0x01: read $r3 0x36c */
+   0x0012d710, /* 0x02: mov $r7 or $r2 $r3 */
+   0x020c0415, /* 0x03: read $r4 0x830 */
+   0x00004211, /* 0x04: mov $r2 0x1 */
+   0x00180611, /* 0x05: mov $r6 0x60 */
+   0x0014bf10, /* 0x06: mov $r7 and $r7 $r2 */
+   0x0000f807, /* 0x07: braz $r7 0xa */
+   0x02100021, /* 0x08: maddr 0x840 */
+   0x00800611, /* 0x09: mov $r6 0x200 */
+   0x00130f10, /* 0x0a: mov $r7 or $r1 $r4 */
+   0x0014bf10, /* 0x0b: mov $r7 and $r7 $r2 */
+   0x0000f807, /* 0x0c: braz $r7 0xf */
+   0x00000841, /* 0x0d: send $r1 */
+   0x00000611, /* 0x0e: mov $r6 0 */
+   0x002ec0a1, /* 0x0f: exit maddr 0xbb */
+   0x00003041, /* 0x10: send $r6 */
+};
+
+/*
+ * [NVC0_3D_SP_SELECT(3)] = arg
+ *
+ * if BIT(31 of [0x3410]) == 0
+ *    [1a24] = 0x7353;
+ *
+ * if (arg == 0x31) {
+ *    if (BIT(2 of [0x3430])) {
+ *       int i = 15; do { --i; } while(i);
+ *       [0x1a2c] = 0;
+ *    }
+ * }
+ *
+ * if ([NVC0_3D_SP_SELECT(4)] == 0x41 || arg == 0x31)
+ *    [02ec] = 0
+ * else
+ * if ([any POLYGON_MODE] == GL_LINE)
+ *    [02ec] = BYTE(1 of [3410]) << 4;
+ * else
+ *    [02ec] = BYTE(0 of [3410]) << 4;
+ */
+static const uint32_t nvc0_9097_tep_select[] = /* 0x10 */
+{
+   0x00dac215, /* 0x00: read $r2 0x36b */
+   0x00db0315, /* 0x01: read $r3 0x36c */
+   0x0012d710, /* 0x02: mov $r7 or $r2 $r3 */
+   0x02100415, /* 0x03: read $r4 0x840 */
+   0x00004211, /* 0x04: mov $r2 0x1 */
+   0x00180611, /* 0x05: mov $r6 0x60 */
+   0x0014bf10, /* 0x06: mov $r7 and $r7 $r2 */
+   0x0000f807, /* 0x07: braz $r7 0xa */
+   0x020c0021, /* 0x08: maddr 0x830 */
+   0x00800611, /* 0x09: mov $r6 0x200 */
+   0x00130f10, /* 0x0a: mov $r7 or $r1 $r4 */
+   0x0014bf10, /* 0x0b: mov $r7 and $r7 $r2 */
+   0x0000f807, /* 0x0c: braz $r7 0xf */
+   0x00000841, /* 0x0d: send $r1 */
+   0x00000611, /* 0x0e: mov $r6 0 */
+   0x002ec0a1, /* 0x0f: exit maddr 0xbb */
+   0x00003041, /* 0x10: send $r6 */
+};
+
+#endif
diff --git a/src/gallium/drivers/nvc0/nvc0_m2mf.xml.h b/src/gallium/drivers/nvc0/nvc0_m2mf.xml.h
new file mode 100644
index 00000000000..3bf628d425e
--- /dev/null
+++ b/src/gallium/drivers/nvc0/nvc0_m2mf.xml.h
@@ -0,0 +1,138 @@
+#ifndef NVC0_M2MF_XML
+#define NVC0_M2MF_XML
+
+/* Autogenerated file, DO NOT EDIT manually!
+
+This file was generated by the rules-ng-ng headergen tool in this git repository:
+http://0x04.net/cgit/index.cgi/rules-ng-ng
+git clone git://0x04.net/rules-ng-ng
+
+The rules-ng-ng source files this header was generated from are:
+- nvc0_m2mf.xml  (   2227 bytes, from 2010-10-16 16:10:29)
+- copyright.xml  (   6498 bytes, from 2010-10-03 13:18:37)
+- nv_object.xml  (  11379 bytes, from 2010-10-16 11:43:24)
+- nvchipsets.xml (   2907 bytes, from 2010-10-15 16:28:21)
+- nv_defs.xml    (   4437 bytes, from 2010-07-06 07:43:58)
+
+Copyright (C) 2006-2010 by the following authors:
+- Artur Huillet <[email protected]> (ahuillet)
+- Ben Skeggs (darktama, darktama_)
+- B. R. <[email protected]> (koala_br)
+- Carlos Martin <[email protected]> (carlosmn)
+- Christoph Bumiller <[email protected]> (calim, chrisbmr)
+- Dawid Gajownik <[email protected]> (gajownik)
+- Dmitry Baryshkov
+- Dmitry Eremin-Solenikov <[email protected]> (lumag)
+- EdB <[email protected]> (edb_)
+- Erik Waling <[email protected]> (erikwaling)
+- Francisco Jerez <[email protected]> (curro, curro_, currojerez)
+- imirkin <[email protected]> (imirkin)
+- jb17bsome <[email protected]> (jb17bsome)
+- Jeremy Kolb <[email protected]> (kjeremy)
+- Laurent Carlier <[email protected]> (lordheavy)
+- Luca Barbieri <[email protected]> (lb, lb1)
+- Maarten Maathuis <[email protected]> (stillunknown)
+- Marcin Kościelnicki <[email protected]> (mwk, koriakin)
+- Mark Carey <[email protected]> (careym)
+- Matthieu Castet <[email protected]> (mat-c)
+- nvidiaman <[email protected]> (nvidiaman)
+- Patrice Mandin <[email protected]> (pmandin, pmdata)
+- Pekka Paalanen <[email protected]> (pq, ppaalanen)
+- Peter Popov <[email protected]> (ironpeter)
+- Richard Hughes <[email protected]> (hughsient)
+- Rudi Cilibrasi <[email protected]> (cilibrar)
+- Serge Martin
+- Simon Raffeiner
+- Stephane Loeuillet <[email protected]> (leroutier)
+- Stephane Marchesin <[email protected]> (marcheu)
+- sturmflut <[email protected]> (sturmflut)
+- Sylvain Munaut <[email protected]>
+- Victor Stinner <[email protected]> (haypo)
+- Wladmir van der Laan <[email protected]> (miathan6)
+- Younes Manton <[email protected]> (ymanton)
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice (including the
+next paragraph) shall be included in all copies or substantial
+portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/
+
+
+
+#define NVC0_M2MF_TILING_MODE_IN				0x00000204
+
+#define NVC0_M2MF_TILING_PITCH_IN				0x00000208
+
+#define NVC0_M2MF_TILING_HEIGHT_IN				0x0000020c
+
+#define NVC0_M2MF_TILING_DEPTH_IN				0x00000210
+
+#define NVC0_M2MF_TILING_POSITION_IN_Z				0x00000214
+
+#define NVC0_M2MF_TILING_MODE_OUT				0x00000220
+
+#define NVC0_M2MF_TILING_PITCH_OUT				0x00000224
+
+#define NVC0_M2MF_TILING_HEIGHT_OUT				0x00000228
+
+#define NVC0_M2MF_TILING_DEPTH_OUT				0x0000022c
+
+#define NVC0_M2MF_TILING_POSITION_OUT_Z				0x00000230
+
+#define NVC0_M2MF_OFFSET_OUT_HIGH				0x00000238
+
+#define NVC0_M2MF_OFFSET_OUT_LOW				0x0000023c
+
+#define NVC0_M2MF_EXEC						0x00000300
+#define NVC0_M2MF_EXEC_PUSH					0x00000001
+#define NVC0_M2MF_EXEC_LINEAR_IN				0x00000010
+#define NVC0_M2MF_EXEC_LINEAR_OUT				0x00000100
+#define NVC0_M2MF_EXEC_NOTIFY					0x00002000
+#define NVC0_M2MF_EXEC_INC__MASK				0x00f00000
+#define NVC0_M2MF_EXEC_INC__SHIFT				20
+
+#define NVC0_M2MF_DATA						0x00000304
+
+#define NVC0_M2MF_OFFSET_IN_HIGH				0x0000030c
+
+#define NVC0_M2MF_OFFSET_IN_LOW					0x00000310
+
+#define NVC0_M2MF_PITCH_IN					0x00000314
+
+#define NVC0_M2MF_PITCH_OUT					0x00000318
+
+#define NVC0_M2MF_LINE_LENGTH_IN				0x0000031c
+
+#define NVC0_M2MF_LINE_COUNT					0x00000320
+
+#define NVC0_M2MF_NOTIFY_ADDRESS_HIGH				0x0000032c
+
+#define NVC0_M2MF_NOTIFY_ADDRESS_LOW				0x00000330
+
+#define NVC0_M2MF_NOTIFY					0x00000334
+
+#define NVC0_M2MF_TILING_POSITION_IN_X				0x00000344
+
+#define NVC0_M2MF_TILING_POSITION_IN_Y				0x00000348
+
+#define NVC0_M2MF_TILING_POSITION_OUT_X				0x0000034c
+
+#define NVC0_M2MF_TILING_POSITION_OUT_Y				0x00000350
+
+
+#endif /* NVC0_M2MF_XML */
diff --git a/src/gallium/drivers/nvc0/nvc0_miptree.c b/src/gallium/drivers/nvc0/nvc0_miptree.c
new file mode 100644
index 00000000000..7c7e134146e
--- /dev/null
+++ b/src/gallium/drivers/nvc0/nvc0_miptree.c
@@ -0,0 +1,327 @@
+/*
+ * Copyright 2008 Ben Skeggs
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF
+ * OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "pipe/p_state.h"
+#include "pipe/p_defines.h"
+#include "util/u_inlines.h"
+#include "util/u_format.h"
+
+#include "nvc0_context.h"
+#include "nvc0_resource.h"
+#include "nvc0_transfer.h"
+
+static INLINE uint32_t
+get_tile_dims(unsigned nx, unsigned ny, unsigned nz)
+{
+   uint32_t tile_mode = 0x000;
+
+   if (ny > 64) tile_mode = 0x040; /* height 128 tiles */
+   else
+   if (ny > 32) tile_mode = 0x030; /* height 64 tiles */
+   else
+   if (ny > 16) tile_mode = 0x020; /* height 32 tiles */
+   else
+   if (ny >  8) tile_mode = 0x010; /* height 16 tiles */
+
+   if (nz == 1)
+      return tile_mode;
+   else
+   if (tile_mode > 0x020)
+      tile_mode = 0x020;
+
+   if (nz > 16 && tile_mode < 0x020)
+      return tile_mode | 0x500; /* depth 32 tiles */
+   if (nz > 8) return tile_mode | 0x400; /* depth 16 tiles */
+   if (nz > 4) return tile_mode | 0x300; /* depth 8 tiles */
+   if (nz > 2) return tile_mode | 0x200; /* depth 4 tiles */
+
+   return tile_mode | 0x100;
+}
+
+static INLINE unsigned
+calc_zslice_offset(uint32_t tile_mode, unsigned z, unsigned pitch, unsigned nbh)
+{
+   unsigned tile_h = NVC0_TILE_HEIGHT(tile_mode);
+   unsigned tile_d_shift = NVC0_TILE_DIM_SHIFT(tile_mode, 2);
+   unsigned tile_d = 1 << tile_d_shift;
+
+   /* stride_2d == to next slice within this volume tile */
+   /* stride_3d == size (in bytes) of a volume tile */
+   unsigned stride_2d = tile_h * NVC0_TILE_PITCH(tile_mode);
+   unsigned stride_3d = tile_d * align(nbh, tile_h) * pitch;
+
+   return (z & (tile_d - 1)) * stride_2d + (z >> tile_d_shift) * stride_3d;
+}
+
+static void
+nvc0_miptree_destroy(struct pipe_screen *pscreen, struct pipe_resource *pt)
+{
+   struct nvc0_miptree *mt = nvc0_miptree(pt);
+
+   nouveau_screen_bo_release(pscreen, mt->base.bo);
+
+   FREE(mt);
+}
+
+static boolean
+nvc0_miptree_get_handle(struct pipe_screen *pscreen,
+                        struct pipe_resource *pt,
+                        struct winsys_handle *whandle)
+{
+   struct nvc0_miptree *mt = nvc0_miptree(pt);
+   unsigned stride;
+
+   if (!mt || !mt->base.bo)
+      return FALSE;
+
+   stride = util_format_get_stride(mt->base.base.format,
+                                   mt->base.base.width0);
+
+   return nouveau_screen_bo_get_handle(pscreen,
+                                       mt->base.bo,
+                                       stride,
+                                       whandle);
+}
+
+const struct u_resource_vtbl nvc0_miptree_vtbl =
+{
+   nvc0_miptree_get_handle,         /* get_handle */
+   nvc0_miptree_destroy,            /* resource_destroy */
+   NULL,                            /* is_resource_referenced */
+   nvc0_miptree_transfer_new,       /* get_transfer */
+   nvc0_miptree_transfer_del,       /* transfer_destroy */
+   nvc0_miptree_transfer_map,	      /* transfer_map */
+   u_default_transfer_flush_region, /* transfer_flush_region */
+   nvc0_miptree_transfer_unmap,     /* transfer_unmap */
+   u_default_transfer_inline_write  /* transfer_inline_write */
+};
+
+struct pipe_resource *
+nvc0_miptree_create(struct pipe_screen *pscreen,
+                    const struct pipe_resource *templ)
+{
+   struct nouveau_device *dev = nouveau_screen(pscreen)->device;
+   struct nvc0_miptree *mt = CALLOC_STRUCT(nvc0_miptree);
+   struct pipe_resource *pt = &mt->base.base;
+   int ret;
+   unsigned w, h, d, l, alloc_size;
+   uint32_t tile_flags;
+
+   if (!mt)
+      return NULL;
+
+   mt->base.vtbl = &nvc0_miptree_vtbl;
+   *pt = *templ;
+   pipe_reference_init(&pt->reference, 1);
+   pt->screen = pscreen;
+
+   mt->layout_3d = pt->target == PIPE_TEXTURE_3D;
+
+   w = pt->width0;
+   h = pt->height0;
+   d = mt->layout_3d ? pt->depth0 : 1;
+
+   switch (pt->format) {
+   case PIPE_FORMAT_Z16_UNORM:
+      tile_flags = 0x0700; /* COMPRESSED */
+      tile_flags = 0x0200; /* NORMAL ? */
+      tile_flags = 0x0100; /* NORMAL ? */
+      break;
+   case PIPE_FORMAT_S8_USCALED_Z24_UNORM:
+      tile_flags = 0x5300; /* MSAA 4, COMPRESSED */
+      tile_flags = 0x4600; /* NORMAL */
+      break;
+   case PIPE_FORMAT_Z24X8_UNORM:
+   case PIPE_FORMAT_Z24_UNORM_S8_USCALED:
+      tile_flags = 0x1100; /* NORMAL */
+      if (w * h >= 128 * 128 && 0)
+         tile_flags = 0x1700; /* COMPRESSED, requires magic */
+      break;
+   case PIPE_FORMAT_R32G32B32A32_FLOAT:
+      tile_flags = 0xf500; /* COMPRESSED */
+      tile_flags = 0xf700; /* MSAA 2 */
+      tile_flags = 0xf900; /* MSAA 4 */
+      tile_flags = 0xfe00; /* NORMAL */
+      break;
+   case PIPE_FORMAT_Z32_FLOAT_S8X24_USCALED:
+      tile_flags = 0xce00; /* COMPRESSED */
+      tile_flags = 0xcf00; /* MSAA 2, COMPRESSED */
+      tile_flags = 0xd000; /* MSAA 4, COMPRESSED */
+      tile_flags = 0xc300; /* NORMAL */
+      break;
+   case PIPE_FORMAT_R16G16B16A16_UNORM:
+      tile_flags = 0xe900; /* COMPRESSED */
+      break;
+   default:
+      tile_flags = 0xe000; /* MSAA 4, COMPRESSED 32 BIT */
+      tile_flags = 0xfe00; /* NORMAL 32 BIT */
+      if (w * h >= 128 * 128 && 0)
+         tile_flags = 0xdb00; /* COMPRESSED 32 BIT, requires magic */
+      break;
+   }
+
+   /* For 3D textures, a mipmap is spanned by all the layers, for array
+    * textures and cube maps, each layer contains its own mipmaps.
+    */
+   for (l = 0; l <= pt->last_level; ++l) {
+      struct nvc0_miptree_level *lvl = &mt->level[l];
+      unsigned nbx = util_format_get_nblocksx(pt->format, w);
+      unsigned nby = util_format_get_nblocksy(pt->format, h);
+      unsigned blocksize = util_format_get_blocksize(pt->format);
+
+      lvl->offset = mt->total_size;
+      lvl->tile_mode = get_tile_dims(nbx, nby, d);
+      lvl->pitch = align(nbx * blocksize, NVC0_TILE_PITCH(lvl->tile_mode));
+
+      mt->total_size += lvl->pitch *
+         align(nby, NVC0_TILE_HEIGHT(lvl->tile_mode)) *
+         align(d, NVC0_TILE_DEPTH(lvl->tile_mode));
+
+      w = u_minify(w, 1);
+      h = u_minify(h, 1);
+      d = u_minify(d, 1);
+   }
+
+   if (pt->array_size > 1) {
+      mt->layer_stride = align(mt->total_size,
+                               NVC0_TILE_SIZE(mt->level[0].tile_mode));
+      mt->total_size = mt->layer_stride * pt->array_size;
+   }
+
+   alloc_size = mt->total_size;
+   if (tile_flags == 0x1700)
+      alloc_size *= 3; /* HiZ, XXX: correct size */
+
+   ret = nouveau_bo_new_tile(dev, NOUVEAU_BO_VRAM, 256, alloc_size,
+                             mt->level[0].tile_mode, tile_flags,
+                             &mt->base.bo);
+   if (ret) {
+      FREE(mt);
+      return NULL;
+   }
+   mt->base.domain = NOUVEAU_BO_VRAM;
+
+   return pt;
+}
+
+struct pipe_resource *
+nvc0_miptree_from_handle(struct pipe_screen *pscreen,
+                         const struct pipe_resource *templ,
+                         struct winsys_handle *whandle)
+{
+   struct nvc0_miptree *mt;
+   unsigned stride;
+
+   /* only supports 2D, non-mipmapped textures for the moment */
+   if ((templ->target != PIPE_TEXTURE_2D &&
+        templ->target != PIPE_TEXTURE_RECT) ||
+       templ->last_level != 0 ||
+       templ->depth0 != 1 ||
+       templ->array_size > 1)
+      return NULL;
+
+   mt = CALLOC_STRUCT(nvc0_miptree);
+   if (!mt)
+      return NULL;
+
+   mt->base.bo = nouveau_screen_bo_from_handle(pscreen, whandle, &stride);
+   if (mt->base.bo == NULL) {
+      FREE(mt);
+      return NULL;
+   }
+
+   mt->base.base = *templ;
+   mt->base.vtbl = &nvc0_miptree_vtbl;
+   pipe_reference_init(&mt->base.base.reference, 1);
+   mt->base.base.screen = pscreen;
+   mt->level[0].pitch = stride;
+   mt->level[0].offset = 0;
+   mt->level[0].tile_mode = mt->base.bo->tile_mode;
+
+   /* no need to adjust bo reference count */
+   return &mt->base.base;
+}
+
+
+/* Surface functions.
+ */
+
+struct pipe_surface *
+nvc0_miptree_surface_new(struct pipe_context *pipe,
+                         struct pipe_resource *pt,
+                         const struct pipe_surface *templ)
+{
+   struct nvc0_miptree *mt = nvc0_miptree(pt); /* guaranteed */
+   struct nvc0_surface *ns;
+   struct pipe_surface *ps;
+   struct nvc0_miptree_level *lvl = &mt->level[templ->u.tex.level];
+
+   ns = CALLOC_STRUCT(nvc0_surface);
+   if (!ns)
+      return NULL;
+   ps = &ns->base;
+
+   pipe_reference_init(&ps->reference, 1);
+   pipe_resource_reference(&ps->texture, pt);
+   ps->context = pipe;
+   ps->format = pt->format;
+   ps->usage = templ->usage;
+   ps->u.tex.level = templ->u.tex.level;
+   ps->u.tex.first_layer = templ->u.tex.first_layer;
+   ps->u.tex.last_layer = templ->u.tex.last_layer;
+
+   ns->width = u_minify(pt->width0, ps->u.tex.level);
+   ns->height = u_minify(pt->height0, ps->u.tex.level);
+   ns->depth = ps->u.tex.last_layer - ps->u.tex.first_layer + 1;
+   ns->offset = lvl->offset;
+
+   /* comment says there are going to be removed, but they're used by the st */
+   ps->width = ns->width;
+   ps->height = ns->height;
+
+   if (mt->layout_3d) {
+      unsigned zslice = ps->u.tex.first_layer;
+
+      /* TODO: re-layout the texture to use only depth 1 tiles in this case: */
+      if (ns->depth > 1 && (zslice & (NVC0_TILE_DEPTH(lvl->tile_mode) - 1)))
+         NOUVEAU_ERR("Creating unsupported 3D surface of slices [%u:%u].\n",
+                     zslice, ps->u.tex.last_layer);
+
+      ns->offset += calc_zslice_offset(lvl->tile_mode, zslice, lvl->pitch,
+                                       util_format_get_nblocksy(pt->format,
+                                                                ns->height));
+   } else {
+      ns->offset += mt->layer_stride * ps->u.tex.first_layer;
+   }
+
+   return ps;
+}
+
+void
+nvc0_miptree_surface_del(struct pipe_context *pipe, struct pipe_surface *ps)
+{
+   struct nvc0_surface *s = nvc0_surface(ps);
+
+   pipe_resource_reference(&ps->texture, NULL);
+
+   FREE(s);
+}
diff --git a/src/gallium/drivers/nvc0/nvc0_mm.c b/src/gallium/drivers/nvc0/nvc0_mm.c
new file mode 100644
index 00000000000..0629dad19c9
--- /dev/null
+++ b/src/gallium/drivers/nvc0/nvc0_mm.c
@@ -0,0 +1,274 @@
+
+#include "util/u_inlines.h"
+#include "util/u_memory.h"
+#include "util/u_double_list.h"
+
+#include "nvc0_screen.h"
+
+#define MM_MIN_ORDER 7
+#define MM_MAX_ORDER 20
+
+#define MM_NUM_BUCKETS (MM_MAX_ORDER - MM_MIN_ORDER + 1)
+
+#define MM_MIN_SIZE (1 << MM_MIN_ORDER)
+#define MM_MAX_SIZE (1 << MM_MAX_ORDER)
+
+struct mm_bucket {
+   struct list_head free;
+   struct list_head used;
+   struct list_head full;
+   int num_free;
+};
+
+struct nvc0_mman {
+   struct nouveau_device *dev;
+   struct mm_bucket bucket[MM_NUM_BUCKETS];
+   uint32_t storage_type;
+   uint32_t domain;
+   uint64_t allocated;
+};
+
+struct mm_slab {
+   struct list_head head;
+   struct nouveau_bo *bo;
+   struct nvc0_mman *cache;
+   int order;
+   int count;
+   int free;
+   uint32_t bits[0];
+};
+
+static int
+mm_slab_alloc(struct mm_slab *slab)
+{
+   int i, n, b;
+
+   if (slab->free == 0)
+      return -1;
+
+   for (i = 0; i < (slab->count + 31) / 32; ++i) {
+      b = ffs(slab->bits[i]) - 1;
+      if (b >= 0) {
+         n = i * 32 + b;
+         assert(n < slab->count);
+         slab->free--;
+         slab->bits[i] &= ~(1 << b);
+         return n;
+      }
+   }
+   return -1;
+}
+
+static INLINE void
+mm_slab_free(struct mm_slab *slab, int i)
+{
+   assert(i < slab->count);
+   slab->bits[i / 32] |= 1 << (i % 32);
+   slab->free++;
+   assert(slab->free <= slab->count);
+}
+
+static INLINE int
+mm_get_order(uint32_t size)
+{
+   int s = __builtin_clz(size) ^ 31;
+
+   if (size > (1 << s))
+      s += 1;
+   return s;
+}
+
+static struct mm_bucket *
+mm_bucket_by_order(struct nvc0_mman *cache, int order)
+{
+   if (order > MM_MAX_ORDER)
+      return NULL;
+   return &cache->bucket[MAX2(order, MM_MIN_ORDER) - MM_MIN_ORDER];
+}
+
+static struct mm_bucket *
+mm_bucket_by_size(struct nvc0_mman *cache, unsigned size)
+{
+   return mm_bucket_by_order(cache, mm_get_order(size));
+}
+
+/* size of bo allocation for slab with chunks of (1 << chunk_order) bytes */
+static INLINE uint32_t
+mm_default_slab_size(unsigned chunk_order)
+{
+   assert(chunk_order <= MM_MAX_ORDER && chunk_order >= MM_MIN_ORDER);
+
+   static const int8_t slab_order[MM_MAX_ORDER - MM_MIN_ORDER + 1] =
+   {
+      12, 12, 13, 14, 14, 17, 17, 17, 17, 19, 19, 20, 21, 22
+   };
+
+   return 1 << slab_order[chunk_order - MM_MIN_ORDER];
+}
+
+static int
+mm_slab_new(struct nvc0_mman *cache, int chunk_order)
+{
+   struct mm_slab *slab;
+   int words, ret;
+   const uint32_t size = mm_default_slab_size(chunk_order);
+
+   words = ((size >> chunk_order) + 31) / 32;
+   assert(words);
+
+   slab = MALLOC(sizeof(struct mm_slab) + words * 4);
+   if (!slab)
+      return PIPE_ERROR_OUT_OF_MEMORY;
+
+   memset(&slab->bits[0], ~0, words * 4);
+
+   slab->bo = NULL;
+   ret = nouveau_bo_new_tile(cache->dev, cache->domain, 0, size,
+                             0, cache->storage_type, &slab->bo);
+   if (ret) {
+      FREE(slab);
+      return PIPE_ERROR_OUT_OF_MEMORY;
+   }
+
+   LIST_INITHEAD(&slab->head);
+
+   slab->cache = cache;
+   slab->order = chunk_order;
+   slab->count = slab->free = size >> chunk_order;
+
+   LIST_ADD(&slab->head, &mm_bucket_by_order(cache, chunk_order)->free);
+
+   cache->allocated += size;
+
+   debug_printf("MM: new slab, total memory = %lu KiB\n",
+                cache->allocated / 1024);
+
+   return PIPE_OK;
+}
+
+/* @return token to identify slab or NULL if we just allocated a new bo */
+struct nvc0_mm_allocation *
+nvc0_mm_allocate(struct nvc0_mman *cache,
+                 uint32_t size, struct nouveau_bo **bo, uint32_t *offset)
+{
+   struct mm_bucket *bucket;
+   struct mm_slab *slab;
+   struct nvc0_mm_allocation *alloc;
+   int ret;
+
+   bucket = mm_bucket_by_size(cache, size);
+   if (!bucket) {
+      ret = nouveau_bo_new_tile(cache->dev, cache->domain, 0, size,
+                                0, cache->storage_type, bo);
+      if (ret)
+         debug_printf("bo_new(%x, %x): %i\n", size, cache->storage_type, ret);
+
+      *offset = 0;
+      return NULL;
+   }
+
+   if (!LIST_IS_EMPTY(&bucket->used)) {
+      slab = LIST_ENTRY(struct mm_slab, bucket->used.next, head);
+   } else {
+      if (LIST_IS_EMPTY(&bucket->free)) {
+         mm_slab_new(cache, MAX2(mm_get_order(size), MM_MIN_ORDER));
+      }
+      slab = LIST_ENTRY(struct mm_slab, bucket->free.next, head);
+
+      LIST_DEL(&slab->head);
+      LIST_ADD(&slab->head, &bucket->used);
+   }
+
+   *offset = mm_slab_alloc(slab) << slab->order;
+
+   alloc = MALLOC_STRUCT(nvc0_mm_allocation);
+   if (!alloc)
+      return NULL;
+
+   nouveau_bo_ref(slab->bo, bo);
+
+   if (slab->free == 0) {
+      LIST_DEL(&slab->head);
+      LIST_ADD(&slab->head, &bucket->full);
+   }
+
+   alloc->next = NULL;
+   alloc->offset = *offset;
+   alloc->priv = (void *)slab;
+
+   return alloc;
+}
+
+void
+nvc0_mm_free(struct nvc0_mm_allocation *alloc)
+{
+   struct mm_slab *slab = (struct mm_slab *)alloc->priv;
+   struct mm_bucket *bucket = mm_bucket_by_order(slab->cache, slab->order);
+
+   mm_slab_free(slab, alloc->offset >> slab->order);
+
+   if (slab->free == 1) {
+      LIST_DEL(&slab->head);
+
+      if (slab->count > 1)
+         LIST_ADDTAIL(&slab->head, &bucket->used);
+      else
+         LIST_ADDTAIL(&slab->head, &bucket->free);
+   }
+
+   FREE(alloc);
+}
+
+struct nvc0_mman *
+nvc0_mm_create(struct nouveau_device *dev, uint32_t domain,
+               uint32_t storage_type)
+{
+   struct nvc0_mman *cache = MALLOC_STRUCT(nvc0_mman);
+   int i;
+
+   if (!cache)
+      return NULL;
+
+   cache->dev = dev;
+   cache->domain = domain;
+   cache->storage_type = storage_type;
+   cache->allocated = 0;
+
+   for (i = 0; i < MM_NUM_BUCKETS; ++i) {
+      LIST_INITHEAD(&cache->bucket[i].free);
+      LIST_INITHEAD(&cache->bucket[i].used);
+      LIST_INITHEAD(&cache->bucket[i].full);
+   }
+
+   return cache;
+}
+
+static INLINE void
+nvc0_mm_free_slabs(struct list_head *head)
+{
+   struct mm_slab *slab, *next;
+
+   LIST_FOR_EACH_ENTRY_SAFE(slab, next, head, head) {
+      LIST_DEL(&slab->head);
+      nouveau_bo_ref(NULL, &slab->bo);
+      FREE(slab);
+   }
+}
+
+void
+nvc0_mm_destroy(struct nvc0_mman *cache)
+{
+   int i;
+
+   for (i = 0; i < MM_NUM_BUCKETS; ++i) {
+      if (!LIST_IS_EMPTY(&cache->bucket[i].used) ||
+          !LIST_IS_EMPTY(&cache->bucket[i].full))
+         debug_printf("WARNING: destroying GPU memory cache "
+                      "with some buffers still in use\n");
+
+      nvc0_mm_free_slabs(&cache->bucket[i].free);
+      nvc0_mm_free_slabs(&cache->bucket[i].used);
+      nvc0_mm_free_slabs(&cache->bucket[i].full);
+   }
+}
+
diff --git a/src/gallium/drivers/nvc0/nvc0_pc.c b/src/gallium/drivers/nvc0/nvc0_pc.c
new file mode 100644
index 00000000000..72483f120ed
--- /dev/null
+++ b/src/gallium/drivers/nvc0/nvc0_pc.c
@@ -0,0 +1,686 @@
+/*
+ * Copyright 2010 Christoph Bumiller
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF
+ * OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#define NOUVEAU_DEBUG 1
+
+#include "nvc0_pc.h"
+#include "nvc0_program.h"
+
+boolean
+nvc0_insn_can_load(struct nv_instruction *nvi, int s,
+                   struct nv_instruction *ld)
+{
+   int i;
+
+   if (ld->opcode == NV_OP_MOV && ld->src[0]->value->reg.file == NV_FILE_IMM) {
+      if (s > 1 || !(nvc0_op_info_table[nvi->opcode].immediate & (1 << s)))
+         return FALSE;
+      if (!(nvc0_op_info_table[nvi->opcode].immediate & 4))
+         if (ld->src[0]->value->reg.imm.u32 & 0xfff)
+            return FALSE;
+   } else
+   if (!(nvc0_op_info_table[nvi->opcode].memory & (1 << s)))
+      return FALSE;
+
+   if (ld->indirect >= 0)
+      return FALSE;
+
+   for (i = 0; i < 3 && nvi->src[i]; ++i)
+      if (nvi->src[i]->value->reg.file == NV_FILE_IMM)
+         return FALSE;
+
+   return TRUE;
+}
+
+/* Return whether this instruction can be executed conditionally. */
+boolean
+nvc0_insn_is_predicateable(struct nv_instruction *nvi)
+{
+   int s;
+
+   if (!nv_op_predicateable(nvi->opcode))
+      return FALSE;
+   if (nvi->predicate >= 0)
+      return FALSE;
+   for (s = 0; s < 4 && nvi->src[s]; ++s)
+      if (nvi->src[s]->value->reg.file == NV_FILE_IMM)
+         return FALSE;
+   return TRUE;
+}
+
+int
+nvc0_insn_refcount(struct nv_instruction *nvi)
+{
+   int rc = 0;
+   int i;
+   for (i = 0; i < 5 && nvi->def[i]; ++i) {
+      if (!nvi->def[i])
+         return rc;
+      rc += nvi->def[i]->refc;
+   }
+   return rc;
+}
+
+int
+nvc0_pc_replace_value(struct nv_pc *pc,
+		      struct nv_value *old_val,
+		      struct nv_value *new_val)
+{
+   int i, n, s;
+
+   if (old_val == new_val)
+      return old_val->refc;
+
+   for (i = 0, n = 0; i < pc->num_refs; ++i) {
+      if (pc->refs[i]->value == old_val) {
+         ++n;
+         for (s = 0; s < 6 && pc->refs[i]->insn->src[s]; ++s)
+            if (pc->refs[i]->insn->src[s] == pc->refs[i])
+               break;
+         assert(s < 6);
+         nv_reference(pc, pc->refs[i]->insn, s, new_val);
+      }
+   }
+   return n;
+}
+
+struct nv_value *
+nvc0_pc_find_constant(struct nv_ref *ref)
+{
+   struct nv_value *src;
+
+   if (!ref)
+      return NULL;
+
+   src = ref->value;
+   while (src->insn && src->insn->opcode == NV_OP_MOV) {
+      assert(!src->insn->src[0]->mod);
+      src = src->insn->src[0]->value;
+   }
+   if ((src->reg.file == NV_FILE_IMM) ||
+       (src->insn &&
+        src->insn->opcode == NV_OP_LD &&
+        src->insn->src[0]->value->reg.file >= NV_FILE_MEM_C(0) &&
+        src->insn->src[0]->value->reg.file <= NV_FILE_MEM_C(15)))
+      return src;
+   return NULL;
+}
+
+struct nv_value *
+nvc0_pc_find_immediate(struct nv_ref *ref)
+{
+   struct nv_value *src = nvc0_pc_find_constant(ref);
+
+   return (src && src->reg.file == NV_FILE_IMM) ? src : NULL;
+}
+
+static void
+nv_pc_free_refs(struct nv_pc *pc)
+{
+   int i;
+   for (i = 0; i < pc->num_refs; i += 64)
+      FREE(pc->refs[i]);
+   FREE(pc->refs);
+}
+
+static const char *
+edge_name(ubyte type)
+{
+   switch (type) {
+   case CFG_EDGE_FORWARD: return "forward";
+   case CFG_EDGE_BACK: return "back";
+   case CFG_EDGE_LOOP_ENTER: return "loop";
+   case CFG_EDGE_LOOP_LEAVE: return "break";
+   case CFG_EDGE_FAKE: return "fake";
+   default:
+      return "?";
+   }
+}
+
+void
+nvc0_pc_pass_in_order(struct nv_basic_block *root, nv_pc_pass_func f,
+                      void *priv)
+{
+   struct nv_basic_block *bb[64], *bbb[16], *b;
+   int j, p, pp;
+
+   bb[0] = root;
+   p = 1;
+   pp = 0;
+
+   while (p > 0) {
+      b = bb[--p];
+      b->priv = 0;
+
+      for (j = 1; j >= 0; --j) {
+         if (!b->out[j])
+            continue;
+
+         switch (b->out_kind[j]) {
+         case CFG_EDGE_BACK:
+            continue;
+         case CFG_EDGE_FORWARD:
+         case CFG_EDGE_FAKE:
+            if (++b->out[j]->priv == b->out[j]->num_in)
+               bb[p++] = b->out[j];
+            break;
+         case CFG_EDGE_LOOP_ENTER:
+            bb[p++] = b->out[j];
+            break;
+         case CFG_EDGE_LOOP_LEAVE:
+            bbb[pp++] = b->out[j];
+            break;
+         default:
+            assert(0);
+            break;
+         }
+      }
+
+      f(priv, b);
+
+      if (!p) {
+         p = pp;
+         for (; pp > 0; --pp)
+            bb[pp - 1] = bbb[pp - 1];
+      }
+   }
+}
+
+static void
+nv_do_print_function(void *priv, struct nv_basic_block *b)
+{
+   struct nv_instruction *i;
+
+   debug_printf("=== BB %i ", b->id);
+   if (b->out[0])
+      debug_printf("[%s -> %i] ", edge_name(b->out_kind[0]), b->out[0]->id);
+   if (b->out[1])
+      debug_printf("[%s -> %i] ", edge_name(b->out_kind[1]), b->out[1]->id);
+   debug_printf("===\n");
+
+   i = b->phi;
+   if (!i)
+      i = b->entry;
+   for (; i; i = i->next)
+      nvc0_print_instruction(i);
+}
+
+void
+nvc0_print_function(struct nv_basic_block *root)
+{
+   if (root->subroutine)
+      debug_printf("SUBROUTINE %i\n", root->subroutine);
+   else
+      debug_printf("MAIN\n");
+
+   nvc0_pc_pass_in_order(root, nv_do_print_function, root);
+}
+
+void
+nvc0_print_program(struct nv_pc *pc)
+{
+   int i;
+   for (i = 0; i < pc->num_subroutines + 1; ++i)
+      if (pc->root[i])
+         nvc0_print_function(pc->root[i]);
+}
+
+#if NOUVEAU_DEBUG > 1
+static void
+nv_do_print_cfgraph(struct nv_pc *pc, FILE *f, struct nv_basic_block *b)
+{
+   int i;
+
+   b->pass_seq = pc->pass_seq;
+
+   fprintf(f, "\t%i [shape=box]\n", b->id);
+
+   for (i = 0; i < 2; ++i) {
+      if (!b->out[i])
+         continue;
+      switch (b->out_kind[i]) {
+      case CFG_EDGE_FORWARD:
+         fprintf(f, "\t%i -> %i;\n", b->id, b->out[i]->id);
+         break;
+      case CFG_EDGE_LOOP_ENTER:
+         fprintf(f, "\t%i -> %i [color=green];\n", b->id, b->out[i]->id);
+         break;
+      case CFG_EDGE_LOOP_LEAVE:
+         fprintf(f, "\t%i -> %i [color=red];\n", b->id, b->out[i]->id);
+         break;
+      case CFG_EDGE_BACK:
+         fprintf(f, "\t%i -> %i;\n", b->id, b->out[i]->id);
+         continue;
+      case CFG_EDGE_FAKE:
+         fprintf(f, "\t%i -> %i [style=dotted];\n", b->id, b->out[i]->id);
+         break;
+      default:
+         assert(0);
+         break;
+      }
+      if (b->out[i]->pass_seq < pc->pass_seq)
+         nv_do_print_cfgraph(pc, f, b->out[i]);
+   }
+}
+
+/* Print the control flow graph of subroutine @subr (0 == MAIN) to a file. */
+static void
+nv_print_cfgraph(struct nv_pc *pc, const char *filepath, int subr)
+{
+   FILE *f;
+
+   f = fopen(filepath, "a");
+   if (!f)
+      return;
+
+   fprintf(f, "digraph G {\n");
+
+   ++pc->pass_seq;
+
+   nv_do_print_cfgraph(pc, f, pc->root[subr]);
+
+   fprintf(f, "}\n");
+
+   fclose(f);
+}
+#endif
+
+static INLINE void
+nvc0_pc_print_binary(struct nv_pc *pc)
+{
+   unsigned i;
+
+   NOUVEAU_DBG("nvc0_pc_print_binary(%u ops)\n", pc->emit_size / 8);
+
+   for (i = 0; i < pc->emit_size / 4; i += 2) {
+      debug_printf("0x%08x ", pc->emit[i + 0]);
+      debug_printf("0x%08x ", pc->emit[i + 1]);
+      if ((i % 16) == 15)
+         debug_printf("\n");
+   }
+   debug_printf("\n");
+}
+
+static int
+nvc0_emit_program(struct nv_pc *pc)
+{
+   uint32_t *code = pc->emit;
+   int n;
+
+   NOUVEAU_DBG("emitting program: size = %u\n", pc->emit_size);
+
+   pc->emit_pos = 0;
+   for (n = 0; n < pc->num_blocks; ++n) {
+      struct nv_instruction *i;
+      struct nv_basic_block *b = pc->bb_list[n];
+
+      for (i = b->entry; i; i = i->next) {
+         nvc0_emit_instruction(pc, i);
+         pc->emit += 2;
+         pc->emit_pos += 8;
+      }
+   }
+   assert(pc->emit == &code[pc->emit_size / 4]);
+
+   pc->emit[0] = 0x00001de7;
+   pc->emit[1] = 0x80000000;
+   pc->emit_size += 8;
+
+   pc->emit = code;
+
+#ifdef NOUVEAU_DEBUG
+   nvc0_pc_print_binary(pc);
+#else
+   debug_printf("not printing binary\n");
+#endif
+   return 0;
+}
+
+int
+nvc0_generate_code(struct nvc0_translation_info *ti)
+{
+   struct nv_pc *pc;
+   int ret;
+   int i;
+
+   pc = CALLOC_STRUCT(nv_pc);
+   if (!pc)
+      return 1;
+
+   pc->is_fragprog = ti->prog->type == PIPE_SHADER_FRAGMENT;
+
+   pc->root = CALLOC(ti->num_subrs + 1, sizeof(pc->root[0]));
+   if (!pc->root) {
+      FREE(pc);
+      return 1;
+   }
+   pc->num_subroutines = ti->num_subrs;
+
+   ret = nvc0_tgsi_to_nc(pc, ti);
+   if (ret)
+      goto out;
+#if NOUVEAU_DEBUG > 1
+   nvc0_print_program(pc);
+#endif
+
+   pc->opt_reload_elim = ti->require_stores ? FALSE : TRUE;
+
+   /* optimization */
+   ret = nvc0_pc_exec_pass0(pc);
+   if (ret)
+      goto out;
+#ifdef NOUVEAU_DEBUG
+   nvc0_print_program(pc);
+#endif
+
+   /* register allocation */
+   ret = nvc0_pc_exec_pass1(pc);
+   if (ret)
+      goto out;
+#if NOUVEAU_DEBUG > 1
+   nvc0_print_program(pc);
+   nv_print_cfgraph(pc, "nvc0_shader_cfgraph.dot", 0);
+#endif
+
+   /* prepare for emission */
+   ret = nvc0_pc_exec_pass2(pc);
+   if (ret)
+      goto out;
+   assert(!(pc->emit_size % 8));
+
+   pc->emit = CALLOC(pc->emit_size / 4 + 2, 4);
+   if (!pc->emit) {
+      ret = 3;
+      goto out;
+   }
+   ret = nvc0_emit_program(pc);
+   if (ret)
+      goto out;
+
+   ti->prog->code = pc->emit;
+   ti->prog->code_base = 0;
+   ti->prog->code_size = pc->emit_size;
+   ti->prog->parm_size = 0;
+
+   ti->prog->max_gpr = MAX2(4, pc->max_reg[NV_FILE_GPR] + 1);
+
+   ti->prog->relocs = pc->reloc_entries;
+   ti->prog->num_relocs = pc->num_relocs;
+
+   NOUVEAU_DBG("SHADER TRANSLATION - %s\n", ret ? "failure" : "success");
+
+out:
+   nv_pc_free_refs(pc);
+
+   for (i = 0; i < pc->num_blocks; ++i)
+      FREE(pc->bb_list[i]);
+   if (pc->root)
+      FREE(pc->root);
+   if (ret) {
+      /* on success, these will be referenced by struct nvc0_program */
+      if (pc->emit)
+         FREE(pc->emit);
+      if (pc->immd_buf)
+         FREE(pc->immd_buf);
+      if (pc->reloc_entries)
+         FREE(pc->reloc_entries);
+   }
+   FREE(pc);
+   return ret;
+}
+
+static void
+nvbb_insert_phi(struct nv_basic_block *b, struct nv_instruction *i)
+{
+   if (!b->phi) {
+      i->prev = NULL;
+      b->phi = i;
+      i->next = b->entry;
+      if (b->entry) {
+         assert(!b->entry->prev && b->exit);
+         b->entry->prev = i;
+      } else {
+         b->entry = i;
+         b->exit = i;
+      }
+   } else {
+      assert(b->entry);
+      if (b->entry->opcode == NV_OP_PHI) { /* insert after entry */
+         assert(b->entry == b->exit);
+         b->entry->next = i;
+         i->prev = b->entry;
+         b->entry = i;
+         b->exit = i;
+      } else { /* insert before entry */
+         assert(b->entry->prev && b->exit);
+         i->next = b->entry;
+         i->prev = b->entry->prev;
+         b->entry->prev = i;
+         i->prev->next = i;
+      }
+   }
+}
+
+void
+nvc0_insn_append(struct nv_basic_block *b, struct nv_instruction *i)
+{
+   if (i->opcode == NV_OP_PHI) {
+      nvbb_insert_phi(b, i);
+   } else {
+      i->prev = b->exit;
+      if (b->exit)
+         b->exit->next = i;
+      b->exit = i;
+      if (!b->entry)
+         b->entry = i;
+      else
+      if (i->prev && i->prev->opcode == NV_OP_PHI)
+         b->entry = i;
+   }
+
+   i->bb = b;
+   b->num_instructions++;
+}
+
+void
+nvc0_insn_insert_after(struct nv_instruction *at, struct nv_instruction *ni)
+{
+   if (!at->next) {
+      nvc0_insn_append(at->bb, ni);
+      return;
+   }
+   ni->next = at->next;
+   ni->prev = at;
+   ni->next->prev = ni;
+   ni->prev->next = ni;
+}
+
+void
+nvc0_insn_delete(struct nv_instruction *nvi)
+{
+   struct nv_basic_block *b = nvi->bb;
+   int s;
+
+   /* debug_printf("REM: "); nv_print_instruction(nvi); */
+
+   for (s = 0; s < 6 && nvi->src[s]; ++s)
+      nv_reference(NULL, nvi, s, NULL);
+
+   if (nvi->next)
+      nvi->next->prev = nvi->prev;
+   else {
+      assert(nvi == b->exit);
+      b->exit = nvi->prev;
+   }
+
+   if (nvi->prev)
+      nvi->prev->next = nvi->next;
+
+   if (nvi == b->entry) {
+      /* PHIs don't get hooked to b->entry */
+      b->entry = nvi->next;
+      assert(!nvi->prev || nvi->prev->opcode == NV_OP_PHI);
+   }
+
+   if (nvi == b->phi) {
+      if (nvi->opcode != NV_OP_PHI)
+         NOUVEAU_DBG("NOTE: b->phi points to non-PHI instruction\n");
+
+      assert(!nvi->prev);
+      if (!nvi->next || nvi->next->opcode != NV_OP_PHI)
+         b->phi = NULL;
+      else
+         b->phi = nvi->next;
+   }
+}
+
+void
+nvc0_insns_permute(struct nv_instruction *i1, struct nv_instruction *i2)
+{
+   struct nv_basic_block *b = i1->bb;
+
+   assert(i1->opcode != NV_OP_PHI &&
+          i2->opcode != NV_OP_PHI);
+   assert(i1->next == i2);
+
+   if (b->exit == i2)
+      b->exit = i1;
+
+   if (b->entry == i1)
+      b->entry = i2;
+
+   i2->prev = i1->prev;
+   i1->next = i2->next;
+   i2->next = i1;
+   i1->prev = i2;
+
+   if (i2->prev)
+      i2->prev->next = i2;
+   if (i1->next)
+      i1->next->prev = i1;
+}
+
+void
+nvc0_bblock_attach(struct nv_basic_block *parent,
+		   struct nv_basic_block *b, ubyte edge_kind)
+{
+   assert(b->num_in < 8);
+
+   if (parent->out[0]) {
+      assert(!parent->out[1]);
+      parent->out[1] = b;
+      parent->out_kind[1] = edge_kind;
+   } else {
+      parent->out[0] = b;
+      parent->out_kind[0] = edge_kind;
+   }
+
+   b->in[b->num_in] = parent;
+   b->in_kind[b->num_in++] = edge_kind;
+}
+
+/* NOTE: all BRKs are treated as conditional, so there are 2 outgoing BBs */
+
+boolean
+nvc0_bblock_dominated_by(struct nv_basic_block *b, struct nv_basic_block *d)
+{
+   int j;
+
+   if (b == d)
+      return TRUE;
+
+   for (j = 0; j < b->num_in; ++j)
+      if ((b->in_kind[j] != CFG_EDGE_BACK) &&
+          !nvc0_bblock_dominated_by(b->in[j], d))
+         return FALSE;
+
+   return j ? TRUE : FALSE;
+}
+
+/* check if @bf (future) can be reached from @bp (past), stop at @bt */
+boolean
+nvc0_bblock_reachable_by(struct nv_basic_block *bf, struct nv_basic_block *bp,
+			 struct nv_basic_block *bt)
+{
+   struct nv_basic_block *q[NV_PC_MAX_BASIC_BLOCKS], *b;
+   int i, p, n;
+
+   p = 0;
+   n = 1;
+   q[0] = bp;
+
+   while (p < n) {
+      b = q[p++];
+
+      if (b == bf)
+         break;
+      if (b == bt)
+         continue;
+      assert(n <= (1024 - 2));
+
+      for (i = 0; i < 2; ++i) {
+         if (b->out[i] && !IS_WALL_EDGE(b->out_kind[i]) && !b->out[i]->priv) {
+            q[n] = b->out[i];
+            q[n++]->priv = 1;
+         }
+      }
+   }
+   for (--n; n >= 0; --n)
+      q[n]->priv = 0;
+
+   return (b == bf);
+}
+
+static struct nv_basic_block *
+nvbb_find_dom_frontier(struct nv_basic_block *b, struct nv_basic_block *df)
+{
+   struct nv_basic_block *out;
+   int i;
+
+   if (!nvc0_bblock_dominated_by(df, b)) {
+      for (i = 0; i < df->num_in; ++i) {
+         if (df->in_kind[i] == CFG_EDGE_BACK)
+            continue;
+         if (nvc0_bblock_dominated_by(df->in[i], b))
+            return df;
+      }
+   }
+   for (i = 0; i < 2 && df->out[i]; ++i) {
+      if (df->out_kind[i] == CFG_EDGE_BACK)
+         continue;
+      if ((out = nvbb_find_dom_frontier(b, df->out[i])))
+         return out;
+   }
+   return NULL;
+}
+
+struct nv_basic_block *
+nvc0_bblock_dom_frontier(struct nv_basic_block *b)
+{
+   struct nv_basic_block *df;
+   int i;
+
+   for (i = 0; i < 2 && b->out[i]; ++i)
+      if ((df = nvbb_find_dom_frontier(b, b->out[i])))
+         return df;
+   return NULL;
+}
diff --git a/src/gallium/drivers/nvc0/nvc0_pc.h b/src/gallium/drivers/nvc0/nvc0_pc.h
new file mode 100644
index 00000000000..74867f02e72
--- /dev/null
+++ b/src/gallium/drivers/nvc0/nvc0_pc.h
@@ -0,0 +1,652 @@
+/*
+ * Copyright 2010 Christoph Bumiller
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF
+ * OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef __NVC0_COMPILER_H__
+#define __NVC0_COMPILER_H__
+
+#include <stdio.h>
+
+#ifndef NOUVEAU_DBG
+#ifdef NOUVEAU_DEBUG
+# define NOUVEAU_DBG(args...) debug_printf(args);
+#else
+# define NOUVEAU_DBG(args...)
+#endif
+#endif
+
+#ifndef NOUVEAU_ERR
+#define NOUVEAU_ERR(fmt, args...) \
+   fprintf(stderr, "%s:%d -  "fmt, __FUNCTION__, __LINE__, ##args);
+#endif
+
+#include "pipe/p_defines.h"
+#include "util/u_inlines.h"
+#include "util/u_memory.h"
+#include "util/u_double_list.h"
+
+/* pseudo opcodes */
+#define NV_OP_UNDEF      0
+#define NV_OP_BIND       1
+#define NV_OP_MERGE      2
+#define NV_OP_PHI        3
+#define NV_OP_SELECT     4
+#define NV_OP_NOP        5
+
+/**
+ * BIND forces source operand i into the same register as destination operand i,
+ *  and the operands will be assigned consecutive registers (needed for TEX)
+ * SELECT forces its multiple source operands and its destination operand into
+ *  one and the same register.
+ */
+
+/* base opcodes */
+#define NV_OP_LD         6
+#define NV_OP_ST         7
+#define NV_OP_MOV        8
+#define NV_OP_AND        9
+#define NV_OP_OR        10
+#define NV_OP_XOR       11
+#define NV_OP_SHL       12
+#define NV_OP_SHR       13
+#define NV_OP_NOT       14
+#define NV_OP_SET       15
+#define NV_OP_ADD       16
+#define NV_OP_SUB       17
+#define NV_OP_MUL       18
+#define NV_OP_MAD       19
+#define NV_OP_ABS       20
+#define NV_OP_NEG       21
+#define NV_OP_MAX       22
+#define NV_OP_MIN       23
+#define NV_OP_CVT       24
+#define NV_OP_CEIL      25
+#define NV_OP_FLOOR     26
+#define NV_OP_TRUNC     27
+#define NV_OP_SAD       28
+
+/* shader opcodes */
+#define NV_OP_VFETCH    29
+#define NV_OP_PFETCH    30
+#define NV_OP_EXPORT    31
+#define NV_OP_LINTERP   32
+#define NV_OP_PINTERP   33
+#define NV_OP_EMIT      34
+#define NV_OP_RESTART   35
+#define NV_OP_TEX       36
+#define NV_OP_TXB       37
+#define NV_OP_TXL       38
+#define NV_OP_TXF       39
+#define NV_OP_TXQ       40
+#define NV_OP_QUADOP    41
+#define NV_OP_DFDX      42
+#define NV_OP_DFDY      43
+#define NV_OP_KIL       44
+
+/* control flow opcodes */
+#define NV_OP_BRA       45
+#define NV_OP_CALL      46
+#define NV_OP_RET       47
+#define NV_OP_EXIT      48
+#define NV_OP_BREAK     49
+#define NV_OP_BREAKADDR 50
+#define NV_OP_JOINAT    51
+#define NV_OP_JOIN      52
+
+/* typed opcodes */
+#define NV_OP_ADD_F32   NV_OP_ADD
+#define NV_OP_ADD_B32   53
+#define NV_OP_MUL_F32   NV_OP_MUL
+#define NV_OP_MUL_B32   54
+#define NV_OP_ABS_F32   NV_OP_ABS
+#define NV_OP_ABS_S32   55
+#define NV_OP_NEG_F32   NV_OP_NEG
+#define NV_OP_NEG_S32   56
+#define NV_OP_MAX_F32   NV_OP_MAX
+#define NV_OP_MAX_S32   57
+#define NV_OP_MAX_U32   58
+#define NV_OP_MIN_F32   NV_OP_MIN
+#define NV_OP_MIN_S32   59
+#define NV_OP_MIN_U32   60
+#define NV_OP_SET_F32   61
+#define NV_OP_SET_S32   62
+#define NV_OP_SET_U32   63
+#define NV_OP_SAR       64
+#define NV_OP_RCP       65
+#define NV_OP_RSQ       66
+#define NV_OP_LG2       67
+#define NV_OP_SIN       68
+#define NV_OP_COS       69
+#define NV_OP_EX2       70
+#define NV_OP_PRESIN    71
+#define NV_OP_PREEX2    72
+#define NV_OP_SAT       73
+
+/* newly added opcodes */
+#define NV_OP_SET_F32_AND 74
+#define NV_OP_SET_F32_OR  75
+#define NV_OP_SET_F32_XOR 76
+#define NV_OP_SELP        77
+#define NV_OP_SLCT        78
+#define NV_OP_SLCT_F32    NV_OP_SLCT
+#define NV_OP_SLCT_S32    79
+#define NV_OP_SLCT_U32    80
+#define NV_OP_SUB_F32     NV_OP_SUB
+#define NV_OP_SUB_S32     81
+#define NV_OP_MAD_F32     NV_OP_MAD
+#define NV_OP_FSET_F32    82
+#define NV_OP_TXG         83
+
+#define NV_OP_COUNT     84
+
+/* nv50 files omitted */
+#define NV_FILE_GPR      0
+#define NV_FILE_COND     1
+#define NV_FILE_PRED     2
+#define NV_FILE_IMM      16
+#define NV_FILE_MEM_S    32
+#define NV_FILE_MEM_V    34
+#define NV_FILE_MEM_A    35
+#define NV_FILE_MEM_L    48
+#define NV_FILE_MEM_G    64
+#define NV_FILE_MEM_C(i) (80 + i)
+
+#define NV_IS_MEMORY_FILE(f) ((f) >= NV_FILE_MEM_S)
+
+#define NV_MOD_NEG 1
+#define NV_MOD_ABS 2
+#define NV_MOD_NOT 4
+#define NV_MOD_SAT 8
+
+#define NV_TYPE_U8  0x00
+#define NV_TYPE_S8  0x01
+#define NV_TYPE_U16 0x02
+#define NV_TYPE_S16 0x03
+#define NV_TYPE_U32 0x04
+#define NV_TYPE_S32 0x05
+#define NV_TYPE_P32 0x07
+#define NV_TYPE_F32 0x09
+#define NV_TYPE_F64 0x0b
+#define NV_TYPE_VEC(x, n) (NV_TYPE_##x | (n << 4))
+#define NV_TYPE_ANY 0xff
+
+#define NV_TYPE_ISINT(t) ((t) < 7)
+#define NV_TYPE_ISSGD(t) ((t) & 1)
+
+#define NV_CC_FL 0x0
+#define NV_CC_LT 0x1
+#define NV_CC_EQ 0x2
+#define NV_CC_LE 0x3
+#define NV_CC_GT 0x4
+#define NV_CC_NE 0x5
+#define NV_CC_GE 0x6
+#define NV_CC_U  0x8
+#define NV_CC_TR 0xf
+#define NV_CC_O  0x10
+#define NV_CC_C  0x11
+#define NV_CC_A  0x12
+#define NV_CC_S  0x13
+
+#define NV_PC_MAX_INSTRUCTIONS 2048
+#define NV_PC_MAX_VALUES (NV_PC_MAX_INSTRUCTIONS * 4)
+
+#define NV_PC_MAX_BASIC_BLOCKS 1024
+
+struct nv_op_info {
+   uint base;                /* e.g. ADD_S32 -> ADD */
+   char name[12];
+   uint8_t type;
+   uint8_t mods;
+   unsigned flow        : 1;
+   unsigned commutative : 1;
+   unsigned vector      : 1;
+   unsigned predicate   : 1;
+   unsigned pseudo      : 1;
+   unsigned immediate   : 3;
+   unsigned memory      : 3;
+};
+
+extern struct nv_op_info nvc0_op_info_table[];
+
+#define NV_BASEOP(op) (nvc0_op_info_table[op].base)
+#define NV_OPTYPE(op) (nvc0_op_info_table[op].type)
+
+static INLINE uint
+nv_op_base(uint opcode)
+{
+   return nvc0_op_info_table[opcode].base;
+}
+
+static INLINE boolean
+nv_is_texture_op(uint opcode)
+{
+   return (opcode >= NV_OP_TEX && opcode <= NV_OP_TXQ);
+}
+
+static INLINE boolean
+nv_is_vector_op(uint opcode)
+{
+   return nvc0_op_info_table[opcode].vector ? TRUE : FALSE;
+}
+
+static INLINE boolean
+nv_op_commutative(uint opcode)
+{
+   return nvc0_op_info_table[opcode].commutative ? TRUE : FALSE;
+}
+
+static INLINE uint8_t
+nv_op_supported_src_mods(uint opcode)
+{
+   return nvc0_op_info_table[opcode].mods;
+}
+
+static INLINE boolean
+nv_op_predicateable(uint opcode)
+{
+   return nvc0_op_info_table[opcode].predicate ? TRUE : FALSE;
+}
+
+static INLINE uint
+nv_type_order(ubyte type)
+{
+   switch (type & 0xf) {
+   case NV_TYPE_U8:
+   case NV_TYPE_S8:
+      return 0;
+   case NV_TYPE_U16:
+   case NV_TYPE_S16:
+      return 1;
+   case NV_TYPE_U32:
+   case NV_TYPE_F32:
+   case NV_TYPE_S32:
+   case NV_TYPE_P32:
+      return 2;
+   case NV_TYPE_F64:
+      return 3;
+   }
+   assert(0);
+   return 0;
+}
+
+static INLINE uint
+nv_type_sizeof(ubyte type)
+{
+   if (type & 0xf0)
+      return (1 << nv_type_order(type)) * (type >> 4);
+   return 1 << nv_type_order(type);
+}
+
+static INLINE uint
+nv_type_sizeof_base(ubyte type)
+{
+   return 1 << nv_type_order(type);
+}
+
+struct nv_reg {
+   uint32_t address; /* for memory locations */
+   int id; /* for registers */
+   ubyte file;
+   ubyte size;
+   union {
+      int32_t s32;
+      int64_t s64;
+      uint64_t u64;
+      uint32_t u32;
+      float f32;
+      double f64;
+   } imm;
+};
+
+struct nv_range {
+   struct nv_range *next;
+   int bgn;
+   int end;
+};
+
+struct nv_ref;
+
+struct nv_value {
+   struct nv_reg reg; 
+   struct nv_instruction *insn;
+   struct nv_value *join;
+   struct nv_ref *last_use;
+   int n;
+   struct nv_range *livei;
+   int refc;
+   struct nv_value *next;
+   struct nv_value *prev;
+};
+
+struct nv_ref {
+   struct nv_value *value;
+   struct nv_instruction *insn;
+   struct list_head list; /* connects uses of the same value */
+   uint8_t mod;
+   uint8_t flags;
+};
+
+struct nv_basic_block;
+
+struct nv_instruction {
+   struct nv_instruction *next;
+   struct nv_instruction *prev;
+   uint opcode;
+   uint serial;
+
+   struct nv_value *def[5];
+   struct nv_ref *src[6];
+
+   int8_t predicate; /* index of predicate src */
+   int8_t indirect;  /* index of pointer src */
+
+   union {
+      struct {
+         uint8_t t; /* TIC binding */
+         uint8_t s; /* TSC binding */
+      } tex;
+      struct {
+         uint8_t d; /* output type */
+         uint8_t s; /* input type */
+      } cvt;
+   } ext;
+
+   struct nv_basic_block *bb;
+   struct nv_basic_block *target; /* target block of control flow insn */
+
+   unsigned cc         : 5; /* condition code */
+   unsigned fixed      : 1; /* don't optimize away (prematurely) */
+   unsigned terminator : 1;
+   unsigned join       : 1;
+   unsigned set_cond   : 4; /* 2nd byte */
+   unsigned saturate   : 1;
+   unsigned centroid   : 1;
+   unsigned flat       : 1;
+   unsigned patch      : 1;
+   unsigned lanes      : 4; /* 3rd byte */
+   unsigned tex_dim    : 2;
+   unsigned tex_array  : 1;
+   unsigned tex_cube   : 1;
+   unsigned tex_shadow : 1; /* 4th byte */
+   unsigned tex_live   : 1;
+   unsigned tex_mask   : 4;
+
+   uint8_t quadop;
+};
+
+static INLINE int
+nvi_vector_size(struct nv_instruction *nvi)
+{
+   int i;
+   assert(nvi);
+   for (i = 0; i < 5 && nvi->def[i]; ++i);
+   return i;
+}
+
+#define CFG_EDGE_FORWARD     0
+#define CFG_EDGE_BACK        1
+#define CFG_EDGE_LOOP_ENTER  2
+#define CFG_EDGE_LOOP_LEAVE  4
+#define CFG_EDGE_FAKE        8
+
+/* 'WALL' edge means where reachability check doesn't follow */
+/* 'LOOP' edge means just having to do with loops */
+#define IS_LOOP_EDGE(k) ((k) & 7)
+#define IS_WALL_EDGE(k) ((k) & 9)
+
+struct nv_basic_block {
+   struct nv_instruction *entry; /* first non-phi instruction */
+   struct nv_instruction *exit;
+   struct nv_instruction *phi; /* very first instruction */
+   int num_instructions;
+
+   struct nv_basic_block *out[2]; /* no indirect branches -> 2 */
+   struct nv_basic_block *in[8]; /* hope that suffices */
+   uint num_in;
+   ubyte out_kind[2];
+   ubyte in_kind[8];
+
+   int id;
+   int subroutine;
+   uint priv; /* reset to 0 after you're done */
+   uint pass_seq;
+
+   uint32_t emit_pos; /* position, size in emitted code (in bytes) */
+   uint32_t emit_size;
+
+   uint32_t live_set[NV_PC_MAX_VALUES / 32];
+};
+
+struct nvc0_translation_info;
+
+struct nv_pc {
+   struct nv_basic_block **root;
+   struct nv_basic_block *current_block;
+   struct nv_basic_block *parent_block;
+
+   int loop_nesting_bound;
+   uint pass_seq;
+
+   struct nv_value values[NV_PC_MAX_VALUES];
+   struct nv_instruction instructions[NV_PC_MAX_INSTRUCTIONS];
+   struct nv_ref **refs;
+   struct nv_basic_block *bb_list[NV_PC_MAX_BASIC_BLOCKS];
+   int num_values;
+   int num_instructions;
+   int num_refs;
+   int num_blocks;
+   int num_subroutines;
+
+   int max_reg[4];
+
+   uint32_t *immd_buf; /* populated on emit */
+   unsigned immd_count;
+
+   uint32_t *emit;
+   uint32_t emit_size;
+   uint32_t emit_pos;
+
+   void *reloc_entries;
+   unsigned num_relocs;
+
+   /* optimization enables */
+   boolean opt_reload_elim;
+   boolean is_fragprog;
+};
+
+void nvc0_insn_append(struct nv_basic_block *, struct nv_instruction *);
+void nvc0_insn_insert_after(struct nv_instruction *, struct nv_instruction *);
+
+static INLINE struct nv_instruction *
+nv_alloc_instruction(struct nv_pc *pc, uint opcode)
+{
+   struct nv_instruction *insn;
+
+   insn = &pc->instructions[pc->num_instructions++];
+   assert(pc->num_instructions < NV_PC_MAX_INSTRUCTIONS);
+
+   insn->opcode = opcode;
+   insn->cc = 0;
+   insn->indirect = -1;
+   insn->predicate = -1;
+
+   return insn;
+}
+
+static INLINE struct nv_instruction *
+new_instruction(struct nv_pc *pc, uint opcode)
+{
+   struct nv_instruction *insn = nv_alloc_instruction(pc, opcode);
+
+   nvc0_insn_append(pc->current_block, insn);
+   return insn;
+}
+
+static INLINE struct nv_instruction *
+new_instruction_at(struct nv_pc *pc, struct nv_instruction *at, uint opcode)
+{
+   struct nv_instruction *insn = nv_alloc_instruction(pc, opcode);
+
+   nvc0_insn_insert_after(at, insn);
+   return insn;
+}
+
+static INLINE struct nv_value *
+new_value(struct nv_pc *pc, ubyte file, ubyte size)
+{
+   struct nv_value *value = &pc->values[pc->num_values];
+
+   assert(pc->num_values < NV_PC_MAX_VALUES - 1);
+
+   value->n = pc->num_values++;
+   value->join = value;
+   value->reg.id = -1;
+   value->reg.file = file;
+   value->reg.size = size;
+   return value;
+}
+
+static INLINE struct nv_value *
+new_value_like(struct nv_pc *pc, struct nv_value *like)
+{
+   return new_value(pc, like->reg.file, like->reg.size);
+}
+
+static INLINE struct nv_ref *
+new_ref(struct nv_pc *pc, struct nv_value *val)
+{
+   int i;
+   struct nv_ref *ref;
+
+   if ((pc->num_refs % 64) == 0) {
+      const unsigned old_size = pc->num_refs * sizeof(struct nv_ref *);
+      const unsigned new_size = (pc->num_refs + 64) * sizeof(struct nv_ref *);
+
+      pc->refs = REALLOC(pc->refs, old_size, new_size);
+
+      ref = CALLOC(64, sizeof(struct nv_ref));
+      for (i = 0; i < 64; ++i)
+         pc->refs[pc->num_refs + i] = &ref[i];
+   }
+
+   ref = pc->refs[pc->num_refs++];
+   ref->value = val;
+
+   LIST_INITHEAD(&ref->list);
+
+   ++val->refc;
+   return ref;
+}
+
+static INLINE struct nv_basic_block *
+new_basic_block(struct nv_pc *pc)
+{
+   struct nv_basic_block *bb;
+
+   if (pc->num_blocks >= NV_PC_MAX_BASIC_BLOCKS)
+      return NULL;
+
+   bb = CALLOC_STRUCT(nv_basic_block);
+
+   bb->id = pc->num_blocks;
+   pc->bb_list[pc->num_blocks++] = bb;
+   return bb;
+}
+
+static INLINE void
+nv_reference(struct nv_pc *pc,
+             struct nv_instruction *nvi, int c, struct nv_value *s)
+{
+   struct nv_ref **d = &nvi->src[c];
+   assert(c < 6);
+
+   if (*d) {
+      --(*d)->value->refc;
+      LIST_DEL(&(*d)->list);
+   }
+
+   if (s) {
+      if (!*d) {
+         *d = new_ref(pc, s);
+         (*d)->insn = nvi;
+      } else {
+         LIST_DEL(&(*d)->list);
+         (*d)->value = s;
+         ++(s->refc);
+      }
+      if (!s->last_use)
+         s->last_use = *d;
+      else
+         LIST_ADDTAIL(&s->last_use->list, &(*d)->list);
+
+      s->last_use = *d;
+      (*d)->insn = nvi;
+   } else {
+      *d = NULL;
+   }
+}
+
+/* nvc0_emit.c */
+void nvc0_emit_instruction(struct nv_pc *, struct nv_instruction *);
+
+/* nvc0_print.c */
+const char *nvc0_opcode_name(uint opcode);
+void nvc0_print_instruction(struct nv_instruction *);
+
+/* nvc0_pc.c */
+void nvc0_print_function(struct nv_basic_block *root);
+void nvc0_print_program(struct nv_pc *);
+
+boolean nvc0_insn_can_load(struct nv_instruction *, int s,
+                           struct nv_instruction *);
+boolean nvc0_insn_is_predicateable(struct nv_instruction *);
+
+int nvc0_insn_refcount(struct nv_instruction *);
+void nvc0_insn_delete(struct nv_instruction *);
+void nvc0_insns_permute(struct nv_instruction *prev, struct nv_instruction *);
+
+void nvc0_bblock_attach(struct nv_basic_block *parent,
+                        struct nv_basic_block *child, ubyte edge_kind);
+boolean nvc0_bblock_dominated_by(struct nv_basic_block *,
+                                 struct nv_basic_block *);
+boolean nvc0_bblock_reachable_by(struct nv_basic_block *future,
+                                 struct nv_basic_block *past,
+                                 struct nv_basic_block *final);
+struct nv_basic_block *nvc0_bblock_dom_frontier(struct nv_basic_block *);
+
+int nvc0_pc_replace_value(struct nv_pc *pc,
+                          struct nv_value *old_val,
+                          struct nv_value *new_val);
+
+struct nv_value *nvc0_pc_find_immediate(struct nv_ref *);
+struct nv_value *nvc0_pc_find_constant(struct nv_ref *);
+
+typedef void (*nv_pc_pass_func)(void *priv, struct nv_basic_block *b);
+
+void nvc0_pc_pass_in_order(struct nv_basic_block *, nv_pc_pass_func, void *);
+
+int nvc0_pc_exec_pass0(struct nv_pc *pc);
+int nvc0_pc_exec_pass1(struct nv_pc *pc);
+int nvc0_pc_exec_pass2(struct nv_pc *pc);
+
+int nvc0_tgsi_to_nc(struct nv_pc *, struct nvc0_translation_info *);
+
+#endif // NV50_COMPILER_H
diff --git a/src/gallium/drivers/nvc0/nvc0_pc_emit.c b/src/gallium/drivers/nvc0/nvc0_pc_emit.c
new file mode 100644
index 00000000000..db8055d91cd
--- /dev/null
+++ b/src/gallium/drivers/nvc0/nvc0_pc_emit.c
@@ -0,0 +1,979 @@
+/*
+ * Copyright 2010 Christoph Bumiller
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF
+ * OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "nvc0_pc.h"
+#include "nvc0_program.h"
+
+#define NVC0_FIXUP_CODE_RELOC 0
+#define NVC0_FIXUP_DATA_RELOC 1
+
+struct nvc0_fixup {
+   uint8_t type;
+   int8_t shift;
+   uint32_t mask;
+   uint32_t data;
+   uint32_t ofst;
+};
+
+void
+nvc0_relocate_program(struct nvc0_program *prog,
+                      uint32_t code_base,
+                      uint32_t data_base)
+{
+   struct nvc0_fixup *f = (struct nvc0_fixup *)prog->relocs;
+   unsigned i;
+
+   for (i = 0; i < prog->num_relocs; ++i) {
+      uint32_t data;
+
+      switch (f[i].type) {
+      case NVC0_FIXUP_CODE_RELOC: data = code_base + f[i].data; break;
+      case NVC0_FIXUP_DATA_RELOC: data = data_base + f[i].data; break;
+      default:
+         data = f[i].data;
+         break;
+      }
+      data = (f[i].shift < 0) ? (data >> -f[i].shift) : (data << f[i].shift);
+
+      prog->code[f[i].ofst / 4] &= ~f[i].mask;
+      prog->code[f[i].ofst / 4] |= data & f[i].mask;
+   }
+}
+
+static void
+create_fixup(struct nv_pc *pc, uint8_t ty,
+             int w, uint32_t data, uint32_t m, int s)
+{
+   struct nvc0_fixup *f;
+
+   const unsigned size = sizeof(struct nvc0_fixup);
+   const unsigned n = pc->num_relocs;
+
+   if (!(n % 8))
+      pc->reloc_entries = REALLOC(pc->reloc_entries, n * size, (n + 8) * size);
+
+   f = (struct nvc0_fixup *)pc->reloc_entries;
+
+   f[n].ofst = pc->emit_pos + w * 4;
+   f[n].type = ty;
+   f[n].data = data;
+   f[n].mask = m;
+   f[n].shift = s;
+
+   ++pc->num_relocs;
+}
+
+static INLINE ubyte
+SSIZE(struct nv_instruction *nvi, int s)
+{
+   return nvi->src[s]->value->reg.size;
+}
+
+static INLINE ubyte
+DSIZE(struct nv_instruction *nvi, int d)
+{
+   return nvi->def[d]->reg.size;
+}
+
+static INLINE struct nv_reg *
+SREG(struct nv_ref *ref)
+{
+   if (!ref)
+      return NULL;
+   return &ref->value->join->reg;
+}
+
+static INLINE struct nv_reg *
+DREG(struct nv_value *val)
+{
+   if (!val)
+      return NULL;
+   return &val->join->reg;
+}
+
+static INLINE ubyte
+SFILE(struct nv_instruction *nvi, int s)
+{
+   return nvi->src[s]->value->reg.file;
+}
+
+static INLINE ubyte
+DFILE(struct nv_instruction *nvi, int d)
+{
+   return nvi->def[0]->reg.file;
+}
+
+static INLINE void
+SID(struct nv_pc *pc, struct nv_ref *ref, int pos)
+{
+   pc->emit[pos / 32] |= (SREG(ref) ? SREG(ref)->id : 63) << (pos % 32);
+}
+
+static INLINE void
+DID(struct nv_pc *pc, struct nv_value *val, int pos)
+{
+   pc->emit[pos / 32] |= (DREG(val) ? DREG(val)->id : 63) << (pos % 32);
+}
+
+static INLINE uint32_t
+get_immd_u32(struct nv_ref *ref) /* XXX: dependent on [0]:2 */
+{
+   assert(ref->value->reg.file == NV_FILE_IMM);
+   return ref->value->reg.imm.u32;
+}
+
+static INLINE void
+set_immd_u32_l(struct nv_pc *pc, uint32_t u32)
+{
+   pc->emit[0] |= (u32 & 0x3f) << 26;
+   pc->emit[1] |= u32 >> 6;
+}
+
+static INLINE void
+set_immd_u32(struct nv_pc *pc, uint32_t u32)
+{
+   if ((pc->emit[0] & 0xf) == 0x2) {
+      set_immd_u32_l(pc, u32);
+   } else
+   if ((pc->emit[0] & 0xf) == 0x3) {
+      assert(!(pc->emit[1] & 0xc000));
+      pc->emit[1] |= 0xc000;
+      assert(!(u32 & 0xfff00000));
+      set_immd_u32_l(pc, u32);
+   } else {
+      assert(!(pc->emit[1] & 0xc000));
+      pc->emit[1] |= 0xc000;
+      assert(!(u32 & 0xfff));
+      set_immd_u32_l(pc, u32 >> 12);
+   }
+}
+
+static INLINE void
+set_immd(struct nv_pc *pc, struct nv_instruction *i, int s)
+{
+   set_immd_u32(pc, get_immd_u32(i->src[s]));
+}
+
+static INLINE void
+DVS(struct nv_pc *pc, struct nv_instruction *i)
+{
+   uint s = i->def[0]->reg.size;
+   int n;
+   for (n = 1; n < 4 && i->def[n]; ++n)
+      s += i->def[n]->reg.size;
+   pc->emit[0] |= ((s / 4) - 1) << 5;
+}
+
+static INLINE void
+SVS(struct nv_pc *pc, struct nv_ref *src)
+{
+   pc->emit[0] |= (SREG(src)->size / 4 - 1) << 5;
+}
+
+static void
+set_pred(struct nv_pc *pc, struct nv_instruction *i)
+{
+   if (i->predicate >= 0) {
+      SID(pc, i->src[i->predicate], 6);
+      if (i->cc)
+         pc->emit[0] |= 0x2000; /* negate */
+   } else {
+      pc->emit[0] |= 0x1c00;
+   }	   
+}
+
+static INLINE void
+set_address_16(struct nv_pc *pc, struct nv_ref *src)
+{
+   pc->emit[0] |= (src->value->reg.address & 0x003f) << 26;
+   pc->emit[1] |= (src->value->reg.address & 0xffc0) >> 6;
+}
+
+static INLINE unsigned
+const_space_index(struct nv_instruction *i, int s)
+{
+   return SFILE(i, s) - NV_FILE_MEM_C(0);
+}
+
+static void
+emit_flow(struct nv_pc *pc, struct nv_instruction *i, uint8_t op)
+{
+   pc->emit[0] = 0x00000007;
+   pc->emit[1] = op << 24;
+
+   if (op == 0x40 || (op >= 0x80 && op <= 0x98)) {
+      /* bra, exit, ret or kil */
+      pc->emit[0] |= 0x1e0;
+      set_pred(pc, i);
+   }
+
+   if (i->target) {
+      int32_t pcrel = i->target->emit_pos - (pc->emit_pos + 8);
+
+      /* we will need relocations only for global functions */
+      /*
+      create_fixup(pc, NVC0_FIXUP_CODE_RELOC, 0, pos, 26, 0xfc000000);
+      create_fixup(pc, NVC0_FIXUP_CODE_RELOC, 1, pos, -6, 0x0001ffff);
+      */
+
+      pc->emit[0] |= (pcrel & 0x3f) << 26;
+      pc->emit[1] |= (pcrel >> 6) & 0x1ffff;
+   }
+}
+
+/* doesn't work for vfetch, export, ld, st, mov ... */
+static void
+emit_form_0(struct nv_pc *pc, struct nv_instruction *i)
+{
+   int s;
+
+   set_pred(pc, i);
+
+   DID(pc, i->def[0], 14);
+
+   for (s = 0; s < 3 && i->src[s]; ++s) {
+      if (SFILE(i, s) >= NV_FILE_MEM_C(0) &&
+          SFILE(i, s) <= NV_FILE_MEM_C(15)) {
+         assert(!(pc->emit[1] & 0xc000));
+         assert(s <= 1);
+         pc->emit[1] |= 0x4000 | (const_space_index(i, s) << 10);
+         set_address_16(pc, i->src[s]);
+      } else
+      if (SFILE(i, s) == NV_FILE_GPR) {
+         SID(pc, i->src[s], s ? ((s == 2) ? 49 : 26) : 20);
+      } else
+      if (SFILE(i, s) == NV_FILE_IMM) {
+         assert(!(pc->emit[1] & 0xc000));
+         assert(s == 1 || i->opcode == NV_OP_MOV);
+         set_immd(pc, i, s);
+      }
+   }
+}
+
+static void
+emit_form_1(struct nv_pc *pc, struct nv_instruction *i)
+{
+   int s;
+
+   set_pred(pc, i);
+
+   DID(pc, i->def[0], 14);
+
+   for (s = 0; s < 1 && i->src[s]; ++s) {
+      if (SFILE(i, s) >= NV_FILE_MEM_C(0) &&
+          SFILE(i, s) <= NV_FILE_MEM_C(15)) {
+         assert(!(pc->emit[1] & 0xc000));
+         assert(s <= 1);
+         pc->emit[1] |= 0x4000 | (const_space_index(i, s) << 10);
+         set_address_16(pc, i->src[s]);
+      } else
+      if (SFILE(i, s) == NV_FILE_GPR) {
+         SID(pc, i->src[s], 26);
+      } else
+      if (SFILE(i, s) == NV_FILE_IMM) {
+         assert(!(pc->emit[1] & 0xc000));
+         assert(s == 1 || i->opcode == NV_OP_MOV);
+         set_immd(pc, i, s);
+      }
+   }
+}
+
+static void
+emit_neg_abs_1_2(struct nv_pc *pc, struct nv_instruction *i)
+{
+   if (i->src[0]->mod & NV_MOD_ABS)
+      pc->emit[0] |= 1 << 7;
+   if (i->src[0]->mod & NV_MOD_NEG)
+      pc->emit[0] |= 1 << 9;
+   if (i->src[1]->mod & NV_MOD_ABS)
+      pc->emit[0] |= 1 << 6;
+   if (i->src[1]->mod & NV_MOD_NEG)
+      pc->emit[0] |= 1 << 8;
+}
+
+static void
+emit_add_f32(struct nv_pc *pc, struct nv_instruction *i)
+{
+   pc->emit[0] = 0x00000000;
+   pc->emit[1] = 0x50000000;
+
+   emit_form_0(pc, i);
+
+   emit_neg_abs_1_2(pc, i);
+
+   if (i->saturate)
+      pc->emit[1] |= 1 << 17;
+}
+
+static void
+emit_mul_f32(struct nv_pc *pc, struct nv_instruction *i)
+{
+   pc->emit[0] = 0x00000000;
+   pc->emit[1] = 0x58000000;
+
+   emit_form_0(pc, i);
+
+   if ((i->src[0]->mod ^ i->src[1]->mod) & NV_MOD_NEG)
+      pc->emit[1] |= 1 << 25;
+
+   if (i->saturate)
+      pc->emit[0] |= 1 << 5;
+}
+
+static void
+emit_mad_f32(struct nv_pc *pc, struct nv_instruction *i)
+{
+   pc->emit[0] = 0x00000000;
+   pc->emit[1] = 0x30000000;
+
+   emit_form_0(pc, i);
+
+   if ((i->src[0]->mod ^ i->src[1]->mod) & NV_MOD_NEG)
+      pc->emit[0] |= 1 << 9;
+
+   if (i->src[2]->mod & NV_MOD_NEG)
+      pc->emit[0] |= 1 << 8;
+
+   if (i->saturate)
+      pc->emit[0] |= 1 << 5;
+}
+
+static void
+emit_minmax(struct nv_pc *pc, struct nv_instruction *i)
+{
+   pc->emit[0] = 0x00000000;
+   pc->emit[1] = 0x08000000;
+
+   if (NV_BASEOP(i->opcode) == NV_OP_MAX)
+      pc->emit[1] |= 0x001e0000;
+   else
+      pc->emit[1] |= 0x000e0000; /* predicate ? */
+
+   emit_form_0(pc, i);
+
+   emit_neg_abs_1_2(pc, i);
+
+   switch (i->opcode) {
+   case NV_OP_MIN_U32:
+   case NV_OP_MAX_U32:
+      pc->emit[0] |= 3;
+      break;
+   case NV_OP_MIN_S32:
+   case NV_OP_MAX_S32:
+      pc->emit[0] |= 3 | (1 << 5);
+      break;
+   case NV_OP_MIN_F32:
+   case NV_OP_MAX_F32:
+   default:
+      break;
+   }
+}
+
+static void
+emit_tex(struct nv_pc *pc, struct nv_instruction *i)
+{
+   int src1 = i->tex_array + i->tex_dim + i->tex_cube;
+
+   pc->emit[0] = 0x00000086;
+   pc->emit[1] = 0x80000000;
+
+   switch (i->opcode) {
+   case NV_OP_TEX: pc->emit[1] = 0x80000000; break;
+   case NV_OP_TXB: pc->emit[1] = 0x84000000; break;
+   case NV_OP_TXL: pc->emit[1] = 0x86000000; break;
+   case NV_OP_TXF: pc->emit[1] = 0x90000000; break;
+   case NV_OP_TXG: pc->emit[1] = 0xe0000000; break;
+   default:
+      assert(0);
+      break;
+   }
+
+   if (i->tex_array)
+      pc->emit[1] |= 0x00080000; /* layer index is u16, first value of SRC0 */
+   if (i->tex_shadow)
+      pc->emit[1] |= 0x01000000; /* shadow is part of SRC1, after bias/lod */
+
+   set_pred(pc, i);
+
+   DID(pc, i->def[0], 14);
+   SID(pc, i->src[0], 20);
+   SID(pc, i->src[src1], 26); /* may be NULL -> $r63 */
+
+   pc->emit[1] |= i->tex_mask << 14;
+   pc->emit[1] |= (i->tex_dim - 1) << 20;
+   if (i->tex_cube)
+      pc->emit[1] |= 3 << 20;
+
+   assert(i->ext.tex.s < 16);
+
+   pc->emit[1] |= i->ext.tex.t;
+   pc->emit[1] |= i->ext.tex.s << 8;
+
+   if (i->tex_live)
+      pc->emit[0] |= 1 << 9;
+}
+
+/* 0: cos, 1: sin, 2: ex2, 3: lg2, 4: rcp, 5: rsqrt */
+static void
+emit_flop(struct nv_pc *pc, struct nv_instruction *i, ubyte op)
+{
+   pc->emit[0] = 0x00000000;
+   pc->emit[1] = 0xc8000000;
+
+   set_pred(pc, i);
+
+   DID(pc, i->def[0], 14);
+   SID(pc, i->src[0], 20);
+
+   pc->emit[0] |= op << 26;
+
+   if (op >= 4) {
+      if (i->src[0]->mod & NV_MOD_NEG) pc->emit[0] |= 1 << 9;
+      if (i->src[0]->mod & NV_MOD_ABS) pc->emit[0] |= 1 << 7;
+   } else {
+      assert(!i->src[0]->mod);
+   }
+}
+
+static void
+emit_quadop(struct nv_pc *pc, struct nv_instruction *i)
+{
+   pc->emit[0] = 0x00000000;
+   pc->emit[1] = 0x48000000;
+
+   set_pred(pc, i);
+
+   assert(SFILE(i, 0) == NV_FILE_GPR && SFILE(i, 1) == NV_FILE_GPR);
+
+   DID(pc, i->def[0], 14);
+   SID(pc, i->src[0], 20);
+   SID(pc, i->src[0], 26);
+
+   pc->emit[0] |= i->lanes << 6; /* l0, l1, l2, l3, dx, dy */
+   pc->emit[1] |= i->quadop;
+}
+
+static void
+emit_ddx(struct nv_pc *pc, struct nv_instruction *i)
+{
+   i->quadop = 0x99;
+   i->lanes = 4;
+   emit_quadop(pc, i);
+}
+
+static void
+emit_ddy(struct nv_pc *pc, struct nv_instruction *i)
+{
+   i->quadop = 0xa5;
+   i->lanes = 5;
+   emit_quadop(pc, i);
+}
+
+/* preparation op (preex2, presin / convert to fixed point) */
+static void
+emit_preop(struct nv_pc *pc, struct nv_instruction *i)
+{
+   pc->emit[0] = 0x00000000;
+   pc->emit[1] = 0x60000000;
+
+   if (i->opcode == NV_OP_PREEX2)
+      pc->emit[0] |= 0x20;
+
+   emit_form_1(pc, i);
+
+   if (i->src[0]->mod & NV_MOD_NEG) pc->emit[0] |= 1 << 8;
+   if (i->src[0]->mod & NV_MOD_ABS) pc->emit[0] |= 1 << 6;
+}
+
+static void
+emit_shift(struct nv_pc *pc, struct nv_instruction *i)
+{
+   pc->emit[0] = 0x00000003;
+
+   switch (i->opcode) {
+   case NV_OP_SAR:
+      pc->emit[0] |= 0x20; /* fall through */
+   case NV_OP_SHR:
+      pc->emit[1] = 0x58000000;
+      break;
+   case NV_OP_SHL:
+   default:
+      pc->emit[1] = 0x60000000;
+      break;
+   }
+
+   emit_form_0(pc, i);
+}
+
+static void
+emit_bitop(struct nv_pc *pc, struct nv_instruction *i)
+{
+   if (SFILE(i, 1) == NV_FILE_IMM) {
+      pc->emit[0] = 0x00000002;
+      pc->emit[1] = 0x38000000;
+   } else {
+      pc->emit[0] = 0x00000003;
+      pc->emit[1] = 0x68000000;
+   }
+   
+   switch (i->opcode) {
+   case NV_OP_OR:
+      pc->emit[0] |= 0x40;
+      break;
+   case NV_OP_XOR:
+      pc->emit[0] |= 0x80;
+      break;
+   case NV_OP_AND:
+   default:
+      break;
+   }
+
+   emit_form_0(pc, i);
+}
+
+static void
+emit_set(struct nv_pc *pc, struct nv_instruction *i)
+{
+   pc->emit[0] = 0x00000000;
+
+   switch (i->opcode) {
+   case NV_OP_SET_S32:
+      pc->emit[0] |= 0x20; /* fall through */
+   case NV_OP_SET_U32:
+      pc->emit[0] |= 0x3;
+      pc->emit[1] = 0x100e0000;
+      break;
+   case NV_OP_SET_F32_AND:
+      pc->emit[1] = 0x18000000;
+      break;
+   case NV_OP_SET_F32_OR:
+      pc->emit[1] = 0x18200000;
+      break;
+   case NV_OP_SET_F32_XOR:
+      pc->emit[1] = 0x18400000;
+      break;
+   case NV_OP_FSET_F32:
+      pc->emit[0] |= 0x20; /* fall through */
+   case NV_OP_SET_F32:
+   default:
+      pc->emit[1] = 0x180e0000;
+      break;
+   }
+
+   if (DFILE(i, 0) == NV_FILE_PRED) {
+      pc->emit[0] |= 0x1c000;
+      pc->emit[1] += 0x08000000;
+   }
+
+   pc->emit[1] |= i->set_cond << 23;
+
+   emit_form_0(pc, i);
+
+   emit_neg_abs_1_2(pc, i); /* maybe assert that U/S32 don't use mods */
+}
+
+static void
+emit_selp(struct nv_pc *pc, struct nv_instruction *i)
+{
+   pc->emit[0] = 0x00000004;
+   pc->emit[1] = 0x20000000;
+
+   emit_form_0(pc, i);
+
+   if (i->cc || (i->src[2]->mod & NV_MOD_NOT))
+      pc->emit[1] |= 1 << 20;
+}
+
+static void
+emit_slct(struct nv_pc *pc, struct nv_instruction *i)
+{
+   pc->emit[0] = 0x00000000;
+
+   switch (i->opcode) {
+   case NV_OP_SLCT_S32:
+      pc->emit[0] |= 0x20; /* fall through */
+   case NV_OP_SLCT_U32:
+      pc->emit[0] |= 0x3;
+      pc->emit[1] = 0x30000000;
+      break;
+   case NV_OP_SLCT_F32:
+   default:
+      pc->emit[1] = 0x38000000;
+      break;
+   }
+
+   emit_form_0(pc, i);
+
+   pc->emit[1] |= i->set_cond << 23;
+}
+
+static void
+emit_cvt(struct nv_pc *pc, struct nv_instruction *i)
+{
+   pc->emit[0] = 0x00000004;
+   pc->emit[1] = 0x10000000;
+
+   if (i->opcode != NV_OP_CVT)
+      i->ext.cvt.d = i->ext.cvt.s = NV_OPTYPE(i->opcode);
+
+   switch (i->ext.cvt.d) {
+   case NV_TYPE_F32:
+      switch (i->ext.cvt.s) {
+      case NV_TYPE_F32: pc->emit[1] = 0x10000000; break;
+      case NV_TYPE_S32: pc->emit[0] |= 0x200;
+      case NV_TYPE_U32: pc->emit[1] = 0x18000000; break;
+      }
+      break;
+   case NV_TYPE_S32: pc->emit[0] |= 0x80;
+   case NV_TYPE_U32:
+      switch (i->ext.cvt.s) {
+      case NV_TYPE_F32: pc->emit[1] = 0x14000000; break;
+      case NV_TYPE_S32: pc->emit[0] |= 0x200;
+      case NV_TYPE_U32: pc->emit[1] = 0x1c000000; break;
+      }
+      break;
+   default:
+      assert(!"cvt: unknown type");
+      break;
+   }
+
+   if (i->opcode == NV_OP_FLOOR)
+      pc->emit[1] |= 0x00020000;
+   else
+   if (i->opcode == NV_OP_CEIL)
+      pc->emit[1] |= 0x00040000;
+   else
+   if (i->opcode == NV_OP_TRUNC)
+      pc->emit[1] |= 0x00060000;
+
+   if (i->saturate || i->opcode == NV_OP_SAT)
+      pc->emit[0] |= 0x20;
+
+   if (NV_BASEOP(i->opcode) == NV_OP_ABS || i->src[0]->mod & NV_MOD_ABS)
+      pc->emit[0] |= 1 << 6;
+   if (NV_BASEOP(i->opcode) == NV_OP_NEG || i->src[0]->mod & NV_MOD_NEG)
+      pc->emit[0] |= 1 << 8;
+
+   pc->emit[0] |= util_logbase2(DREG(i->def[0])->size) << 20;
+   pc->emit[0] |= util_logbase2(SREG(i->src[0])->size) << 23;
+
+   emit_form_1(pc, i);
+}
+
+static void
+emit_interp(struct nv_pc *pc, struct nv_instruction *i)
+{
+   pc->emit[0] = 0x00000000;
+   pc->emit[1] = 0xc07e0000;
+
+   DID(pc, i->def[0], 14);
+
+   set_pred(pc, i);
+
+   if (i->indirect)
+      SID(pc, i->src[i->indirect], 20);
+   else
+      SID(pc, NULL, 20);
+
+   if (i->opcode == NV_OP_PINTERP) {
+      pc->emit[0] |= 0x040;
+      SID(pc, i->src[1], 26);
+   } else {
+      SID(pc, NULL, 26);
+   }
+
+   pc->emit[1] |= i->src[0]->value->reg.address & 0xffff;
+
+   if (i->centroid)
+      pc->emit[0] |= 0x100;
+   else
+   if (i->flat)
+      pc->emit[0] |= 0x080;
+}
+
+static void
+emit_vfetch(struct nv_pc *pc, struct nv_instruction *i)
+{
+   pc->emit[0] = 0x03f00006;
+   pc->emit[1] = 0x06000000 | i->src[0]->value->reg.address;
+   if (i->patch)
+      pc->emit[0] |= 0x100;
+
+   set_pred(pc, i);
+
+   DVS(pc, i);
+   DID(pc, i->def[0], 14);
+
+   SID(pc, (i->indirect >= 0) ? i->src[i->indirect] : NULL, 26);
+}
+
+static void
+emit_export(struct nv_pc *pc, struct nv_instruction *i)
+{
+   pc->emit[0] = 0x00000006;
+   pc->emit[1] = 0x0a000000;
+   if (i->patch)
+      pc->emit[0] |= 0x100;
+
+   set_pred(pc, i);
+
+   assert(SFILE(i, 0) == NV_FILE_MEM_V);
+   assert(SFILE(i, 1) == NV_FILE_GPR);
+
+   SID(pc, i->src[1], 26); /* register source */
+   SVS(pc, i->src[0]);
+
+   pc->emit[1] |= i->src[0]->value->reg.address & 0xfff;
+
+   SID(pc, (i->indirect >= 0) ? i->src[i->indirect] : NULL, 20);
+}
+
+static void
+emit_mov(struct nv_pc *pc, struct nv_instruction *i)
+{
+   if (i->opcode == NV_OP_MOV)
+      i->lanes = 0xf;
+
+   if (SFILE(i, 0) == NV_FILE_IMM) {
+      pc->emit[0] = 0x000001e2;
+      pc->emit[1] = 0x18000000;
+   } else
+   if (SFILE(i, 0) == NV_FILE_PRED) {
+      pc->emit[0] = 0x1c000004;
+      pc->emit[1] = 0x080e0000;
+   } else {
+      pc->emit[0] = 0x00000004 | (i->lanes << 5);
+      pc->emit[1] = 0x28000000;
+   }
+
+   emit_form_1(pc, i);
+}
+
+static void
+emit_ldst_size(struct nv_pc *pc, struct nv_instruction *i)
+{
+   assert(NV_IS_MEMORY_FILE(SFILE(i, 0)));
+
+   switch (SSIZE(i, 0)) {
+   case 1:
+      if (NV_TYPE_ISSGD(i->ext.cvt.s))
+         pc->emit[0] |= 0x20;
+      break;
+   case 2:
+      pc->emit[0] |= 0x40;
+      if (NV_TYPE_ISSGD(i->ext.cvt.s))
+         pc->emit[0] |= 0x20;
+      break;
+   case 4: pc->emit[0] |= 0x80; break;
+   case 8: pc->emit[0] |= 0xa0; break;
+   case 16: pc->emit[0] |= 0xc0; break;
+   default:
+      NOUVEAU_ERR("invalid load/store size %u\n", SSIZE(i, 0));
+      break;
+   }
+}
+
+static void
+emit_ld_const(struct nv_pc *pc, struct nv_instruction *i)
+{
+   pc->emit[0] = 0x00000006;
+   pc->emit[1] = 0x14000000 | (const_space_index(i, 0) << 10);
+
+   emit_ldst_size(pc, i);
+
+   set_pred(pc, i);
+   set_address_16(pc, i->src[0]);
+
+   SID(pc, (i->indirect >= 0) ? i->src[i->indirect] : NULL, 20);
+   DID(pc, i->def[0], 14);
+}
+
+static void
+emit_ld(struct nv_pc *pc, struct nv_instruction *i)
+{
+   if (SFILE(i, 0) >= NV_FILE_MEM_C(0) &&
+       SFILE(i, 0) <= NV_FILE_MEM_C(15)) {
+      if (SSIZE(i, 0) == 4 && i->indirect < 0) {
+         i->lanes = 0xf;
+         emit_mov(pc, i);
+      } else {
+         emit_ld_const(pc, i);
+      }
+   } else {
+      NOUVEAU_ERR("emit_ld(%u): not handled yet\n", SFILE(i, 0));
+      abort();
+   }
+}
+
+static void
+emit_st(struct nv_pc *pc, struct nv_instruction *i)
+{
+   NOUVEAU_ERR("emit_st: not handled yet\n");
+   abort();
+}
+
+void
+nvc0_emit_instruction(struct nv_pc *pc, struct nv_instruction *i)
+{
+   debug_printf("EMIT: "); nvc0_print_instruction(i);
+
+   switch (i->opcode) {
+   case NV_OP_VFETCH:
+      emit_vfetch(pc, i);
+      break;
+   case NV_OP_EXPORT:
+      if (!pc->is_fragprog)
+         emit_export(pc, i);
+      break;
+   case NV_OP_MOV:
+      emit_mov(pc, i);
+      break;
+   case NV_OP_LD:
+      emit_ld(pc, i);
+      break;
+   case NV_OP_ST:
+      emit_st(pc, i);
+      break;
+   case NV_OP_LINTERP:
+   case NV_OP_PINTERP:
+      emit_interp(pc, i);
+      break;
+   case NV_OP_ADD_F32:
+      emit_add_f32(pc, i);
+      break;
+   case NV_OP_AND:
+   case NV_OP_OR:
+   case NV_OP_XOR:
+      emit_bitop(pc, i);
+      break;
+   case NV_OP_CVT:
+   case NV_OP_ABS_F32:
+   case NV_OP_ABS_S32:
+   case NV_OP_NEG_F32:
+   case NV_OP_NEG_S32:
+   case NV_OP_SAT:
+   case NV_OP_CEIL:
+   case NV_OP_FLOOR:
+   case NV_OP_TRUNC:
+      emit_cvt(pc, i);
+      break;
+   case NV_OP_DFDX:
+      emit_ddx(pc, i);
+      break;
+   case NV_OP_DFDY:
+      emit_ddy(pc, i);
+      break;
+   case NV_OP_COS:
+      emit_flop(pc, i, 0);
+      break;
+   case NV_OP_SIN:
+      emit_flop(pc, i, 1);
+      break;
+   case NV_OP_EX2:
+      emit_flop(pc, i, 2);
+      break;
+   case NV_OP_LG2:
+      emit_flop(pc, i, 3);
+      break;
+   case NV_OP_RCP:
+      emit_flop(pc, i, 4);
+      break;
+   case NV_OP_RSQ:
+      emit_flop(pc, i, 5);
+      break;
+   case NV_OP_PRESIN:
+   case NV_OP_PREEX2:
+      emit_preop(pc, i);
+      break;
+   case NV_OP_MAD_F32:
+      emit_mad_f32(pc, i);
+      break;
+   case NV_OP_MAX_F32:
+   case NV_OP_MAX_S32:
+   case NV_OP_MAX_U32:
+   case NV_OP_MIN_F32:
+   case NV_OP_MIN_S32:
+   case NV_OP_MIN_U32:
+      emit_minmax(pc, i);
+      break;
+   case NV_OP_MUL_F32:
+      emit_mul_f32(pc, i);
+      break;
+   case NV_OP_SET_F32:
+   case NV_OP_SET_F32_AND:
+   case NV_OP_SET_F32_OR:
+   case NV_OP_SET_F32_XOR:
+   case NV_OP_SET_S32:
+   case NV_OP_SET_U32:
+   case NV_OP_FSET_F32:
+      emit_set(pc, i);
+      break;
+   case NV_OP_SHL:
+   case NV_OP_SHR:
+   case NV_OP_SAR:
+      emit_shift(pc, i);
+      break;
+   case NV_OP_TEX:
+   case NV_OP_TXB:
+   case NV_OP_TXL:
+      emit_tex(pc, i);
+      break;
+   case NV_OP_BRA:
+      emit_flow(pc, i, 0x40);
+      break;
+   case NV_OP_CALL:
+      emit_flow(pc, i, 0x50);
+      break;
+   case NV_OP_JOINAT:
+      emit_flow(pc, i, 0x60);
+      break;
+   case NV_OP_EXIT:
+      emit_flow(pc, i, 0x80);
+      break;
+   case NV_OP_RET:
+      emit_flow(pc, i, 0x90);
+      break;
+   case NV_OP_KIL:
+      emit_flow(pc, i, 0x98);
+      break;
+   case NV_OP_JOIN:
+   case NV_OP_NOP:
+      pc->emit[0] = 0x00003de4;
+      pc->emit[1] = 0x40000000;
+      break;
+   case NV_OP_SELP:
+      emit_selp(pc, i);
+      break;
+   case NV_OP_SLCT_F32:
+   case NV_OP_SLCT_S32:
+   case NV_OP_SLCT_U32:
+      emit_slct(pc, i);
+      break;
+   default:
+      NOUVEAU_ERR("unhandled NV_OP: %d\n", i->opcode);
+      abort();
+      break;
+   }
+
+   if (i->join)
+      pc->emit[0] |= 0x10;
+}
diff --git a/src/gallium/drivers/nvc0/nvc0_pc_optimize.c b/src/gallium/drivers/nvc0/nvc0_pc_optimize.c
new file mode 100644
index 00000000000..e4449c285b5
--- /dev/null
+++ b/src/gallium/drivers/nvc0/nvc0_pc_optimize.c
@@ -0,0 +1,1192 @@
+/*
+ * Copyright 2010 Christoph Bumiller
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF
+ * OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "nvc0_pc.h"
+#include "nvc0_program.h"
+
+#define DESCEND_ARBITRARY(j, f)                                 \
+do {                                                            \
+   b->pass_seq = ctx->pc->pass_seq;                             \
+                                                                \
+   for (j = 0; j < 2; ++j)                                      \
+      if (b->out[j] && b->out[j]->pass_seq < ctx->pc->pass_seq) \
+         f(ctx, b->out[j]);	                                  \
+} while (0)
+
+static INLINE boolean
+registers_interfere(struct nv_value *a, struct nv_value *b)
+{
+   if (a->reg.file != b->reg.file)
+      return FALSE;
+   if (NV_IS_MEMORY_FILE(a->reg.file) || NV_IS_MEMORY_FILE(b->reg.file))
+      return FALSE;
+
+   assert(a->join->reg.id >= 0 && b->join->reg.id >= 0);
+
+   if (a->join->reg.id < b->join->reg.id) {
+      return (a->join->reg.id + a->reg.size >= b->join->reg.id);
+   } else
+   if (a->join->reg.id > b->join->reg.id) {
+      return (b->join->reg.id + b->reg.size >= a->join->reg.id);
+   }
+
+   return FALSE;
+}
+
+static INLINE boolean
+values_equal(struct nv_value *a, struct nv_value *b)
+{
+   if (a->reg.file != b->reg.file || a->reg.size != b->reg.size)
+      return FALSE;
+   if (NV_IS_MEMORY_FILE(a->reg.file))
+      return a->reg.address == b->reg.address;
+   else
+      return a->join->reg.id == b->join->reg.id;
+}
+
+#if 0
+static INLINE boolean
+inst_commutation_check(struct nv_instruction *a, struct nv_instruction *b)
+{
+   int si, di;
+
+   for (di = 0; di < 4 && a->def[di]; ++di)
+      for (si = 0; si < 5 && b->src[si]; ++si)
+         if (registers_interfere(a->def[di], b->src[si]->value))
+            return FALSE;
+
+   return TRUE;
+}
+
+/* Check whether we can swap the order of the instructions,
+ * where a & b may be either the earlier or the later one.
+ */
+static boolean
+inst_commutation_legal(struct nv_instruction *a, struct nv_instruction *b)
+{
+   return inst_commutation_check(a, b) && inst_commutation_check(b, a);
+}
+#endif
+
+static INLINE boolean
+inst_removable(struct nv_instruction *nvi)
+{
+   if (nvi->opcode == NV_OP_ST)
+      return FALSE;
+   return (!(nvi->terminator ||
+             nvi->join ||
+             nvi->target ||
+             nvi->fixed ||
+             nvc0_insn_refcount(nvi)));
+}
+
+static INLINE boolean
+inst_is_noop(struct nv_instruction *nvi)
+{
+   if (nvi->opcode == NV_OP_UNDEF || nvi->opcode == NV_OP_BIND)
+      return TRUE;
+   if (nvi->terminator || nvi->join)
+      return FALSE;
+   if (nvi->def[0] && nvi->def[0]->join->reg.id < 0)
+      return TRUE;
+   if (nvi->opcode != NV_OP_MOV && nvi->opcode != NV_OP_SELECT)
+      return FALSE;
+   if (nvi->def[0]->reg.file != nvi->src[0]->value->reg.file)
+      return FALSE;
+
+   if (nvi->src[0]->value->join->reg.id < 0) {
+      NOUVEAU_DBG("inst_is_noop: orphaned value detected\n");
+      return TRUE;
+   }
+
+   if (nvi->opcode == NV_OP_SELECT)
+      if (!values_equal(nvi->def[0], nvi->src[1]->value))
+         return FALSE;
+   return values_equal(nvi->def[0], nvi->src[0]->value);
+}
+
+struct nv_pass {
+   struct nv_pc *pc;
+   int n;
+   void *priv;
+};
+
+static int
+nv_pass_flatten(struct nv_pass *ctx, struct nv_basic_block *b);
+
+static void
+nv_pc_pass_pre_emission(void *priv, struct nv_basic_block *b)
+{
+   struct nv_pc *pc = (struct nv_pc *)priv;
+   struct nv_basic_block *in;
+   struct nv_instruction *nvi, *next;
+   int j;
+
+   for (j = pc->num_blocks - 1; j >= 0 && !pc->bb_list[j]->emit_size; --j);
+
+   if (j >= 0) {
+      in = pc->bb_list[j];
+
+      /* check for no-op branches (BRA $PC+8) */
+      if (in->exit && in->exit->opcode == NV_OP_BRA && in->exit->target == b) {
+         in->emit_size -= 8;
+         pc->emit_size -= 8;
+
+         for (++j; j < pc->num_blocks; ++j)
+            pc->bb_list[j]->emit_pos -= 8;
+
+         nvc0_insn_delete(in->exit);
+      }
+      b->emit_pos = in->emit_pos + in->emit_size;
+   }
+
+   pc->bb_list[pc->num_blocks++] = b;
+
+   /* visit node */
+
+   for (nvi = b->entry; nvi; nvi = next) {
+      next = nvi->next;
+      if (inst_is_noop(nvi) ||
+          (pc->is_fragprog && nvi->opcode == NV_OP_EXPORT)) {
+         nvc0_insn_delete(nvi);
+      } else
+         b->emit_size += 8;
+   }
+   pc->emit_size += b->emit_size;
+
+#ifdef NOUVEAU_DEBUG
+   if (!b->entry)
+      debug_printf("BB:%i is now empty\n", b->id);
+   else
+      debug_printf("BB:%i size = %u\n", b->id, b->emit_size);
+#endif
+}
+
+static int
+nv_pc_pass2(struct nv_pc *pc, struct nv_basic_block *root)
+{
+   struct nv_pass pass;
+
+   pass.pc = pc;
+
+   pc->pass_seq++;
+   nv_pass_flatten(&pass, root);
+
+   nvc0_pc_pass_in_order(root, nv_pc_pass_pre_emission, pc);
+
+   return 0;
+}
+
+int
+nvc0_pc_exec_pass2(struct nv_pc *pc)
+{
+   int i, ret;
+
+   NOUVEAU_DBG("preparing %u blocks for emission\n", pc->num_blocks);
+
+   pc->num_blocks = 0; /* will reorder bb_list */
+
+   for (i = 0; i < pc->num_subroutines + 1; ++i)
+      if (pc->root[i] && (ret = nv_pc_pass2(pc, pc->root[i])))
+         return ret;
+   return 0;
+}
+
+static INLINE boolean
+is_cspace_load(struct nv_instruction *nvi)
+{
+   if (!nvi)
+      return FALSE;
+   assert(nvi->indirect != 0);
+   return (nvi->opcode == NV_OP_LD &&
+           nvi->src[0]->value->reg.file >= NV_FILE_MEM_C(0) &&
+           nvi->src[0]->value->reg.file <= NV_FILE_MEM_C(15));
+}
+
+static INLINE boolean
+is_immd32_load(struct nv_instruction *nvi)
+{
+   if (!nvi)
+      return FALSE;
+   return (nvi->opcode == NV_OP_MOV &&
+           nvi->src[0]->value->reg.file == NV_FILE_IMM &&
+           nvi->src[0]->value->reg.size == 4);
+}
+
+static INLINE void
+check_swap_src_0_1(struct nv_instruction *nvi)
+{
+   static const uint8_t cc_swapped[8] = { 0, 4, 2, 6, 1, 5, 3, 7 };
+
+   struct nv_ref *src0 = nvi->src[0];
+   struct nv_ref *src1 = nvi->src[1];
+
+   if (!nv_op_commutative(nvi->opcode))
+      return;
+   assert(src0 && src1 && src0->value && src1->value);
+
+   if (is_cspace_load(src0->value->insn)) {
+      if (!is_cspace_load(src1->value->insn)) {
+         nvi->src[0] = src1;
+         nvi->src[1] = src0;
+      }
+   }
+
+   if (nvi->src[0] != src0 && nvi->opcode == NV_OP_SET)
+      nvi->set_cond = cc_swapped[nvi->set_cond];
+}
+
+static void
+nvi_set_indirect_load(struct nv_pc *pc,
+                      struct nv_instruction *nvi, struct nv_value *val)
+{
+   for (nvi->indirect = 0; nvi->indirect < 6 && nvi->src[nvi->indirect];
+        ++nvi->indirect);
+   assert(nvi->indirect < 6);
+   nv_reference(pc, nvi, nvi->indirect, val);
+}
+
+static int
+nvc0_pass_fold_loads(struct nv_pass *ctx, struct nv_basic_block *b)
+{
+   struct nv_instruction *nvi, *ld;
+   int s;
+
+   for (nvi = b->entry; nvi; nvi = nvi->next) {
+      check_swap_src_0_1(nvi);
+
+      for (s = 0; s < 3 && nvi->src[s]; ++s) {
+         ld = nvi->src[s]->value->insn;
+         if (!ld || (ld->opcode != NV_OP_LD && ld->opcode != NV_OP_MOV))
+            continue;
+         if (!nvc0_insn_can_load(nvi, s, ld))
+            continue;
+
+         /* fold it ! */
+         nv_reference(ctx->pc, nvi, s, ld->src[0]->value);
+         if (ld->indirect >= 0)
+            nvi_set_indirect_load(ctx->pc, nvi, ld->src[ld->indirect]->value);
+
+         if (!nvc0_insn_refcount(ld))
+            nvc0_insn_delete(ld);
+      }
+   }
+   DESCEND_ARBITRARY(s, nvc0_pass_fold_loads);
+
+   return 0;
+}
+
+/* NOTE: Assumes loads have not yet been folded. */
+static int
+nv_pass_lower_mods(struct nv_pass *ctx, struct nv_basic_block *b)
+{
+   struct nv_instruction *nvi, *mi, *next;
+   int j;
+   uint8_t mod;
+
+   for (nvi = b->entry; nvi; nvi = next) {
+      next = nvi->next;
+      if (nvi->opcode == NV_OP_SUB) {
+         nvi->src[1]->mod ^= NV_MOD_NEG;
+         nvi->opcode = NV_OP_ADD;
+      }
+
+      for (j = 0; j < 3 && nvi->src[j]; ++j) {
+         mi = nvi->src[j]->value->insn;
+         if (!mi)
+            continue;
+         if (mi->def[0]->refc > 1 || mi->predicate >= 0)
+            continue;
+
+         if (NV_BASEOP(mi->opcode) == NV_OP_NEG) mod = NV_MOD_NEG;
+         else
+         if (NV_BASEOP(mi->opcode) == NV_OP_ABS) mod = NV_MOD_ABS;
+         else
+            continue;
+         assert(!(mod & mi->src[0]->mod & NV_MOD_NEG));
+
+         mod |= mi->src[0]->mod;
+
+         if ((nvi->opcode == NV_OP_ABS) || (nvi->src[j]->mod & NV_MOD_ABS)) {
+            /* abs neg [abs] = abs */
+            mod &= ~(NV_MOD_NEG | NV_MOD_ABS);
+         } else
+         if ((nvi->opcode == NV_OP_NEG) && (mod & NV_MOD_NEG)) {
+            /* neg as opcode and modifier on same insn cannot occur */
+            /* neg neg abs = abs, neg neg = identity */
+            assert(j == 0);
+            if (mod & NV_MOD_ABS)
+               nvi->opcode = NV_OP_ABS;
+            else
+               nvi->opcode = NV_OP_MOV;
+            mod = 0;
+         }
+
+         if ((nv_op_supported_src_mods(nvi->opcode) & mod) != mod)
+            continue;
+
+         nv_reference(ctx->pc, nvi, j, mi->src[0]->value);
+
+         nvi->src[j]->mod ^= mod;
+      }
+
+      if (nvi->opcode == NV_OP_SAT) {
+         mi = nvi->src[0]->value->insn;
+
+         if (mi->def[0]->refc > 1 ||
+             (mi->opcode != NV_OP_ADD &&
+              mi->opcode != NV_OP_MUL &&
+              mi->opcode != NV_OP_MAD))
+            continue;
+         mi->saturate = 1;
+         mi->def[0] = nvi->def[0];
+         mi->def[0]->insn = mi;
+         nvc0_insn_delete(nvi);
+      }
+   }
+   DESCEND_ARBITRARY(j, nv_pass_lower_mods);
+
+   return 0;
+}
+
+#define SRC_IS_MUL(s) ((s)->insn && (s)->insn->opcode == NV_OP_MUL)
+
+static void
+apply_modifiers(uint32_t *val, uint8_t type, uint8_t mod)
+{
+   if (mod & NV_MOD_ABS) {
+      if (type == NV_TYPE_F32)
+         *val &= 0x7fffffff;
+      else
+      if ((*val) & (1 << 31))
+         *val = ~(*val) + 1;
+   }
+   if (mod & NV_MOD_NEG) {
+      if (type == NV_TYPE_F32)
+         *val ^= 0x80000000;
+      else
+         *val = ~(*val) + 1;
+   }
+   if (mod & NV_MOD_SAT) {
+      union {
+         float f;
+         uint32_t u;
+         int32_t i;
+      } u;
+      u.u = *val;
+      if (type == NV_TYPE_F32) {
+         u.f = CLAMP(u.f, -1.0f, 1.0f);
+      } else
+      if (type == NV_TYPE_U16) {
+         u.u = MIN2(u.u, 0xffff);
+      } else
+      if (type == NV_TYPE_S16) {
+         u.i = CLAMP(u.i, -32768, 32767);
+      }
+      *val = u.u;
+   }
+   if (mod & NV_MOD_NOT)
+      *val = ~*val;
+}
+
+static void
+constant_expression(struct nv_pc *pc, struct nv_instruction *nvi,
+                    struct nv_value *src0, struct nv_value *src1)
+{
+   struct nv_value *val;
+   union {
+      float f32;
+      uint32_t u32;
+      int32_t s32;
+   } u0, u1, u;
+   ubyte type;
+
+   if (!nvi->def[0])
+      return;
+   type = NV_OPTYPE(nvi->opcode);
+
+   u.u32 = 0;
+   u0.u32 = src0->reg.imm.u32;
+   u1.u32 = src1->reg.imm.u32;
+
+   apply_modifiers(&u0.u32, type, nvi->src[0]->mod);
+   apply_modifiers(&u1.u32, type, nvi->src[1]->mod);
+
+   switch (nvi->opcode) {
+   case NV_OP_MAD_F32:
+      if (nvi->src[2]->value->reg.file != NV_FILE_GPR)
+         return;
+      /* fall through */
+   case NV_OP_MUL_F32:
+      u.f32 = u0.f32 * u1.f32;
+      break;
+   case NV_OP_MUL_B32:
+      u.u32 = u0.u32 * u1.u32;
+      break;
+   case NV_OP_ADD_F32:
+      u.f32 = u0.f32 + u1.f32;
+      break;
+   case NV_OP_ADD_B32:
+      u.u32 = u0.u32 + u1.u32;
+      break;
+   case NV_OP_SUB_F32:
+      u.f32 = u0.f32 - u1.f32;
+      break;
+      /*
+   case NV_OP_SUB_B32:
+      u.u32 = u0.u32 - u1.u32;
+      break;
+      */
+   default:
+      return;
+   }
+
+   val = new_value(pc, NV_FILE_IMM, nv_type_sizeof(type));
+   val->reg.imm.u32 = u.u32;
+
+   nv_reference(pc, nvi, 1, NULL);
+   nv_reference(pc, nvi, 0, val);
+
+   if (nvi->opcode == NV_OP_MAD_F32) {
+      nvi->src[1] = nvi->src[0];
+      nvi->src[0] = nvi->src[2];
+      nvi->src[2] = NULL;
+      nvi->opcode = NV_OP_ADD_F32;
+
+      if (val->reg.imm.u32 == 0) {
+         nvi->src[1] = NULL;
+         nvi->opcode = NV_OP_MOV;
+      }
+   } else {
+      nvi->opcode = NV_OP_MOV;
+   }
+}
+
+static void
+constant_operand(struct nv_pc *pc,
+                 struct nv_instruction *nvi, struct nv_value *val, int s)
+{
+   union {
+      float f32;
+      uint32_t u32;
+      int32_t s32;
+   } u;
+   int shift;
+   int t = s ? 0 : 1;
+   uint op;
+   ubyte type;
+
+   if (!nvi->def[0])
+      return;
+   type = NV_OPTYPE(nvi->opcode);
+
+   u.u32 = val->reg.imm.u32;
+   apply_modifiers(&u.u32, type, nvi->src[s]->mod);
+
+   if (u.u32 == 0 && NV_BASEOP(nvi->opcode) == NV_OP_MUL) {
+      nvi->opcode = NV_OP_MOV;
+      nv_reference(pc, nvi, t, NULL);
+      if (s) {
+         nvi->src[0] = nvi->src[1];
+         nvi->src[1] = NULL;
+      }
+      return;
+   }
+
+   switch (nvi->opcode) {
+   case NV_OP_MUL_F32:
+      if (u.f32 == 1.0f || u.f32 == -1.0f) {
+         if (u.f32 == -1.0f)
+            nvi->src[t]->mod ^= NV_MOD_NEG;
+         switch (nvi->src[t]->mod) {
+         case 0: op = nvi->saturate ? NV_OP_SAT : NV_OP_MOV; break;
+         case NV_MOD_NEG: op = NV_OP_NEG_F32; break;
+         case NV_MOD_ABS: op = NV_OP_ABS_F32; break;
+         default:
+            return;
+         }
+         nvi->opcode = op;
+         nv_reference(pc, nvi, 0, nvi->src[t]->value);
+         nv_reference(pc, nvi, 1, NULL);
+         nvi->src[0]->mod = 0;
+      } else
+      if (u.f32 == 2.0f || u.f32 == -2.0f) {
+         if (u.f32 == -2.0f)
+            nvi->src[t]->mod ^= NV_MOD_NEG;
+         nvi->opcode = NV_OP_ADD_F32;
+         nv_reference(pc, nvi, s, nvi->src[t]->value);
+         nvi->src[s]->mod = nvi->src[t]->mod;
+      }
+   case NV_OP_ADD_F32:
+      if (u.u32 == 0) {
+         switch (nvi->src[t]->mod) {
+         case 0: op = nvi->saturate ? NV_OP_SAT : NV_OP_MOV; break;
+         case NV_MOD_NEG: op = NV_OP_NEG_F32; break;
+         case NV_MOD_ABS: op = NV_OP_ABS_F32; break;
+         case NV_MOD_NEG | NV_MOD_ABS:
+            op = NV_OP_CVT;
+            nvi->ext.cvt.s = nvi->ext.cvt.d = type;
+            break;
+         default:
+            return;
+         }
+         nvi->opcode = op;
+         nv_reference(pc, nvi, 0, nvi->src[t]->value);
+         nv_reference(pc, nvi, 1, NULL);
+         if (nvi->opcode != NV_OP_CVT)
+            nvi->src[0]->mod = 0;
+      }
+   case NV_OP_ADD_B32:
+      if (u.u32 == 0) {
+         assert(nvi->src[t]->mod == 0);
+         nvi->opcode = nvi->saturate ? NV_OP_CVT : NV_OP_MOV;
+         nvi->ext.cvt.s = nvi->ext.cvt.d = type;
+         nv_reference(pc, nvi, 0, nvi->src[t]->value);
+         nv_reference(pc, nvi, 1, NULL);
+      }
+      break;
+   case NV_OP_MUL_B32:
+      /* multiplication by 0 already handled above */
+      assert(nvi->src[s]->mod == 0);
+      shift = ffs(u.s32) - 1;
+      if (shift == 0) {
+         nvi->opcode = NV_OP_MOV;
+         nv_reference(pc, nvi, 0, nvi->src[t]->value);
+         nv_reference(pc, nvi, 1, NULL);
+      } else
+      if (u.s32 > 0 && u.s32 == (1 << shift)) {
+         nvi->opcode = NV_OP_SHL;
+         (val = new_value(pc, NV_FILE_IMM, NV_TYPE_U32))->reg.imm.s32 = shift;
+         nv_reference(pc, nvi, 0, nvi->src[t]->value);
+         nv_reference(pc, nvi, 1, val);
+         break;
+      }
+      break;
+   case NV_OP_RCP:
+      u.f32 = 1.0f / u.f32;
+      (val = new_value(pc, NV_FILE_IMM, NV_TYPE_F32))->reg.imm.f32 = u.f32;
+      nvi->opcode = NV_OP_MOV;
+      assert(s == 0);
+      nv_reference(pc, nvi, 0, val);
+      break;
+   case NV_OP_RSQ:
+      u.f32 = 1.0f / sqrtf(u.f32);
+      (val = new_value(pc, NV_FILE_IMM, NV_TYPE_F32))->reg.imm.f32 = u.f32;
+      nvi->opcode = NV_OP_MOV;
+      assert(s == 0);
+      nv_reference(pc, nvi, 0, val);
+      break;
+   default:
+      break;
+   }
+}
+
+static int
+nv_pass_lower_arith(struct nv_pass *ctx, struct nv_basic_block *b)
+{
+   struct nv_instruction *nvi, *next;
+   int j;
+
+   for (nvi = b->entry; nvi; nvi = next) {
+      struct nv_value *src0, *src1, *src;
+      int s;
+      uint8_t mod[4];
+
+      next = nvi->next;
+
+      src0 = nvc0_pc_find_immediate(nvi->src[0]);
+      src1 = nvc0_pc_find_immediate(nvi->src[1]);
+
+      if (src0 && src1)
+         constant_expression(ctx->pc, nvi, src0, src1);
+      else {
+         if (src0)
+            constant_operand(ctx->pc, nvi, src0, 0);
+         else
+         if (src1)
+            constant_operand(ctx->pc, nvi, src1, 1);
+      }
+
+      /* check if we can MUL + ADD -> MAD/FMA */
+      if (nvi->opcode != NV_OP_ADD)
+         continue;
+
+      src0 = nvi->src[0]->value;
+      src1 = nvi->src[1]->value;
+
+      if (SRC_IS_MUL(src0) && src0->refc == 1)
+         src = src0;
+      else
+      if (SRC_IS_MUL(src1) && src1->refc == 1)
+         src = src1;
+      else
+         continue;
+
+      /* could have an immediate from above constant_*  */
+      if (src0->reg.file != NV_FILE_GPR || src1->reg.file != NV_FILE_GPR)
+         continue;
+      s = (src == src0) ? 0 : 1;
+
+      mod[0] = nvi->src[0]->mod;
+      mod[1] = nvi->src[1]->mod;
+      mod[2] = src->insn->src[0]->mod;
+      mod[3] = src->insn->src[0]->mod;
+
+      if ((mod[0] | mod[1] | mod[2] | mod[3]) & ~NV_MOD_NEG)
+         continue;
+
+      nvi->opcode = NV_OP_MAD;
+      nv_reference(ctx->pc, nvi, s, NULL);
+      nvi->src[2] = nvi->src[!s];
+
+      nvi->src[0] = new_ref(ctx->pc, src->insn->src[0]->value);
+      nvi->src[1] = new_ref(ctx->pc, src->insn->src[1]->value);
+      nvi->src[0]->mod = mod[2] ^ mod[s];
+      nvi->src[1]->mod = mod[3];
+   }
+   DESCEND_ARBITRARY(j, nv_pass_lower_arith);
+
+   return 0;
+}
+
+/* TODO: redundant store elimination */
+
+struct mem_record {
+   struct mem_record *next;
+   struct nv_instruction *insn;
+   uint32_t ofst;
+   uint32_t base;
+   uint32_t size;
+};
+
+#define MEM_RECORD_POOL_SIZE 1024
+
+struct pass_reld_elim {
+   struct nv_pc *pc;
+
+   struct mem_record *imm;
+   struct mem_record *mem_v;
+   struct mem_record *mem_a;
+   struct mem_record *mem_c[16];
+   struct mem_record *mem_l;
+
+   struct mem_record pool[MEM_RECORD_POOL_SIZE];
+   int alloc;
+};
+
+static void
+combine_load(struct mem_record *rec, struct nv_instruction *ld)
+{
+   struct nv_instruction *fv = rec->insn;
+   struct nv_value *mem = ld->src[0]->value;
+   uint32_t size = rec->size + mem->reg.size;
+   int j;
+   int d = rec->size / 4;
+
+   assert(rec->size < 16);
+   if (rec->ofst > mem->reg.address) {
+      if ((size == 8 && mem->reg.address & 3) ||
+          (size > 8 && mem->reg.address & 7))
+         return;
+      rec->ofst = mem->reg.address;
+      for (j = 0; j < d; ++j)
+         fv->def[d + j] = fv->def[j];
+      d = 0;
+   } else
+   if ((size == 8 && rec->ofst & 3) ||
+       (size > 8 && rec->ofst & 7)) {
+      return;
+   }
+
+   for (j = 0; j < mem->reg.size / 4; ++j) {
+      fv->def[d] = ld->def[j];
+      fv->def[d++]->insn = fv;
+   }
+
+   fv->src[0]->value->reg.size = rec->size = size;
+
+   nvc0_insn_delete(ld);
+}
+
+static void
+combine_export(struct mem_record *rec, struct nv_instruction *ex)
+{
+
+}
+
+static INLINE void
+add_mem_record(struct pass_reld_elim *ctx, struct mem_record **rec,
+               uint32_t base, uint32_t ofst, struct nv_instruction *nvi)
+{
+   struct mem_record *it = &ctx->pool[ctx->alloc++];
+
+   it->next = *rec;
+   *rec = it;
+   it->base = base;
+   it->ofst = ofst;
+   it->insn = nvi;
+   it->size = nvi->src[0]->value->reg.size;
+}
+
+/* vectorize and reuse loads from memory or of immediates */
+static int
+nv_pass_mem_opt(struct pass_reld_elim *ctx, struct nv_basic_block *b)
+{
+   struct mem_record **rec, *it;
+   struct nv_instruction *ld, *next;
+   struct nv_value *mem;
+   uint32_t base, ofst;
+   int s;
+
+   for (ld = b->entry; ld; ld = next) {
+      next = ld->next;
+
+      if (is_cspace_load(ld)) {
+         mem = ld->src[0]->value;
+         rec = &ctx->mem_c[ld->src[0]->value->reg.file - NV_FILE_MEM_C(0)];
+      } else
+      if (ld->opcode == NV_OP_VFETCH) {
+         mem = ld->src[0]->value;
+         rec = &ctx->mem_a;
+      } else
+      if (ld->opcode == NV_OP_EXPORT) {
+         mem = ld->src[0]->value;
+         if (mem->reg.file != NV_FILE_MEM_V)
+            continue;
+         rec = &ctx->mem_v;
+      } else {
+         continue;
+      }
+      if (ld->def[0] && ld->def[0]->refc == 0)
+         continue;
+      ofst = mem->reg.address;
+      base = (ld->indirect >= 0) ? ld->src[ld->indirect]->value->n : 0;
+
+      for (it = *rec; it; it = it->next) {
+         if (it->base == base &&
+             ((it->ofst >> 4) == (ofst >> 4)) &&
+             ((it->ofst + it->size == ofst) ||
+              (it->ofst - mem->reg.size == ofst))) {
+            if (ld->opcode == NV_OP_LD && it->size + mem->reg.size == 12)
+               continue;
+            if (it->ofst < ofst) {
+               if ((it->ofst & 0xf) == 4)
+                  continue;
+            } else
+            if ((ofst & 0xf) == 4)
+               continue;
+            break;
+         }
+      }
+      if (it) {
+         switch (ld->opcode) {
+         case NV_OP_EXPORT: combine_export(it, ld); break;
+         default:
+            combine_load(it, ld);
+            break;
+         }
+      } else
+      if (ctx->alloc < MEM_RECORD_POOL_SIZE) {
+         add_mem_record(ctx, rec, base, ofst, ld);
+      }
+   }
+
+   DESCEND_ARBITRARY(s, nv_pass_mem_opt);
+   return 0;
+}
+
+static void
+eliminate_store(struct mem_record *rec, struct nv_instruction *st)
+{
+}
+
+/* elimination of redundant stores */
+static int
+pass_store_elim(struct pass_reld_elim *ctx, struct nv_basic_block *b)
+{
+   struct mem_record **rec, *it;
+   struct nv_instruction *st, *next;
+   struct nv_value *mem;
+   uint32_t base, ofst, size;
+   int s;
+
+   for (st = b->entry; st; st = next) {
+      next = st->next;
+
+      if (st->opcode == NV_OP_ST) {
+         mem = st->src[0]->value;
+         rec = &ctx->mem_l;
+      } else
+      if (st->opcode == NV_OP_EXPORT) {
+         mem = st->src[0]->value;
+         if (mem->reg.file != NV_FILE_MEM_V)
+            continue;
+         rec = &ctx->mem_v;
+      } else
+      if (st->opcode == NV_OP_ST) {
+         /* TODO: purge */
+      }
+      ofst = mem->reg.address;
+      base = (st->indirect >= 0) ? st->src[st->indirect]->value->n : 0;
+      size = mem->reg.size;
+
+      for (it = *rec; it; it = it->next) {
+         if (it->base == base &&
+             (it->ofst <= ofst && (it->ofst + size) > ofst))
+            break;
+      }
+      if (it)
+         eliminate_store(it, st);
+      else
+         add_mem_record(ctx, rec, base, ofst, st);
+   }
+
+   DESCEND_ARBITRARY(s, nv_pass_mem_opt);
+   return 0;
+}
+
+/* TODO: properly handle loads from l[] memory in the presence of stores */
+static int
+nv_pass_reload_elim(struct pass_reld_elim *ctx, struct nv_basic_block *b)
+{
+#if 0
+   struct load_record **rec, *it;
+   struct nv_instruction *ld, *next;
+   uint64_t data[2];
+   struct nv_value *val;
+   int j;
+
+   for (ld = b->entry; ld; ld = next) {
+      next = ld->next;
+      if (!ld->src[0])
+         continue;
+      val = ld->src[0]->value;
+      rec = NULL;
+
+      if (ld->opcode == NV_OP_LINTERP || ld->opcode == NV_OP_PINTERP) {
+         data[0] = val->reg.id;
+         data[1] = 0;
+         rec = &ctx->mem_v;
+      } else
+      if (ld->opcode == NV_OP_LDA) {
+         data[0] = val->reg.id;
+         data[1] = ld->src[4] ? ld->src[4]->value->n : ~0ULL;
+         if (val->reg.file >= NV_FILE_MEM_C(0) &&
+             val->reg.file <= NV_FILE_MEM_C(15))
+            rec = &ctx->mem_c[val->reg.file - NV_FILE_MEM_C(0)];
+         else
+         if (val->reg.file == NV_FILE_MEM_S)
+            rec = &ctx->mem_s;
+         else
+         if (val->reg.file == NV_FILE_MEM_L)
+            rec = &ctx->mem_l;
+      } else
+      if ((ld->opcode == NV_OP_MOV) && (val->reg.file == NV_FILE_IMM)) {
+         data[0] = val->reg.imm.u32;
+         data[1] = 0;
+         rec = &ctx->imm;
+      }
+
+      if (!rec || !ld->def[0]->refc)
+         continue;
+
+      for (it = *rec; it; it = it->next)
+         if (it->data[0] == data[0] && it->data[1] == data[1])
+            break;
+
+      if (it) {
+         if (ld->def[0]->reg.id >= 0)
+            it->value = ld->def[0];
+         else
+         if (!ld->fixed)
+            nvc0_pc_replace_value(ctx->pc, ld->def[0], it->value);
+      } else {
+         if (ctx->alloc == LOAD_RECORD_POOL_SIZE)
+            continue;
+         it = &ctx->pool[ctx->alloc++];
+         it->next = *rec;
+         it->data[0] = data[0];
+         it->data[1] = data[1];
+         it->value = ld->def[0];
+         *rec = it;
+      }
+   }
+
+   ctx->imm = NULL;
+   ctx->mem_s = NULL;
+   ctx->mem_v = NULL;
+   for (j = 0; j < 16; ++j)
+      ctx->mem_c[j] = NULL;
+   ctx->mem_l = NULL;
+   ctx->alloc = 0;
+
+   DESCEND_ARBITRARY(j, nv_pass_reload_elim);
+#endif
+   return 0;
+}
+
+static int
+nv_pass_tex_mask(struct nv_pass *ctx, struct nv_basic_block *b)
+{
+   int i, c, j;
+
+   for (i = 0; i < ctx->pc->num_instructions; ++i) {
+      struct nv_instruction *nvi = &ctx->pc->instructions[i];
+      struct nv_value *def[4];
+
+      if (!nv_is_texture_op(nvi->opcode))
+         continue;
+      nvi->tex_mask = 0;
+
+      for (c = 0; c < 4; ++c) {
+         if (nvi->def[c]->refc)
+            nvi->tex_mask |= 1 << c;
+         def[c] = nvi->def[c];
+      }
+
+      j = 0;
+      for (c = 0; c < 4; ++c)
+         if (nvi->tex_mask & (1 << c))
+            nvi->def[j++] = def[c];
+      for (c = 0; c < 4; ++c)
+         if (!(nvi->tex_mask & (1 << c)))
+           nvi->def[j++] = def[c];
+      assert(j == 4);
+   }
+   return 0;
+}
+
+struct nv_pass_dce {
+   struct nv_pc *pc;
+   uint removed;
+};
+
+static int
+nv_pass_dce(struct nv_pass_dce *ctx, struct nv_basic_block *b)
+{
+   int j;
+   struct nv_instruction *nvi, *next;
+
+   for (nvi = b->phi ? b->phi : b->entry; nvi; nvi = next) {
+      next = nvi->next;
+
+      if (inst_removable(nvi)) {
+         nvc0_insn_delete(nvi);
+         ++ctx->removed;
+      }
+   }
+   DESCEND_ARBITRARY(j, nv_pass_dce);
+
+   return 0;
+}
+
+#if 0
+/* Register allocation inserted ELSE blocks for all IF/ENDIF without ELSE.
+ * Returns TRUE if @bb initiates an IF/ELSE/ENDIF clause, or is an IF with
+ * BREAK and dummy ELSE block.
+ */
+static INLINE boolean
+bb_is_if_else_endif(struct nv_basic_block *bb)
+{
+   if (!bb->out[0] || !bb->out[1])
+      return FALSE;
+
+   if (bb->out[0]->out_kind[0] == CFG_EDGE_LOOP_LEAVE) {
+      return (bb->out[0]->out[1] == bb->out[1]->out[0] &&
+              !bb->out[1]->out[1]);
+   } else {
+      return (bb->out[0]->out[0] == bb->out[1]->out[0] &&
+              !bb->out[0]->out[1] &&
+              !bb->out[1]->out[1]);
+   }
+}
+
+/* predicate instructions and remove branch at the end */
+static void
+predicate_instructions(struct nv_pc *pc, struct nv_basic_block *b,
+                       struct nv_value *p, ubyte cc)
+{
+
+}
+#endif
+
+/* NOTE: Run this after register allocation, we can just cut out the cflow
+ * instructions and hook the predicates to the conditional OPs if they are
+ * not using immediates; better than inserting SELECT to join definitions.
+ *
+ * NOTE: Should adapt prior optimization to make this possible more often.
+ */
+static int
+nv_pass_flatten(struct nv_pass *ctx, struct nv_basic_block *b)
+{
+   return 0;
+}
+
+/* local common subexpression elimination, stupid O(n^2) implementation */
+static int
+nv_pass_cse(struct nv_pass *ctx, struct nv_basic_block *b)
+{
+   struct nv_instruction *ir, *ik, *next;
+   struct nv_instruction *entry = b->phi ? b->phi : b->entry;
+   int s;
+   unsigned int reps;
+
+   do {
+      reps = 0;
+      for (ir = entry; ir; ir = next) {
+         next = ir->next;
+         for (ik = entry; ik != ir; ik = ik->next) {
+            if (ir->opcode != ik->opcode || ir->fixed)
+               continue;
+
+            if (!ir->def[0] || !ik->def[0] || ir->def[1] || ik->def[1])
+               continue;
+
+            if (ik->indirect != ir->indirect || ik->predicate != ir->predicate)
+               continue;
+
+            if (!values_equal(ik->def[0], ir->def[0]))
+               continue;
+
+            for (s = 0; s < 3; ++s) {
+               struct nv_value *a, *b;
+
+               if (!ik->src[s]) {
+                  if (ir->src[s])
+                     break;
+                  continue;
+               }
+               if (ik->src[s]->mod != ir->src[s]->mod)
+                  break;
+               a = ik->src[s]->value;
+               b = ir->src[s]->value;
+               if (a == b)
+                  continue;
+               if (a->reg.file != b->reg.file ||
+                   a->reg.id < 0 ||
+                   a->reg.id != b->reg.id)
+                  break;
+            }
+            if (s == 3) {
+               nvc0_insn_delete(ir);
+               ++reps;
+               nvc0_pc_replace_value(ctx->pc, ir->def[0], ik->def[0]);
+               break;
+            }
+         }
+      }
+   } while(reps);
+
+   DESCEND_ARBITRARY(s, nv_pass_cse);
+
+   return 0;
+}
+
+static int
+nv_pc_pass0(struct nv_pc *pc, struct nv_basic_block *root)
+{
+   struct pass_reld_elim *reldelim;
+   struct nv_pass pass;
+   struct nv_pass_dce dce;
+   int ret;
+
+   pass.n = 0;
+   pass.pc = pc;
+
+   /* Do this first, so we don't have to pay attention
+    * to whether sources are supported memory loads.
+    */
+   pc->pass_seq++;
+   ret = nv_pass_lower_arith(&pass, root);
+   if (ret)
+      return ret;
+
+   pc->pass_seq++;
+   ret = nv_pass_lower_mods(&pass, root);
+   if (ret)
+      return ret;
+
+   pc->pass_seq++;
+   ret = nvc0_pass_fold_loads(&pass, root);
+   if (ret)
+      return ret;
+
+   if (pc->opt_reload_elim) {
+      reldelim = CALLOC_STRUCT(pass_reld_elim);
+      reldelim->pc = pc;
+
+      pc->pass_seq++;
+      ret = nv_pass_reload_elim(reldelim, root);
+      if (ret) {
+         FREE(reldelim);
+         return ret;
+      }
+      memset(reldelim, 0, sizeof(struct pass_reld_elim));
+      reldelim->pc = pc;
+   }
+
+   pc->pass_seq++;
+   ret = nv_pass_cse(&pass, root);
+   if (ret)
+      return ret;
+
+   dce.pc = pc;
+   do {
+      dce.removed = 0;
+      pc->pass_seq++;
+      ret = nv_pass_dce(&dce, root);
+      if (ret)
+         return ret;
+   } while (dce.removed);
+
+   if (pc->opt_reload_elim) {
+      pc->pass_seq++;
+      ret = nv_pass_mem_opt(reldelim, root);
+      if (!ret) {
+         memset(reldelim, 0, sizeof(struct pass_reld_elim));
+         reldelim->pc = pc;
+
+         pc->pass_seq++;
+         ret = nv_pass_mem_opt(reldelim, root);
+      }
+      FREE(reldelim);
+      if (ret)
+         return ret;
+   }
+
+   ret = nv_pass_tex_mask(&pass, root);
+   if (ret)
+      return ret;
+
+   return ret;
+}
+
+int
+nvc0_pc_exec_pass0(struct nv_pc *pc)
+{
+   int i, ret;
+
+   for (i = 0; i < pc->num_subroutines + 1; ++i)
+      if (pc->root[i] && (ret = nv_pc_pass0(pc, pc->root[i])))
+         return ret;
+   return 0;
+}
diff --git a/src/gallium/drivers/nvc0/nvc0_pc_print.c b/src/gallium/drivers/nvc0/nvc0_pc_print.c
new file mode 100644
index 00000000000..b03826484e4
--- /dev/null
+++ b/src/gallium/drivers/nvc0/nvc0_pc_print.c
@@ -0,0 +1,377 @@
+/*
+ * Copyright 2010 Christoph Bumiller
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF
+ * OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "nvc0_pc.h"
+
+#define PRINT(args...) debug_printf(args)
+
+#ifndef ARRAY_SIZE
+#define ARRAY_SIZE(a) (sizeof(a) / sizeof(a[0]))
+#endif
+
+static const char *norm = "\x1b[00m";
+static const char *gree = "\x1b[32m";
+static const char *blue = "\x1b[34m";
+static const char *cyan = "\x1b[36m";
+static const char *yllw = "\x1b[33m";
+static const char *mgta = "\x1b[35m";
+
+static const char *nv_cond_names[] =
+{
+   "never", "lt" , "eq" , "le" , "gt" , "ne" , "ge" , "",
+   "never", "ltu", "equ", "leu", "gtu", "neu", "geu", "",
+   "o", "c", "a", "s"
+};
+
+static const char *nv_modifier_strings[] =
+{
+   "",
+   "neg",
+   "abs",
+   "neg abs",
+   "not",
+   "not neg"
+   "not abs",
+   "not neg abs",
+   "sat",
+   "BAD_MOD"
+};
+
+const char *
+nvc0_opcode_name(uint opcode)
+{
+   return nvc0_op_info_table[MIN2(opcode, NV_OP_COUNT)].name;
+}
+
+static INLINE const char *
+nv_type_name(ubyte type, ubyte size)
+{
+   switch (type) {
+   case NV_TYPE_U16: return "u16";
+   case NV_TYPE_S16: return "s16";
+   case NV_TYPE_F32: return "f32";
+   case NV_TYPE_U32: return "u32";
+   case NV_TYPE_S32: return "s32";
+   case NV_TYPE_P32: return "p32";
+   case NV_TYPE_F64: return "f64";
+   case NV_TYPE_ANY:
+   {
+      switch (size) {
+      case 1: return "b8";
+      case 2: return "b16";
+      case 4: return "b32";
+      case 8: return "b64";
+      case 12: return "b96";
+      case 16: return "b128";
+      default:
+         return "BAD_SIZE";
+      }
+   }
+   default:
+      return "BAD_TYPE";
+   }
+}
+
+static INLINE const char *
+nv_cond_name(ubyte cc)
+{
+   return nv_cond_names[MIN2(cc, 19)];
+}
+
+static INLINE const char *
+nv_modifier_string(ubyte mod)
+{
+   return nv_modifier_strings[MIN2(mod, 9)];
+}
+
+static INLINE int
+nv_value_id(struct nv_value *value)
+{
+   if (value->join->reg.id >= 0)
+      return value->join->reg.id;
+   return value->n;
+}
+
+static INLINE boolean
+nv_value_allocated(struct nv_value *value)
+{
+   return (value->reg.id >= 0) ? TRUE : FALSE;
+}
+
+static INLINE void
+nv_print_address(const char c, int buf, struct nv_value *a, int offset)
+{
+   const char ac = (a && nv_value_allocated(a)) ? '$' : '%';
+   char sg;
+
+   if (offset < 0) {
+      sg = '-';
+      offset = -offset;
+   } else {
+      sg = '+';
+   }
+
+   if (buf >= 0)
+      PRINT(" %s%c%i[", cyan, c, buf);
+   else
+      PRINT(" %s%c[", cyan, c);
+   if (a)
+      PRINT("%s%ca%i%s%c", mgta, ac, nv_value_id(a), cyan, sg);
+   PRINT("%s0x%x%s]", yllw, offset, cyan);
+}
+
+static INLINE void
+nv_print_value(struct nv_value *value, struct nv_value *indir, ubyte type)
+{
+   char reg_pfx = nv_value_allocated(value->join) ? '$' : '%';
+
+   if (value->reg.file != NV_FILE_PRED)
+      PRINT(" %s%s", gree, nv_type_name(type, value->reg.size));
+
+   switch (value->reg.file) {
+   case NV_FILE_GPR:
+      PRINT(" %s%cr%i", blue, reg_pfx, nv_value_id(value));
+      if (value->reg.size == 8)
+         PRINT("d");
+      if (value->reg.size == 16)
+         PRINT("q");
+      break;
+   case NV_FILE_PRED:
+      PRINT(" %s%cp%i", mgta, reg_pfx, nv_value_id(value));
+      break;
+   case NV_FILE_COND:
+      PRINT(" %s%cc%i", mgta, reg_pfx, nv_value_id(value));
+      break;
+   case NV_FILE_MEM_L:
+      nv_print_address('l', -1, indir, value->reg.address);
+      break;
+   case NV_FILE_MEM_G:
+      nv_print_address('g', -1, indir, value->reg.address);
+      break;
+   case NV_FILE_MEM_A:
+      nv_print_address('a', -1, indir, value->reg.address);
+      break;
+   case NV_FILE_MEM_V:
+      nv_print_address('v', -1, indir, value->reg.address);
+      break;
+   case NV_FILE_IMM:
+      switch (type) {
+      case NV_TYPE_U16:
+      case NV_TYPE_S16:
+         PRINT(" %s0x%04x", yllw, value->reg.imm.u32);
+         break;
+      case NV_TYPE_F32:
+         PRINT(" %s%f", yllw, value->reg.imm.f32);
+         break;
+      case NV_TYPE_F64:
+         PRINT(" %s%f", yllw, value->reg.imm.f64);
+         break;
+      case NV_TYPE_U32:
+      case NV_TYPE_S32:
+      case NV_TYPE_P32:
+      case NV_TYPE_ANY:
+         PRINT(" %s0x%08x", yllw, value->reg.imm.u32);
+         break;
+      }
+      break;
+   default:
+      if (value->reg.file >= NV_FILE_MEM_C(0) &&
+          value->reg.file <= NV_FILE_MEM_C(15))
+         nv_print_address('c', value->reg.file - NV_FILE_MEM_C(0), indir,
+                          value->reg.address);
+      else
+         NOUVEAU_ERR(" BAD_FILE[%i]", nv_value_id(value));
+      break;
+   }
+}
+
+static INLINE void
+nv_print_ref(struct nv_ref *ref, struct nv_value *indir, ubyte type)
+{
+   nv_print_value(ref->value, indir, type);
+}
+
+void
+nvc0_print_instruction(struct nv_instruction *i)
+{
+   int s;
+
+   PRINT("%i: ", i->serial);
+
+   if (i->predicate >= 0) {
+      PRINT("%s%s", gree, i->cc ? "fl" : "tr");
+      nv_print_ref(i->src[i->predicate], NULL, NV_TYPE_U8);
+      PRINT(" ");
+   }
+
+   PRINT("%s", gree);
+   if (NV_BASEOP(i->opcode) == NV_OP_SET)
+      PRINT("set %s", nv_cond_name(i->set_cond));
+   else
+   if (i->saturate)
+      PRINT("sat %s", nvc0_opcode_name(i->opcode));
+   else
+      PRINT("%s", nvc0_opcode_name(i->opcode));
+
+   if (i->opcode == NV_OP_CVT)
+      nv_print_value(i->def[0], NULL, i->ext.cvt.d);
+   else
+   if (i->def[0])
+      nv_print_value(i->def[0], NULL, NV_OPTYPE(i->opcode));
+   else
+   if (i->target)
+      PRINT(" %s(BB:%i)", yllw, i->target->id);
+   else
+      PRINT(" #");
+
+   for (s = 1; s < 4 && i->def[s]; ++s)
+      nv_print_value(i->def[s], NULL, NV_OPTYPE(i->opcode));
+   if (s > 1)
+      PRINT("%s ,", norm);
+
+   for (s = 0; s < 6 && i->src[s]; ++s) {
+      ubyte type;
+      if (s == i->indirect || s == i->predicate)
+         continue;
+      if (i->opcode == NV_OP_CVT)
+         type = i->ext.cvt.s;
+      else
+         type = NV_OPTYPE(i->opcode);
+
+      if (i->src[s]->mod)
+         PRINT(" %s%s", gree, nv_modifier_string(i->src[s]->mod));
+
+      if (i->indirect >= 0 &&
+          NV_IS_MEMORY_FILE(i->src[s]->value->reg.file))
+         nv_print_ref(i->src[s], i->src[i->indirect]->value, type);
+      else
+         nv_print_ref(i->src[s], NULL, type);
+   }
+   PRINT(" %s\n", norm);
+}
+
+#define NV_MOD_SGN NV_MOD_ABS | NV_MOD_NEG
+
+struct nv_op_info nvc0_op_info_table[NV_OP_COUNT + 1] =
+{
+   { NV_OP_UNDEF,  "undef",  NV_TYPE_ANY, 0, /* fcvpoi */ 0, 0, 0, 0, 1, 0, 0 },
+   { NV_OP_BIND,   "bind",   NV_TYPE_ANY, 0, /* fcvpoi */ 0, 0, 1, 0, 1, 0, 0 },
+   { NV_OP_MERGE,  "merge",  NV_TYPE_ANY, 0, /* fcvpoi */ 0, 0, 1, 0, 1, 0, 0 },
+   { NV_OP_PHI,    "phi",    NV_TYPE_ANY, 0, /* fcvpoi */ 0, 0, 0, 0, 1, 0, 0 },
+   { NV_OP_SELECT, "select", NV_TYPE_ANY, 0, /* fcvpoi */ 0, 0, 0, 0, 1, 0, 0 },
+   { NV_OP_NOP,    "nop",    NV_TYPE_ANY, 0, /* fcvpoi */ 0, 0, 0, 0, 0, 0, 0 },
+
+   { NV_OP_LD,  "ld",  NV_TYPE_ANY, 0, 0, 0, 0, 0, 0, 0, 0 },
+   { NV_OP_ST,  "st",  NV_TYPE_ANY, 0, 0, 0, 0, 0, 0, 0, 0 },
+   { NV_OP_MOV, "mov", NV_TYPE_ANY, 0, 0, 0, 0, 1, 0, 1, 0 },
+   { NV_OP_AND, "and", NV_TYPE_U32, NV_MOD_NOT, 0, 1, 0, 1, 0, 6, 0 },
+   { NV_OP_OR,  "or",  NV_TYPE_U32, NV_MOD_NOT, 0, 1, 0, 1, 0, 6, 0 },
+   { NV_OP_XOR, "xor", NV_TYPE_U32, NV_MOD_NOT, 0, 1, 0, 1, 0, 6, 0 },
+   { NV_OP_SHL, "shl", NV_TYPE_U32, 0, 0, 0, 0, 1, 0, 1, 0 },
+   { NV_OP_SHR, "shr", NV_TYPE_U32, 0, 0, 0, 0, 1, 0, 1, 0 },
+   { NV_OP_NOT, "not", NV_TYPE_U32, 0, 0, 0, 0, 1, 0, 0, 0 },
+   { NV_OP_SET, "set", NV_TYPE_ANY, NV_MOD_SGN, 0, 0, 0, 1, 0, 0, 0 },
+   { NV_OP_ADD, "add", NV_TYPE_F32, NV_MOD_SGN, 0, 1, 0, 1, 0, 2, 2 },
+   { NV_OP_SUB, "sub", NV_TYPE_F32, NV_MOD_SGN, 0, 0, 0, 1, 0, 2, 2 },
+   { NV_OP_MUL, "mul", NV_TYPE_F32, NV_MOD_SGN, 0, 1, 0, 1, 0, 2, 2 },
+   { NV_OP_MAD, "mad", NV_TYPE_F32, NV_MOD_SGN, 0, 1, 0, 1, 0, 2, 2 },
+   { NV_OP_ABS, "abs", NV_TYPE_F32, 0, 0, 0, 0, 1, 0, 0, 0 },
+   { NV_OP_NEG, "neg", NV_TYPE_F32, NV_MOD_ABS, 0, 0, 0, 1, 0, 0, 0 },
+   { NV_OP_MAX, "max", NV_TYPE_F32, NV_MOD_SGN, 0, 1, 0, 1, 0, 2, 2 },
+   { NV_OP_MIN, "min", NV_TYPE_F32, NV_MOD_SGN, 0, 1, 0, 1, 0, 2, 2 },
+   { NV_OP_CVT, "cvt", NV_TYPE_ANY, NV_MOD_SGN, 0, 0, 0, 1, 0, 0, 0 },
+
+   { NV_OP_CEIL,  "ceil",  NV_TYPE_F32, NV_MOD_SGN, 0, 0, 0, 1, 0, 0, 0 },
+   { NV_OP_FLOOR, "floor", NV_TYPE_F32, NV_MOD_SGN, 0, 0, 0, 1, 0, 0, 0 },
+   { NV_OP_TRUNC, "floor", NV_TYPE_F32, NV_MOD_SGN, 0, 0, 0, 1, 0, 0, 0 },
+
+   { NV_OP_SAD, "sad", NV_TYPE_S32, 0, 0, 1, 0, 1, 0, 0, 0 },
+
+   { NV_OP_VFETCH,  "vfetch",  NV_TYPE_ANY, 0, 0, 0, 1, 1, 0, 0, 0 },
+   { NV_OP_PFETCH,  "pfetch",  NV_TYPE_U32, 0, 0, 0, 0, 1, 0, 0, 0 },
+   { NV_OP_EXPORT,  "export",  NV_TYPE_ANY, 0, 0, 0, 1, 1, 0, 0, 0 },
+   { NV_OP_LINTERP, "linterp", NV_TYPE_F32, 0, 0, 0, 0, 1, 0, 0, 0 },
+   { NV_OP_PINTERP, "pinterp", NV_TYPE_F32, 0, 0, 0, 0, 1, 0, 0, 0 },
+   { NV_OP_EMIT,    "emit",    NV_TYPE_ANY, 0, 0, 0, 0, 1, 0, 0, 0 },
+   { NV_OP_RESTART, "restart", NV_TYPE_ANY, 0, 0, 0, 0, 1, 0, 0, 0 },
+
+   { NV_OP_TEX, "tex",      NV_TYPE_F32, 0, 0, 0, 1, 1, 0, 0, 0 },
+   { NV_OP_TXB, "texbias",  NV_TYPE_F32, 0, 0, 0, 1, 1, 0, 0, 0 },
+   { NV_OP_TXL, "texlod",   NV_TYPE_F32, 0, 0, 0, 1, 1, 0, 0, 0 },
+   { NV_OP_TXF, "texfetch", NV_TYPE_U32, 0, 0, 0, 1, 1, 0, 0, 0 },
+   { NV_OP_TXQ, "texquery", NV_TYPE_U32, 0, 0, 0, 1, 1, 0, 0, 0 },
+
+   { NV_OP_QUADOP, "quadop", NV_TYPE_F32, 0, 0, 0, 0, 1, 0, 0, 0 },
+   { NV_OP_DFDX,   "dfdx",   NV_TYPE_F32, 0, 0, 0, 0, 1, 0, 0, 0 },
+   { NV_OP_DFDY,   "dfdy",   NV_TYPE_F32, 0, 0, 0, 0, 1, 0, 0, 0 },
+
+   { NV_OP_KIL,  "kil",  NV_TYPE_ANY, 0, 0, 0, 0, 1, 0, 0, 0 },
+   { NV_OP_BRA,  "bra",  NV_TYPE_ANY, 0, 1, 0, 0, 1, 0, 0, 0 },
+   { NV_OP_CALL, "call", NV_TYPE_ANY, 0, 1, 0, 0, 1, 0, 0, 0 },
+   { NV_OP_RET,  "ret",  NV_TYPE_ANY, 0, 1, 0, 0, 1, 0, 0, 0 },
+   { NV_OP_RET,  "exit", NV_TYPE_ANY, 0, 1, 0, 0, 1, 0, 0, 0 },
+   { NV_OP_NOP,  "ud",   NV_TYPE_ANY, 0, 1, 0, 0, 1, 0, 0, 0 },
+   { NV_OP_NOP,  "ud",   NV_TYPE_ANY, 0, 1, 0, 0, 1, 0, 0, 0 },
+
+   { NV_OP_JOINAT, "joinat", NV_TYPE_ANY, 0, 1, 0, 0, 1, 0, 0, 0 },
+   { NV_OP_JOIN,   "join",   NV_TYPE_ANY, 0, 1, 0, 0, 1, 0, 0, 0 },
+
+   { NV_OP_ADD, "add", NV_TYPE_S32, 0, 0, 1, 0, 1, 0, 1, 0 },
+   { NV_OP_MUL, "mul", NV_TYPE_S32, 0, 0, 1, 0, 1, 0, 1, 0 },
+   { NV_OP_ABS, "abs", NV_TYPE_S32, 0, 0, 0, 0, 1, 0, 0, 0 },
+   { NV_OP_NEG, "neg", NV_TYPE_S32, 0, 0, 0, 0, 1, 0, 0, 0 },
+   { NV_OP_MAX, "max", NV_TYPE_S32, 0, 0, 1, 0, 1, 0, 0, 0 },
+   { NV_OP_MIN, "max", NV_TYPE_U32, 0, 0, 1, 0, 1, 0, 0, 0 },
+   { NV_OP_MAX, "min", NV_TYPE_S32, 0, 0, 1, 0, 1, 0, 0, 0 },
+   { NV_OP_MIN, "min", NV_TYPE_U32, 0, 0, 1, 0, 1, 0, 0, 0 },
+   { NV_OP_SET, "set", NV_TYPE_F32, NV_MOD_SGN, 0, 0, 0, 1, 0, 0, 2 },
+   { NV_OP_SET, "set", NV_TYPE_S32, 0, 0, 0, 0, 1, 0, 0, 0 },
+   { NV_OP_SET, "set", NV_TYPE_U32, 0, 0, 0, 0, 1, 0, 0, 0 },
+   { NV_OP_SHR, "sar", NV_TYPE_S32, 0, 0, 0, 0, 1, 0, 1, 0 },
+   { NV_OP_RCP, "rcp", NV_TYPE_F32, NV_MOD_SGN, 0, 0, 0, 0, 0, 0, 0 },
+   { NV_OP_RSQ, "rsqrt", NV_TYPE_F32, NV_MOD_SGN, 0, 0, 0, 0, 0, 0, 0 },
+   { NV_OP_LG2, "lg2", NV_TYPE_F32, NV_MOD_SGN, 0, 0, 0, 0, 0, 0, 0 },
+   { NV_OP_SIN, "sin", NV_TYPE_F32, 0, 0, 0, 0, 0, 0, 0, 0 },
+   { NV_OP_COS, "cos", NV_TYPE_F32, 0, 0, 0, 0, 0, 0, 0, 0 },
+   { NV_OP_EX2, "ex2", NV_TYPE_F32, 0, 0, 0, 0, 0, 0, 0, 0 },
+   { NV_OP_PRESIN, "presin", NV_TYPE_F32, NV_MOD_SGN, 0, 0, 0, 0, 0, 0, 0 },
+   { NV_OP_PREEX2, "preex2", NV_TYPE_F32, NV_MOD_SGN, 0, 0, 0, 0, 0, 0, 0 },
+   { NV_OP_SAT, "sat", NV_TYPE_F32, 0, 0, 0, 0, 1, 0, 0, 0 },
+
+   { NV_OP_SET_F32_AND, "and set", NV_TYPE_F32, 0, 0, 0, 0, 1, 0, 0, 0 },
+   { NV_OP_SET_F32_OR,  "or set",  NV_TYPE_F32, 0, 0, 0, 0, 1, 0, 0, 0 },
+   { NV_OP_SET_F32_XOR, "xor set", NV_TYPE_F32, 0, 0, 0, 0, 1, 0, 0, 0 },
+
+   { NV_OP_SELP, "selp", NV_TYPE_U32, 0, 0, 0, 0, 1, 0, 0, 0 },
+
+   { NV_OP_SLCT_F32, "slct", NV_TYPE_F32, 0, 0, 0, 0, 1, 0, 0, 0 },
+   { NV_OP_SLCT_F32, "slct", NV_TYPE_S32, 0, 0, 0, 0, 1, 0, 0, 0 },
+   { NV_OP_SLCT_F32, "slct", NV_TYPE_U32, 0, 0, 0, 0, 1, 0, 0, 0 },
+
+   { NV_OP_ADD, "sub", NV_TYPE_F32, 0, 0, 0, 0, 1, 0, 1, 0 },
+
+   { NV_OP_FSET_F32, "fset", NV_TYPE_F32, NV_MOD_SGN, 0, 0, 0, 1, 0, 0, 2 },
+
+   { NV_OP_TXG, "texgrad", NV_TYPE_F32, 0, 0, 0, 1, 1, 0, 0, 0 },
+
+   { NV_OP_UNDEF, "BAD_OP", NV_TYPE_ANY, 0, 0, 0, 0, 0, 0, 0, 0 }
+};
diff --git a/src/gallium/drivers/nvc0/nvc0_pc_regalloc.c b/src/gallium/drivers/nvc0/nvc0_pc_regalloc.c
new file mode 100644
index 00000000000..d24f09a1507
--- /dev/null
+++ b/src/gallium/drivers/nvc0/nvc0_pc_regalloc.c
@@ -0,0 +1,927 @@
+/*
+ * Copyright 2010 Christoph Bumiller
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF
+ * OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#define NOUVEAU_DEBUG 1
+
+/* #define NVC0_RA_DEBUG_LIVEI */
+/* #define NVC0_RA_DEBUG_LIVE_SETS */
+/* #define NVC0_RA_DEBUG_JOIN */
+
+#include "nvc0_pc.h"
+#include "util/u_simple_list.h"
+
+#define NVC0_NUM_REGISTER_FILES 3
+
+/* @unit_shift: log2 of min allocation unit for register */
+struct register_set {
+   uint32_t bits[NVC0_NUM_REGISTER_FILES][2];
+   uint32_t last[NVC0_NUM_REGISTER_FILES];
+   int log2_unit[NVC0_NUM_REGISTER_FILES];
+   struct nv_pc *pc;
+};
+
+struct nv_pc_pass {
+   struct nv_pc *pc;
+   struct nv_instruction **insns;
+   uint num_insns;
+   uint pass_seq;
+};
+
+static void
+ranges_coalesce(struct nv_range *range)
+{
+   while (range->next && range->end >= range->next->bgn) {
+      struct nv_range *rnn = range->next->next;
+      assert(range->bgn <= range->next->bgn);
+      range->end = MAX2(range->end, range->next->end);
+      FREE(range->next);
+      range->next = rnn;
+   }
+}
+
+static boolean
+add_range_ex(struct nv_value *val, int bgn, int end, struct nv_range *new_range)
+{
+   struct nv_range *range, **nextp = &val->livei;
+
+   for (range = val->livei; range; range = range->next) {
+      if (end < range->bgn)
+         break; /* insert before */
+
+      if (bgn > range->end) {
+         nextp = &range->next;
+         continue; /* insert after */
+      }
+
+      /* overlap */
+      if (bgn < range->bgn) {
+         range->bgn = bgn;
+         if (end > range->end)
+            range->end = end;
+         ranges_coalesce(range);
+         return TRUE;
+      }
+      if (end > range->end) {
+         range->end = end;
+         ranges_coalesce(range);
+         return TRUE;
+      }
+      assert(bgn >= range->bgn);
+      assert(end <= range->end);
+      return TRUE;
+   }
+
+   if (!new_range)
+      new_range = CALLOC_STRUCT(nv_range);
+
+   new_range->bgn = bgn;
+   new_range->end = end;
+   new_range->next = range;
+   *(nextp) = new_range;
+   return FALSE;
+}
+
+static void
+add_range(struct nv_value *val, struct nv_basic_block *b, int end)
+{
+   int bgn;
+
+   if (!val->insn) /* ignore non-def values */
+      return;
+   assert(b->entry->serial <= b->exit->serial);
+   assert(b->phi->serial <= end);
+   assert(b->exit->serial + 1 >= end);
+
+   bgn = val->insn->serial;
+   if (bgn < b->entry->serial || bgn > b->exit->serial)
+      bgn = b->entry->serial;
+
+   assert(bgn <= end);
+
+   add_range_ex(val, bgn, end, NULL);
+}
+
+#if defined(NVC0_RA_DEBUG_JOIN) || defined(NVC0_RA_DEBUG_LIVEI)
+static void
+livei_print(struct nv_value *a)
+{
+   struct nv_range *r = a->livei;
+
+   debug_printf("livei %i: ", a->n);
+   while (r) {
+      debug_printf("[%i, %i) ", r->bgn, r->end);
+      r = r->next;
+   }
+   debug_printf("\n");
+}
+#endif
+
+static void
+livei_unify(struct nv_value *dst, struct nv_value *src)
+{
+   struct nv_range *range, *next;
+
+   for (range = src->livei; range; range = next) {
+      next = range->next;
+      if (add_range_ex(dst, range->bgn, range->end, range))
+         FREE(range);
+   }
+   src->livei = NULL;
+}
+
+static void
+livei_release(struct nv_value *val)
+{
+   struct nv_range *range, *next;
+
+   for (range = val->livei; range; range = next) {
+      next = range->next;
+      FREE(range);
+   }
+}
+
+static boolean
+livei_have_overlap(struct nv_value *a, struct nv_value *b)
+{
+   struct nv_range *r_a, *r_b;
+
+   for (r_a = a->livei; r_a; r_a = r_a->next) {
+      for (r_b = b->livei; r_b; r_b = r_b->next) {
+         if (r_b->bgn < r_a->end &&
+             r_b->end > r_a->bgn)
+            return TRUE;
+      }
+   }
+   return FALSE;
+}
+
+static int
+livei_end(struct nv_value *a)
+{
+   struct nv_range *r = a->livei;
+
+   assert(r);
+   while (r->next)
+      r = r->next;
+   return r->end;
+}
+
+static boolean
+livei_contains(struct nv_value *a, int pos)
+{
+   struct nv_range *r;
+
+   for (r = a->livei; r && r->bgn <= pos; r = r->next)
+      if (r->end > pos)
+         return TRUE;
+   return FALSE;
+}
+
+static boolean
+reg_assign(struct register_set *set, struct nv_value **def, int n)
+{
+   int i, id, s, k;
+   uint32_t m;
+   int f = def[0]->reg.file;
+
+   k = n;
+   if (k == 3)
+      k = 4;
+   s = (k * def[0]->reg.size) >> set->log2_unit[f];
+   m = (1 << s) - 1;
+
+   id = set->last[f];
+
+   for (i = 0; i * 32 < set->last[f]; ++i) {
+      if (set->bits[f][i] == 0xffffffff)
+         continue;
+
+      for (id = 0; id < 32; id += s)
+         if (!(set->bits[f][i] & (m << id)))
+            break;
+      if (id < 32)
+         break;
+   }
+   if (i * 32 + id > set->last[f])
+      return FALSE;
+
+   set->bits[f][i] |= m << id;
+
+   id += i * 32;
+
+   set->pc->max_reg[f] = MAX2(set->pc->max_reg[f], id + s - 1);
+
+   for (i = 0; i < n; ++i)
+      if (def[i]->livei)
+         def[i]->reg.id = id++;
+
+   return TRUE;
+}
+
+static INLINE void
+reg_occupy(struct register_set *set, struct nv_value *val)
+{
+   int id = val->reg.id, f = val->reg.file;
+   uint32_t m;
+
+   if (id < 0)
+      return;
+   m = (1 << (val->reg.size >> set->log2_unit[f])) - 1;
+
+   set->bits[f][id / 32] |= m << (id % 32);
+
+   if (set->pc->max_reg[f] < id)
+      set->pc->max_reg[f] = id;
+}
+
+static INLINE void
+reg_release(struct register_set *set, struct nv_value *val)
+{
+   int id = val->reg.id, f = val->reg.file;
+   uint32_t m;
+
+   if (id < 0)
+      return;
+   m = (1 << (val->reg.size >> set->log2_unit[f])) - 1;
+
+   set->bits[f][id / 32] &= ~(m << (id % 32));
+}
+
+static INLINE boolean
+join_allowed(struct nv_pc_pass *ctx, struct nv_value *a, struct nv_value *b)
+{
+   int i;
+   struct nv_value *val;
+
+   if (a->reg.file != b->reg.file || a->reg.size != b->reg.size)
+      return FALSE;
+
+   if (a->join->reg.id == b->join->reg.id)
+      return TRUE;
+
+   /* either a or b or both have been assigned */
+
+   if (a->join->reg.id >= 0 && b->join->reg.id >= 0)
+      return FALSE;
+   else
+   if (b->join->reg.id >= 0) {
+      if (b->join->reg.id == 63)
+         return FALSE;
+      val = a;
+      a = b;
+      b = val;
+   } else
+   if (a->join->reg.id == 63)
+      return FALSE;
+
+   for (i = 0; i < ctx->pc->num_values; ++i) {
+      val = &ctx->pc->values[i];
+
+      if (val->join->reg.id != a->join->reg.id)
+         continue;
+      if (val->join != a->join && livei_have_overlap(val->join, b->join))
+         return FALSE;
+   }
+   return TRUE;
+}
+
+static INLINE void
+do_join_values(struct nv_pc_pass *ctx, struct nv_value *a, struct nv_value *b)
+{
+   int j;
+   struct nv_value *bjoin = b->join;
+
+   if (b->join->reg.id >= 0)
+      a->join->reg.id = b->join->reg.id;
+
+   livei_unify(a->join, b->join);
+
+#ifdef NVC0_RA_DEBUG_JOIN
+   debug_printf("joining %i to %i\n", b->n, a->n);
+#endif
+   
+   /* make a->join the new representative */
+   for (j = 0; j < ctx->pc->num_values; ++j) 
+      if (ctx->pc->values[j].join == bjoin)
+         ctx->pc->values[j].join = a->join;
+
+   assert(b->join == a->join);
+}
+
+static INLINE void
+try_join_values(struct nv_pc_pass *ctx, struct nv_value *a, struct nv_value *b)
+{
+   if (!join_allowed(ctx, a, b)) {
+#ifdef NVC0_RA_DEBUG_JOIN
+      debug_printf("cannot join %i to %i: not allowed\n", b->n, a->n);
+#endif
+      return;
+   }
+   if (livei_have_overlap(a->join, b->join)) {
+#ifdef NVC0_RA_DEBUG_JOIN
+      debug_printf("cannot join %i to %i: livei overlap\n", b->n, a->n);
+      livei_print(a);
+      livei_print(b);
+#endif
+      return;
+   }
+
+   do_join_values(ctx, a, b);
+}
+
+static INLINE boolean
+need_new_else_block(struct nv_basic_block *b, struct nv_basic_block *p)
+{
+   int i = 0, n = 0;
+
+   for (; i < 2; ++i)
+      if (p->out[i] && !IS_LOOP_EDGE(p->out_kind[i]))
+         ++n;
+
+   return (b->num_in > 1) && (n == 2);
+}
+
+static int
+phi_opnd_for_bb(struct nv_instruction *phi, struct nv_basic_block *b,
+                struct nv_basic_block *tb)
+{
+   int i, j;
+
+   for (j = -1, i = 0; i < 6 && phi->src[i]; ++i) {
+      if (!nvc0_bblock_reachable_by(b, phi->src[i]->value->insn->bb, tb))
+         continue;
+      /* NOTE: back-edges are ignored by the reachable-by check */
+      if (j < 0 || !nvc0_bblock_reachable_by(phi->src[j]->value->insn->bb,
+                                             phi->src[i]->value->insn->bb, tb))
+         j = i;
+   }
+   return j;
+}
+
+/* For each operand of each PHI in b, generate a new value by inserting a MOV
+ * at the end of the block it is coming from and replace the operand with its
+ * result. This eliminates liveness conflicts and enables us to let values be
+ * copied to the right register if such a conflict exists nonetheless.
+ *
+ * These MOVs are also crucial in making sure the live intervals of phi srces
+ * are extended until the end of the loop, since they are not included in the
+ * live-in sets.
+ */
+static int
+pass_generate_phi_movs(struct nv_pc_pass *ctx, struct nv_basic_block *b)
+{
+   struct nv_instruction *i, *ni;
+   struct nv_value *val;
+   struct nv_basic_block *p, *pn;
+   int n, j;
+
+   b->pass_seq = ctx->pc->pass_seq;
+
+   for (n = 0; n < b->num_in; ++n) {
+      p = pn = b->in[n];
+      assert(p);
+
+      if (need_new_else_block(b, p)) {
+         pn = new_basic_block(ctx->pc);
+
+         if (p->out[0] == b)
+            p->out[0] = pn;
+         else
+            p->out[1] = pn;
+
+         if (p->exit->target == b) /* target to new else-block */
+            p->exit->target = pn;
+
+         b->in[n] = pn;
+
+         pn->out[0] = b;
+         pn->in[0] = p;
+         pn->num_in = 1;
+      }
+      ctx->pc->current_block = pn;
+
+      for (i = b->phi; i && i->opcode == NV_OP_PHI; i = i->next) {
+         if ((j = phi_opnd_for_bb(i, p, b)) < 0)
+            continue;
+         val = i->src[j]->value;
+
+         if (i->src[j]->flags) {
+            /* value already encountered from a different in-block */
+            val = val->insn->src[0]->value;
+            while (j < 6 && i->src[j])
+               ++j;
+            assert(j < 6);
+         }
+
+         ni = new_instruction(ctx->pc, NV_OP_MOV);
+
+         /* TODO: insert instruction at correct position in the first place */
+         if (ni->prev && ni->prev->target)
+            nvc0_insns_permute(ni->prev, ni);
+
+         ni->def[0] = new_value_like(ctx->pc, val);
+         ni->def[0]->insn = ni;
+         nv_reference(ctx->pc, ni, 0, val);
+         nv_reference(ctx->pc, i, j, ni->def[0]); /* new phi source = MOV def */
+         i->src[j]->flags = 1;
+      }
+
+      if (pn != p && pn->exit) {
+         ctx->pc->current_block = b->in[n ? 0 : 1];
+         ni = new_instruction(ctx->pc, NV_OP_BRA);
+         ni->target = b;
+         ni->terminator = 1;
+      }
+   }
+
+   for (j = 0; j < 2; ++j)
+      if (b->out[j] && b->out[j]->pass_seq < ctx->pc->pass_seq)
+         pass_generate_phi_movs(ctx, b->out[j]);
+
+   return 0;
+}
+
+static int
+pass_join_values(struct nv_pc_pass *ctx, int iter)
+{
+   int c, n;
+
+   for (n = 0; n < ctx->num_insns; ++n) {
+      struct nv_instruction *i = ctx->insns[n];
+
+      switch (i->opcode) {
+      case NV_OP_PHI:
+         if (iter != 2)
+            break;
+         for (c = 0; c < 6 && i->src[c]; ++c)
+            try_join_values(ctx, i->def[0], i->src[c]->value);
+         break;
+      case NV_OP_MOV:
+         if ((iter == 2) && i->src[0]->value->insn &&
+             !nv_is_texture_op(i->src[0]->value->join->insn->opcode))
+            try_join_values(ctx, i->def[0], i->src[0]->value);
+         break;
+      case NV_OP_SELECT:
+         if (iter != 1)
+            break;
+         for (c = 0; c < 6 && i->src[c]; ++c) {
+            assert(join_allowed(ctx, i->def[0], i->src[c]->value));
+            do_join_values(ctx, i->def[0], i->src[c]->value);
+         }
+         break;
+      case NV_OP_TEX:
+      case NV_OP_TXB:
+      case NV_OP_TXL:
+      case NV_OP_TXQ:
+         /* on nvc0, TEX src and dst can differ */
+         break;
+      case NV_OP_BIND:
+         if (iter)
+            break;
+         for (c = 0; c < 6 && i->src[c]; ++c)
+            do_join_values(ctx, i->def[c], i->src[c]->value);
+         break;
+      default:
+         break;
+      }
+   }
+   return 0;
+}
+
+/* Order the instructions so that live intervals can be expressed in numbers. */
+static void
+pass_order_instructions(void *priv, struct nv_basic_block *b)
+{
+   struct nv_pc_pass *ctx = (struct nv_pc_pass *)priv;
+   struct nv_instruction *i;
+
+   b->pass_seq = ctx->pc->pass_seq;
+
+   assert(!b->exit || !b->exit->next);
+   for (i = b->phi; i; i = i->next) {
+      i->serial = ctx->num_insns;
+      ctx->insns[ctx->num_insns++] = i;
+   }
+}
+
+static void
+bb_live_set_print(struct nv_pc *pc, struct nv_basic_block *b)
+{
+#ifdef NVC0_RA_DEBUG_LIVE_SETS
+   struct nv_value *val;
+   int j;
+
+   debug_printf("LIVE-INs of BB:%i: ", b->id);
+
+   for (j = 0; j < pc->num_values; ++j) {
+      if (!(b->live_set[j / 32] & (1 << (j % 32))))
+         continue;
+      val = &pc->values[j];
+      if (!val->insn)
+         continue;
+      debug_printf("%i ", val->n);
+   }
+   debug_printf("\n");
+#endif
+}
+
+static INLINE void
+live_set_add(struct nv_basic_block *b, struct nv_value *val)
+{
+   if (!val->insn) /* don't add non-def values */
+      return;
+   b->live_set[val->n / 32] |= 1 << (val->n % 32);
+}
+
+static INLINE void
+live_set_rem(struct nv_basic_block *b, struct nv_value *val)
+{
+   b->live_set[val->n / 32] &= ~(1 << (val->n % 32));
+}
+
+static INLINE boolean
+live_set_test(struct nv_basic_block *b, struct nv_ref *ref)
+{
+   int n = ref->value->n;
+   return b->live_set[n / 32] & (1 << (n % 32));
+}
+
+/* The live set of a block contains those values that are live immediately
+ * before the beginning of the block, so do a backwards scan.
+ */
+static int
+pass_build_live_sets(struct nv_pc_pass *ctx, struct nv_basic_block *b)
+{
+   struct nv_instruction *i;
+   int j, n, ret = 0;
+
+   if (b->pass_seq >= ctx->pc->pass_seq)
+      return 0;
+   b->pass_seq = ctx->pc->pass_seq;
+
+   /* slight hack for undecidedness: set phi = entry if it's undefined */
+   if (!b->phi)
+      b->phi = b->entry;
+
+   for (n = 0; n < 2; ++n) {
+      if (!b->out[n] || b->out[n] == b)
+         continue;
+      ret = pass_build_live_sets(ctx, b->out[n]);
+      if (ret)
+         return ret;
+
+      if (n == 0) {
+         for (j = 0; j < (ctx->pc->num_values + 31) / 32; ++j)
+            b->live_set[j] = b->out[n]->live_set[j];
+      } else {
+         for (j = 0; j < (ctx->pc->num_values + 31) / 32; ++j)
+            b->live_set[j] |= b->out[n]->live_set[j];
+      }
+   }
+
+   if (!b->entry)
+      return 0;
+
+   bb_live_set_print(ctx->pc, b);
+
+   for (i = b->exit; i != b->entry->prev; i = i->prev) {
+      for (j = 0; j < 5 && i->def[j]; j++)
+         live_set_rem(b, i->def[j]);
+      for (j = 0; j < 6 && i->src[j]; j++)
+         live_set_add(b, i->src[j]->value);
+   }
+   for (i = b->phi; i && i->opcode == NV_OP_PHI; i = i->next)
+      live_set_rem(b, i->def[0]);
+
+   bb_live_set_print(ctx->pc, b);
+
+   return 0;
+}
+
+static void collect_live_values(struct nv_basic_block *b, const int n)
+{
+   int i;
+
+   if (b->out[0]) {
+      if (b->out[1]) { /* what to do about back-edges ? */
+         for (i = 0; i < n; ++i)
+            b->live_set[i] = b->out[0]->live_set[i] | b->out[1]->live_set[i];
+      } else {
+         memcpy(b->live_set, b->out[0]->live_set, n * sizeof(uint32_t));
+      }
+   } else
+   if (b->out[1]) {
+      memcpy(b->live_set, b->out[1]->live_set, n * sizeof(uint32_t));
+   } else {
+      memset(b->live_set, 0, n * sizeof(uint32_t));
+   }
+}
+
+/* NOTE: the live intervals of phi functions start at the first non-phi insn. */
+static int
+pass_build_intervals(struct nv_pc_pass *ctx, struct nv_basic_block *b)
+{
+   struct nv_instruction *i, *i_stop;
+   int j, s;
+   const int n = (ctx->pc->num_values + 31) / 32;
+
+   /* verify that first block does not have live-in values */
+   if (b->num_in == 0)
+      for (j = 0; j < n; ++j)
+         assert(b->live_set[j] == 0);
+
+   collect_live_values(b, n);
+
+   /* remove live-outs def'd in a parallel block, hopefully they're all phi'd */
+   for (j = 0; j < 2; ++j) {
+      if (!b->out[j] || !b->out[j]->phi)
+         continue;
+      for (i = b->out[j]->phi; i->opcode == NV_OP_PHI; i = i->next) {
+         live_set_rem(b, i->def[0]);
+
+         for (s = 0; s < 6 && i->src[s]; ++s) {
+            assert(i->src[s]->value->insn);
+            if (nvc0_bblock_reachable_by(b, i->src[s]->value->insn->bb,
+                                         b->out[j]))
+               live_set_add(b, i->src[s]->value);
+            else
+               live_set_rem(b, i->src[s]->value);
+         }
+      }
+   }
+
+   /* remaining live-outs are live until the end */
+   if (b->exit) {
+      for (j = 0; j < ctx->pc->num_values; ++j) {
+         if (!(b->live_set[j / 32] & (1 << (j % 32))))
+            continue;
+         add_range(&ctx->pc->values[j], b, b->exit->serial + 1);
+#ifdef NVC0_RA_DEBUG_LIVEI
+         debug_printf("adding range for live value %i: ", j);
+         livei_print(&ctx->pc->values[j]);
+#endif
+      }
+   }
+
+   i_stop = b->entry ? b->entry->prev : NULL;
+
+   /* don't have to include phi functions here (will have 0 live range) */
+   for (i = b->exit; i != i_stop; i = i->prev) {
+      assert(i->serial >= b->phi->serial && i->serial <= b->exit->serial);
+      for (j = 0; j < 4 && i->def[j]; ++j)
+         live_set_rem(b, i->def[j]);
+
+      for (j = 0; j < 6 && i->src[j]; ++j) {
+         if (!live_set_test(b, i->src[j])) {
+            live_set_add(b, i->src[j]->value);
+            add_range(i->src[j]->value, b, i->serial);
+#ifdef NVC0_RA_DEBUG_LIVEI
+            debug_printf("adding range for source %i (ends living): ",
+                         i->src[j]->value->n);
+            livei_print(i->src[j]->value);
+#endif
+         }
+      }
+   }
+
+   b->pass_seq = ctx->pc->pass_seq;
+
+   if (b->out[0] && b->out[0]->pass_seq < ctx->pc->pass_seq)
+      pass_build_intervals(ctx, b->out[0]);
+
+   if (b->out[1] && b->out[1]->pass_seq < ctx->pc->pass_seq)
+      pass_build_intervals(ctx, b->out[1]);
+
+   return 0;
+}
+
+static INLINE void
+nvc0_ctor_register_set(struct nv_pc *pc, struct register_set *set)
+{
+   memset(set, 0, sizeof(*set));
+
+   set->last[NV_FILE_GPR] = 62;
+   set->last[NV_FILE_PRED] = 6;
+   set->last[NV_FILE_COND] = 1;
+
+   set->log2_unit[NV_FILE_GPR] = 2;
+   set->log2_unit[NV_FILE_COND] = 0;
+   set->log2_unit[NV_FILE_PRED] = 0;
+
+   set->pc = pc;
+}
+
+static void
+insert_ordered_tail(struct nv_value *list, struct nv_value *nval)
+{
+   struct nv_value *elem;
+
+   for (elem = list->prev;
+        elem != list && elem->livei->bgn > nval->livei->bgn;
+        elem = elem->prev);
+   /* now elem begins before or at the same time as val */
+
+   nval->prev = elem;
+   nval->next = elem->next;
+   elem->next->prev = nval;
+   elem->next = nval;
+}
+
+static int
+pass_linear_scan(struct nv_pc_pass *ctx, int iter)
+{
+   struct nv_instruction *i;
+   struct register_set f, free;
+   int k, n;
+   struct nv_value *cur, *val, *tmp[2];
+   struct nv_value active, inactive, handled, unhandled;
+
+   make_empty_list(&active);
+   make_empty_list(&inactive);
+   make_empty_list(&handled);
+   make_empty_list(&unhandled);
+
+   nvc0_ctor_register_set(ctx->pc, &free);
+
+   /* joined values should have range = NULL and thus not be added;
+    * also, fixed memory values won't be added because they're not
+    * def'd, just used
+    */
+   for (n = 0; n < ctx->num_insns; ++n) {
+      i = ctx->insns[n];
+
+      for (k = 0; k < 5; ++k) {
+         if (i->def[k] && i->def[k]->livei)
+            insert_ordered_tail(&unhandled, i->def[k]);
+         else
+         if (0 && i->def[k])
+            debug_printf("skipping def'd value %i: no livei\n", i->def[k]->n);
+      }
+   }
+
+   for (val = unhandled.next; val != unhandled.prev; val = val->next) {
+      assert(val->join == val);
+      assert(val->livei->bgn <= val->next->livei->bgn);
+   }
+
+   foreach_s(cur, tmp[0], &unhandled) {
+      remove_from_list(cur);
+
+      foreach_s(val, tmp[1], &active) {
+         if (livei_end(val) <= cur->livei->bgn) {
+            reg_release(&free, val);
+            move_to_head(&handled, val);
+         } else
+         if (!livei_contains(val, cur->livei->bgn)) {
+            reg_release(&free, val);
+            move_to_head(&inactive, val);
+         }
+      }
+
+      foreach_s(val, tmp[1], &inactive) {
+         if (livei_end(val) <= cur->livei->bgn)
+            move_to_head(&handled, val);
+         else
+         if (livei_contains(val, cur->livei->bgn)) {
+            reg_occupy(&free, val);
+            move_to_head(&active, val);
+         }
+      }
+
+      f = free;
+
+      foreach(val, &inactive)
+         if (livei_have_overlap(val, cur))
+            reg_occupy(&f, val);
+
+      foreach(val, &unhandled)
+         if (val->reg.id >= 0 && livei_have_overlap(val, cur))
+            reg_occupy(&f, val);
+
+      if (cur->reg.id < 0) {
+         boolean mem = FALSE;
+         int v = nvi_vector_size(cur->insn);
+
+         if (v > 1)
+            mem = !reg_assign(&f, &cur->insn->def[0], v);
+         else
+         if (iter)
+            mem = !reg_assign(&f, &cur, 1);
+
+         if (mem) {
+            NOUVEAU_ERR("out of registers\n");
+            abort();
+         }
+      }
+      insert_at_head(&active, cur);
+      reg_occupy(&free, cur);
+   }
+
+   return 0;
+}
+
+static int
+nv_pc_pass1(struct nv_pc *pc, struct nv_basic_block *root)
+{
+   struct nv_pc_pass *ctx;
+   int i, ret;
+
+   NOUVEAU_DBG("REGISTER ALLOCATION - entering\n");
+
+   ctx = CALLOC_STRUCT(nv_pc_pass);
+   if (!ctx)
+      return -1;
+   ctx->pc = pc;
+
+   ctx->insns = CALLOC(NV_PC_MAX_INSTRUCTIONS, sizeof(struct nv_instruction *));
+   if (!ctx->insns) {
+      FREE(ctx);
+      return -1;
+   }
+
+   pc->pass_seq++;
+   ret = pass_generate_phi_movs(ctx, root);
+   assert(!ret);
+
+   for (i = 0; i < pc->loop_nesting_bound; ++i) {
+      pc->pass_seq++;
+      ret = pass_build_live_sets(ctx, root);
+      assert(!ret && "live sets");
+      if (ret) {
+         NOUVEAU_ERR("failed to build live sets (iteration %d)\n", i);
+         goto out;
+      }
+   }
+
+   pc->pass_seq++;
+   nvc0_pc_pass_in_order(root, pass_order_instructions, ctx);
+
+   pc->pass_seq++;
+   ret = pass_build_intervals(ctx, root);
+   assert(!ret && "build intervals");
+   if (ret) {
+      NOUVEAU_ERR("failed to build live intervals\n");
+      goto out;
+   }
+
+#ifdef NVC0_RA_DEBUG_LIVEI
+   for (i = 0; i < pc->num_values; ++i)
+      livei_print(&pc->values[i]);
+#endif
+
+   ret = pass_join_values(ctx, 0);
+   if (ret)
+      goto out;
+   ret = pass_linear_scan(ctx, 0);
+   if (ret)
+      goto out;
+   ret = pass_join_values(ctx, 1);
+   if (ret)
+      goto out;
+   ret = pass_join_values(ctx, 2);
+   if (ret)
+      goto out;
+   ret = pass_linear_scan(ctx, 1);
+   if (ret)
+      goto out;
+
+   for (i = 0; i < pc->num_values; ++i)
+      livei_release(&pc->values[i]);
+
+   NOUVEAU_DBG("REGISTER ALLOCATION - leaving\n");
+
+out:
+   FREE(ctx->insns);
+   FREE(ctx);
+   return ret;
+}
+
+int
+nvc0_pc_exec_pass1(struct nv_pc *pc)
+{
+   int i, ret;
+
+   for (i = 0; i < pc->num_subroutines + 1; ++i)
+      if (pc->root[i] && (ret = nv_pc_pass1(pc, pc->root[i])))
+         return ret;
+   return 0;
+}
diff --git a/src/gallium/drivers/nvc0/nvc0_program.c b/src/gallium/drivers/nvc0/nvc0_program.c
new file mode 100644
index 00000000000..3e7fc4d350e
--- /dev/null
+++ b/src/gallium/drivers/nvc0/nvc0_program.c
@@ -0,0 +1,671 @@
+/*
+ * Copyright 2010 Christoph Bumiller
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF
+ * OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "pipe/p_shader_tokens.h"
+#include "pipe/p_defines.h"
+
+#define NOUVEAU_DEBUG
+
+#include "tgsi/tgsi_parse.h"
+#include "tgsi/tgsi_util.h"
+#include "tgsi/tgsi_dump.h"
+
+#include "nvc0_context.h"
+#include "nvc0_pc.h"
+
+static unsigned
+nvc0_tgsi_src_mask(const struct tgsi_full_instruction *inst, int c)
+{
+   unsigned mask = inst->Dst[0].Register.WriteMask;
+
+   switch (inst->Instruction.Opcode) {
+   case TGSI_OPCODE_COS:
+   case TGSI_OPCODE_SIN:
+      return (mask & 0x8) | ((mask & 0x7) ? 0x1 : 0x0);
+   case TGSI_OPCODE_DP3:
+      return 0x7;
+   case TGSI_OPCODE_DP4:
+   case TGSI_OPCODE_DPH:
+   case TGSI_OPCODE_KIL: /* WriteMask ignored */
+      return 0xf;
+   case TGSI_OPCODE_DST:
+      return mask & (c ? 0xa : 0x6);
+   case TGSI_OPCODE_EX2:
+   case TGSI_OPCODE_EXP:
+   case TGSI_OPCODE_LG2:
+   case TGSI_OPCODE_LOG:
+   case TGSI_OPCODE_POW:
+   case TGSI_OPCODE_RCP:
+   case TGSI_OPCODE_RSQ:
+   case TGSI_OPCODE_SCS:
+      return 0x1;
+   case TGSI_OPCODE_IF:
+      return 0x1;
+   case TGSI_OPCODE_LIT:
+      return 0xb;
+   case TGSI_OPCODE_TEX:
+   case TGSI_OPCODE_TXB:
+   case TGSI_OPCODE_TXL:
+   case TGSI_OPCODE_TXP:
+   {
+      const struct tgsi_instruction_texture *tex;
+
+      assert(inst->Instruction.Texture);
+      tex = &inst->Texture;
+
+      mask = 0x7;
+      if (inst->Instruction.Opcode != TGSI_OPCODE_TEX &&
+          inst->Instruction.Opcode != TGSI_OPCODE_TXD)
+         mask |= 0x8; /* bias, lod or proj */
+
+      switch (tex->Texture) {
+      case TGSI_TEXTURE_1D:
+         mask &= 0x9;
+         break;
+      case TGSI_TEXTURE_SHADOW1D:
+         mask &= 0x5;
+         break;
+      case TGSI_TEXTURE_2D:
+         mask &= 0xb;
+         break;
+      default:
+         break;
+      }
+   }
+  	   return mask;
+   case TGSI_OPCODE_XPD:
+   {
+      unsigned x = 0;
+      if (mask & 1) x |= 0x6;
+      if (mask & 2) x |= 0x5;
+      if (mask & 4) x |= 0x3;
+      return x;
+   }
+   default:
+      break;
+   }
+
+   return mask;
+}
+
+static void
+nvc0_indirect_inputs(struct nvc0_translation_info *ti, int id)
+{
+   int i, c;
+
+   for (i = 0; i < PIPE_MAX_SHADER_INPUTS; ++i)
+      for (c = 0; c < 4; ++c)
+         ti->input_access[i][c] = id;
+
+   ti->indirect_inputs = TRUE;
+}
+
+static void
+nvc0_indirect_outputs(struct nvc0_translation_info *ti, int id)
+{
+   int i, c;
+
+   for (i = 0; i < PIPE_MAX_SHADER_OUTPUTS; ++i)
+      for (c = 0; c < 4; ++c)
+         ti->output_access[i][c] = id;
+
+   ti->indirect_outputs = TRUE;
+}
+
+static INLINE unsigned
+nvc0_system_value_location(unsigned sn, unsigned si)
+{
+   /* NOTE: locations 0xfxx indicate special regs */
+   switch (sn) {
+      /*
+   case TGSI_SEMANTIC_VERTEXID:
+      return 0x2fc;
+      */
+   case TGSI_SEMANTIC_PRIMID:
+      return 0x60;
+      /*
+   case TGSI_SEMANTIC_LAYER_INDEX:
+      return 0x64;
+   case TGSI_SEMANTIC_VIEWPORT_INDEX:
+      return 0x68;
+      */
+   case TGSI_SEMANTIC_INSTANCEID:
+      return 0x2f8;
+   case TGSI_SEMANTIC_FACE:
+      return 0x3fc;
+      /*
+   case TGSI_SEMANTIC_INVOCATIONID:
+      return 0xf11;
+      */
+   default:
+      assert(0);
+      return 0x000;
+   }
+}
+
+static INLINE unsigned
+nvc0_varying_location(unsigned sn, unsigned si)
+{
+   switch (sn) {
+   case TGSI_SEMANTIC_POSITION:
+      return 0x70;
+   case TGSI_SEMANTIC_COLOR:
+      return 0x280 + (si * 16); /* are these hard-wired ? */
+   case TGSI_SEMANTIC_BCOLOR:
+      return 0x2a0 + (si * 16);
+   case TGSI_SEMANTIC_FOG:
+      return 0x270;
+   case TGSI_SEMANTIC_PSIZE:
+      return 0x6c;
+      /*
+   case TGSI_SEMANTIC_PNTC:
+      return 0x2e0;
+      */
+   case TGSI_SEMANTIC_GENERIC:
+      assert(si < 31);
+      return 0x80 + (si * 16);
+   case TGSI_SEMANTIC_NORMAL:
+      return 0x360;
+   case TGSI_SEMANTIC_PRIMID:
+      return 0x40;
+   case TGSI_SEMANTIC_FACE:
+      return 0x3fc;
+      /*
+   case TGSI_SEMANTIC_CLIP_DISTANCE:
+      return 0x2c0 + (si * 4);
+      */
+   default:
+      assert(0);
+      return 0x000;
+   }
+}
+
+static INLINE unsigned
+nvc0_interp_mode(const struct tgsi_full_declaration *decl)
+{
+   unsigned mode;
+
+   if (decl->Declaration.Interpolate == TGSI_INTERPOLATE_CONSTANT)
+      mode = NVC0_INTERP_FLAT;
+   else
+   if (decl->Declaration.Interpolate == TGSI_INTERPOLATE_PERSPECTIVE)
+      mode = NVC0_INTERP_PERSPECTIVE;
+   else
+      mode = NVC0_INTERP_LINEAR;
+
+   if (decl->Declaration.Centroid)
+      mode |= NVC0_INTERP_CENTROID;
+
+   return mode;
+}
+
+static void
+prog_immediate(struct nvc0_translation_info *ti,
+               const struct tgsi_full_immediate *imm)
+{
+   int c;
+   unsigned n = ti->immd32_nr++;
+
+   assert(ti->immd32_nr <= ti->scan.immediate_count);
+
+   for (c = 0; c < 4; ++c)
+      ti->immd32[n * 4 + c] = imm->u[c].Uint;
+
+   ti->immd32_ty[n] = imm->Immediate.DataType;
+}
+
+static boolean
+prog_decl(struct nvc0_translation_info *ti,
+          const struct tgsi_full_declaration *decl)
+{
+   unsigned i, c;
+   unsigned sn = TGSI_SEMANTIC_GENERIC;
+   unsigned si = 0;
+   const unsigned first = decl->Range.First;
+   const unsigned last = decl->Range.Last;
+
+   if (decl->Declaration.Semantic) {
+      sn = decl->Semantic.Name;
+      si = decl->Semantic.Index;
+   }
+   
+   switch (decl->Declaration.File) {
+   case TGSI_FILE_INPUT:
+      for (i = first; i <= last; ++i) {
+         if (ti->prog->type == PIPE_SHADER_VERTEX) {
+            sn = TGSI_SEMANTIC_GENERIC;
+            si = i;
+         }
+         for (c = 0; c < 4; ++c)
+            ti->input_loc[i][c] = nvc0_varying_location(sn, si) + c * 4;
+
+         if (ti->prog->type == PIPE_SHADER_FRAGMENT)
+            ti->interp_mode[i] = nvc0_interp_mode(decl);
+      }
+      break;
+   case TGSI_FILE_OUTPUT:
+      for (i = first; i <= last; ++i, ++si) {
+         if (ti->prog->type == PIPE_SHADER_FRAGMENT) {
+            si = i;
+            if (i == ti->fp_depth_output) {
+               ti->output_loc[i][2] = (ti->scan.num_outputs - 1) * 4;
+            } else {
+               if (i > ti->fp_depth_output)
+                  si -= 1;
+               for (c = 0; c < 4; ++c)
+                  ti->output_loc[i][c] = si * 4 + c;
+            }
+         } else {
+            for (c = 0; c < 4; ++c)
+               ti->output_loc[i][c] = nvc0_varying_location(sn, si) + c * 4;
+         }
+      }
+      break;
+   case TGSI_FILE_SYSTEM_VALUE:
+      ti->sysval_loc[i] = nvc0_system_value_location(sn, si);
+      assert(first == last);
+      break;
+   case TGSI_FILE_NULL:
+   case TGSI_FILE_CONSTANT:
+   case TGSI_FILE_TEMPORARY:
+   case TGSI_FILE_SAMPLER:
+   case TGSI_FILE_ADDRESS:
+   case TGSI_FILE_IMMEDIATE:
+   case TGSI_FILE_PREDICATE:
+      break;
+   default:
+      NOUVEAU_ERR("unhandled TGSI_FILE %d\n", decl->Declaration.File);
+      return FALSE;
+   }
+   return TRUE;
+}
+
+static void
+prog_inst(struct nvc0_translation_info *ti,
+          const struct tgsi_full_instruction *inst, int id)
+{
+   const struct tgsi_dst_register *dst;
+   const struct tgsi_src_register *src;
+   int s, c, k;
+   unsigned mask;
+
+   if (inst->Instruction.Opcode == TGSI_OPCODE_BGNSUB) {
+      ti->subr[ti->num_subrs].first_insn = id - 1;
+      ti->subr[ti->num_subrs].id = ti->num_subrs + 1; /* id 0 is main program */
+      ++ti->num_subrs;
+   }
+
+   if (inst->Dst[0].Register.File == TGSI_FILE_OUTPUT) {
+      dst = &inst->Dst[0].Register;
+
+      for (c = 0; c < 4; ++c) {
+         if (dst->Indirect)
+            nvc0_indirect_outputs(ti, id);
+         if (!(dst->WriteMask & (1 << c)))
+            continue;
+         ti->output_access[dst->Index][c] = id;
+      }
+
+      if (inst->Instruction.Opcode == TGSI_OPCODE_MOV &&
+          inst->Src[0].Register.File == TGSI_FILE_INPUT &&
+          dst->Index == ti->edgeflag_out)
+         ti->prog->vp.edgeflag = inst->Src[0].Register.Index;
+   } else
+   if (inst->Dst[0].Register.File == TGSI_FILE_TEMPORARY) {
+      if (inst->Dst[0].Register.Indirect)
+         ti->require_stores = TRUE;
+   }
+
+   for (s = 0; s < inst->Instruction.NumSrcRegs; ++s) {
+      src = &inst->Src[s].Register;
+      if (src->File == TGSI_FILE_TEMPORARY)
+         if (inst->Src[s].Register.Indirect)
+            ti->require_stores = TRUE;
+      if (src->File != TGSI_FILE_INPUT)
+         continue;
+      mask = nvc0_tgsi_src_mask(inst, s);
+
+      if (inst->Src[s].Register.Indirect)
+         nvc0_indirect_inputs(ti, id);
+
+      for (c = 0; c < 4; ++c) {
+         if (!(mask & (1 << c)))
+            continue;
+         k = tgsi_util_get_full_src_register_swizzle(&inst->Src[s], c);
+         if (k <= TGSI_SWIZZLE_W)
+            ti->input_access[src->Index][k] = id;
+      }
+   }
+}
+
+/* Probably should introduce something like struct tgsi_function_declaration
+ * instead of trying to guess inputs/outputs.
+ */
+static void
+prog_subroutine_inst(struct nvc0_subroutine *subr,
+                     const struct tgsi_full_instruction *inst)
+{
+   const struct tgsi_dst_register *dst;
+   const struct tgsi_src_register *src;
+   int s, c, k;
+   unsigned mask;
+
+   for (s = 0; s < inst->Instruction.NumSrcRegs; ++s) {
+      src = &inst->Src[s].Register;
+      if (src->File != TGSI_FILE_TEMPORARY)
+         continue;
+      mask = nvc0_tgsi_src_mask(inst, s);
+
+      for (c = 0; c < 4; ++c) {
+         k = tgsi_util_get_full_src_register_swizzle(&inst->Src[s], c);
+
+         if ((mask & (1 << c)) && k < TGSI_SWIZZLE_W)
+            if (!(subr->retv[src->Index / 32][k] & (1 << (src->Index % 32))))
+               subr->argv[src->Index / 32][k] |= 1 << (src->Index % 32);
+      }
+   }
+
+   if (inst->Dst[0].Register.File == TGSI_FILE_TEMPORARY) {
+      dst = &inst->Dst[0].Register;
+
+      for (c = 0; c < 4; ++c)
+         if (dst->WriteMask & (1 << c))
+            subr->retv[dst->Index / 32][c] |= 1 << (dst->Index % 32);
+   }
+}
+
+static int
+nvc0_vp_gp_gen_header(struct nvc0_program *vp, struct nvc0_translation_info *ti)
+{
+   int i, c;
+   unsigned a;
+
+   for (a = 0x80/4, i = 0; i <= ti->scan.file_max[TGSI_FILE_INPUT]; ++i) {
+      for (c = 0; c < 4; ++c, ++a)
+         if (ti->input_access[i][c])
+            vp->hdr[5 + a / 32] |= 1 << (a % 32); /* VP_ATTR_EN */
+   }
+
+   for (i = 0; i <= ti->scan.file_max[TGSI_FILE_OUTPUT]; ++i) {
+      a = (ti->output_loc[i][0] - 0x40) / 4;
+      for (c = 0; c < 4; ++c, ++a) {
+         if (!ti->output_access[i][c])
+            continue;
+         vp->hdr[13 + a / 32] |= 1 << (a % 32); /* VP_EXPORT_EN */
+      }
+   }
+
+   return 0;
+}
+
+static int
+nvc0_vp_gen_header(struct nvc0_program *vp, struct nvc0_translation_info *ti)
+{
+   vp->hdr[0] = 0x20461;
+   vp->hdr[4] = 0xff000;
+
+   vp->hdr[18] = (1 << vp->vp.num_ucps) - 1;
+
+   return nvc0_vp_gp_gen_header(vp, ti);
+}
+
+static int
+nvc0_gp_gen_header(struct nvc0_program *gp, struct nvc0_translation_info *ti)
+{
+   unsigned invocations = 1;
+   unsigned max_output_verts, output_prim;
+   unsigned i;
+
+   gp->hdr[0] = 0x21061;
+
+   for (i = 0; i < ti->scan.num_properties; ++i) {
+      switch (ti->scan.properties[i].name) {
+      case TGSI_PROPERTY_GS_OUTPUT_PRIM:
+         output_prim = ti->scan.properties[i].data[0];
+         break;
+      case TGSI_PROPERTY_GS_MAX_OUTPUT_VERTICES:
+         max_output_verts = ti->scan.properties[i].data[0];
+         assert(max_output_verts < 512);
+         break;
+         /*
+      case TGSI_PROPERTY_GS_INVOCATIONS:
+         invocations = ti->scan.properties[i].data[0];
+         assert(invocations <= 32);
+         break;
+         */
+      default:
+         break;
+      }
+   }
+
+   gp->hdr[2] = MIN2(invocations, 32) << 24;
+
+   switch (output_prim) {
+   case PIPE_PRIM_POINTS:
+      gp->hdr[3] = 0x01000000;
+      gp->hdr[0] |= 0xf0000000;
+      break;
+   case PIPE_PRIM_LINE_STRIP:
+      gp->hdr[3] = 0x06000000;
+      gp->hdr[0] |= 0x10000000;
+      break;
+   case PIPE_PRIM_TRIANGLE_STRIP:
+      gp->hdr[3] = 0x07000000;
+      gp->hdr[0] |= 0x10000000;
+      break;
+   default:
+      assert(0);
+      break;
+   }
+
+   gp->hdr[4] = max_output_verts & 0x1ff;
+
+   return nvc0_vp_gp_gen_header(gp, ti);
+}
+
+static int
+nvc0_fp_gen_header(struct nvc0_program *fp, struct nvc0_translation_info *ti)
+{
+   int i, c;
+   unsigned a, m;
+   
+   fp->hdr[0] = 0x21462;
+   fp->hdr[5] = 0x80000000; /* getting a trap if FRAG_COORD_UMASK.w = 0 */
+
+   if (ti->scan.uses_kill)
+      fp->hdr[0] |= 0x8000;
+   if (ti->scan.writes_z) {
+      fp->hdr[19] |= 0x2;
+      if (ti->scan.num_outputs > 2)
+         fp->hdr[0] |= 0x4000; /* FP_MULTIPLE_COLOR_OUTPUTS */
+   } else {
+   if (ti->scan.num_outputs > 1)
+      fp->hdr[0] |= 0x4000; /* FP_MULTIPLE_COLOR_OUTPUTS */
+   }
+
+   for (i = 0; i <= ti->scan.file_max[TGSI_FILE_INPUT]; ++i) {
+      m = ti->interp_mode[i];
+      for (c = 0; c < 4; ++c) {
+         if (!ti->input_access[i][c])
+            continue;
+         a = ti->input_loc[i][c] / 2;
+         if ((a & ~7) == 0x70/2)
+            fp->hdr[5] |= 1 << (28 + (a & 7) / 2); /* FRAG_COORD_UMASK */
+         else
+            fp->hdr[4 + a / 32] |= m << (a % 32);
+      }
+   }
+
+   for (i = 0; i <= ti->scan.file_max[TGSI_FILE_OUTPUT]; ++i) {
+      if (i != ti->fp_depth_output)
+         fp->hdr[18] |= 0xf << ti->output_loc[i][0];
+   }
+
+   return 0;
+}
+
+static boolean
+nvc0_prog_scan(struct nvc0_translation_info *ti)
+{
+   struct nvc0_program *prog = ti->prog;
+   struct tgsi_parse_context parse;
+   int ret;
+   unsigned i;
+
+#ifdef NOUVEAU_DEBUG
+   tgsi_dump(prog->pipe.tokens, 0);
+#endif
+
+   tgsi_scan_shader(prog->pipe.tokens, &ti->scan);
+
+   if (ti->prog->type == PIPE_SHADER_FRAGMENT) {
+      ti->fp_depth_output = 255;
+      for (i = 0; i < ti->scan.num_outputs; ++i)
+         if (ti->scan.output_semantic_name[i] == TGSI_SEMANTIC_POSITION)
+            ti->fp_depth_output = i;
+   }
+
+   ti->subr =
+      CALLOC(ti->scan.opcode_count[TGSI_OPCODE_BGNSUB], sizeof(ti->subr[0]));
+
+   ti->immd32 = (uint32_t *)MALLOC(ti->scan.immediate_count * 16);
+   ti->immd32_ty = (ubyte *)MALLOC(ti->scan.immediate_count * sizeof(ubyte));
+
+   ti->insns = MALLOC(ti->scan.num_instructions * sizeof(ti->insns[0]));
+
+   tgsi_parse_init(&parse, prog->pipe.tokens);
+   while (!tgsi_parse_end_of_tokens(&parse)) {
+      tgsi_parse_token(&parse);
+
+      switch (parse.FullToken.Token.Type) {
+      case TGSI_TOKEN_TYPE_IMMEDIATE:
+         prog_immediate(ti, &parse.FullToken.FullImmediate);
+         break;
+      case TGSI_TOKEN_TYPE_DECLARATION:
+         prog_decl(ti, &parse.FullToken.FullDeclaration);
+         break;
+      case TGSI_TOKEN_TYPE_INSTRUCTION:
+         ti->insns[ti->num_insns] = parse.FullToken.FullInstruction;
+         prog_inst(ti, &parse.FullToken.FullInstruction, ++ti->num_insns);
+         break;
+      default:
+         break;
+      }
+   }
+
+   for (i = 0; i < ti->num_subrs; ++i) {
+      unsigned pc = ti->subr[i].id;
+      while (ti->insns[pc].Instruction.Opcode != TGSI_OPCODE_ENDSUB)
+         prog_subroutine_inst(&ti->subr[i], &ti->insns[pc++]);
+   }
+
+   switch (prog->type) {
+   case PIPE_SHADER_VERTEX:
+      ti->input_file = NV_FILE_MEM_A;
+      ti->output_file = NV_FILE_MEM_V;
+      ret = nvc0_vp_gen_header(prog, ti);
+      break;
+      /*
+   case PIPE_SHADER_TESSELLATION_CONTROL:
+      ret = nvc0_tcp_gen_header(ti);
+      break;
+   case PIPE_SHADER_TESSELLATION_EVALUATION:
+      ret = nvc0_tep_gen_header(ti);
+      break;
+   case PIPE_SHADER_GEOMETRY:
+      ret = nvc0_gp_gen_header(ti);
+      break;
+      */
+   case PIPE_SHADER_FRAGMENT:
+      ti->input_file = NV_FILE_MEM_V;
+      ti->output_file = NV_FILE_GPR;
+      ret = nvc0_fp_gen_header(prog, ti);
+      break;
+   default:
+      assert(!"unsupported program type");
+      ret = -1;
+      break;
+   }
+
+   assert(!ret);
+   return ret;
+}
+
+boolean
+nvc0_program_translate(struct nvc0_program *prog)
+{
+   struct nvc0_translation_info *ti;
+   int ret;
+
+   ti = CALLOC_STRUCT(nvc0_translation_info);
+   ti->prog = prog;
+
+   ti->edgeflag_out = PIPE_MAX_SHADER_OUTPUTS;
+
+   if (prog->type == PIPE_SHADER_VERTEX && prog->vp.num_ucps)
+      ti->append_ucp = TRUE;
+
+   ret = nvc0_prog_scan(ti);
+   if (ret) {
+      NOUVEAU_ERR("unsupported shader program\n");
+      goto out;
+   }
+
+   ret = nvc0_generate_code(ti);
+   if (ret)
+      NOUVEAU_ERR("shader translation failed\n");
+
+   {
+      unsigned i;
+      for (i = 0; i < sizeof(prog->hdr) / sizeof(prog->hdr[0]); ++i)
+         debug_printf("HDR[%02lx] = 0x%08x\n",
+                      i * sizeof(prog->hdr[0]), prog->hdr[i]);
+   }
+
+out:
+   if (ti->immd32)
+      FREE(ti->immd32);
+   if (ti->immd32_ty)
+      FREE(ti->immd32_ty);
+   if (ti->insns)
+      FREE(ti->insns);
+   if (ti->subr)
+      FREE(ti->subr);
+   FREE(ti);
+   return ret ? FALSE : TRUE;
+}
+
+void
+nvc0_program_destroy(struct nvc0_context *nvc0, struct nvc0_program *prog)
+{
+   if (prog->res)
+      nouveau_resource_free(&prog->res);
+
+   if (prog->code)
+      FREE(prog->code);
+   if (prog->relocs)
+      FREE(prog->relocs);
+
+   memset(prog->hdr, 0, sizeof(prog->hdr));
+
+   prog->translated = FALSE;
+}
diff --git a/src/gallium/drivers/nvc0/nvc0_program.h b/src/gallium/drivers/nvc0/nvc0_program.h
new file mode 100644
index 00000000000..1271303144e
--- /dev/null
+++ b/src/gallium/drivers/nvc0/nvc0_program.h
@@ -0,0 +1,84 @@
+
+#ifndef __NVC0_PROGRAM_H__
+#define __NVC0_PROGRAM_H__
+
+#include "pipe/p_state.h"
+#include "tgsi/tgsi_scan.h"
+
+#define NVC0_CAP_MAX_PROGRAM_TEMPS 64
+
+#define NVC0_SHADER_HEADER_SIZE (20 * 4)
+
+struct nvc0_program {
+   struct pipe_shader_state pipe;
+
+   ubyte type;
+   boolean translated;
+   ubyte max_gpr;
+
+   uint32_t *code;
+   unsigned code_base;
+   unsigned code_size;
+   unsigned parm_size;
+
+   uint32_t hdr[20];
+
+   uint32_t flags[2]; /* FP_ZORDER */
+
+   struct {
+      uint8_t edgeflag;
+      uint8_t num_ucps;
+   } vp;
+
+   void *relocs;
+   unsigned num_relocs;
+
+   struct nouveau_resource *res;
+};
+
+/* first 2 bits are written into the program header, for each input */
+#define NVC0_INTERP_FLAT          (1 << 0)
+#define NVC0_INTERP_PERSPECTIVE   (2 << 0)
+#define NVC0_INTERP_LINEAR        (3 << 0)
+#define NVC0_INTERP_CENTROID      (1 << 2)
+
+/* analyze TGSI and see which TEMP[] are used as subroutine inputs/outputs */
+struct nvc0_subroutine {
+   unsigned id;
+   unsigned first_insn;
+   uint32_t argv[NVC0_CAP_MAX_PROGRAM_TEMPS][4];
+   uint32_t retv[NVC0_CAP_MAX_PROGRAM_TEMPS][4];
+};
+
+struct nvc0_translation_info {
+   struct nvc0_program *prog;
+   struct tgsi_full_instruction *insns;
+   unsigned num_insns;
+   ubyte input_file;
+   ubyte output_file;
+   ubyte fp_depth_output;
+   uint16_t input_loc[PIPE_MAX_SHADER_INPUTS][4];
+   uint16_t output_loc[PIPE_MAX_SHADER_OUTPUTS][4];
+   uint16_t sysval_loc[TGSI_SEMANTIC_COUNT];
+   int input_access[PIPE_MAX_SHADER_INPUTS][4];
+   int output_access[PIPE_MAX_SHADER_OUTPUTS][4];
+   ubyte interp_mode[PIPE_MAX_SHADER_INPUTS];
+   boolean indirect_inputs;
+   boolean indirect_outputs;
+   boolean require_stores;
+   uint32_t *immd32;
+   ubyte *immd32_ty;
+   unsigned immd32_nr;
+   ubyte edgeflag_out;
+   struct nvc0_subroutine *subr;
+   unsigned num_subrs;
+   boolean append_ucp;
+   struct tgsi_shader_info scan;
+};
+
+int nvc0_generate_code(struct nvc0_translation_info *);
+
+void nvc0_relocate_program(struct nvc0_program *,
+                           uint32_t code_base, uint32_t data_base);
+
+#endif
diff --git a/src/gallium/drivers/nvc0/nvc0_push.c b/src/gallium/drivers/nvc0/nvc0_push.c
new file mode 100644
index 00000000000..ccbb776447f
--- /dev/null
+++ b/src/gallium/drivers/nvc0/nvc0_push.c
@@ -0,0 +1,273 @@
+
+#include "pipe/p_context.h"
+#include "pipe/p_state.h"
+#include "util/u_inlines.h"
+#include "util/u_format.h"
+#include "translate/translate.h"
+
+#include "nvc0_context.h"
+#include "nvc0_resource.h"
+
+#include "nvc0_3d.xml.h"
+
+struct push_context {
+   struct nouveau_channel *chan;
+
+   void *idxbuf;
+
+   float edgeflag;
+   int edgeflag_attr;
+
+   uint32_t vertex_words;
+   uint32_t packet_vertex_limit;
+
+   struct translate *translate;
+
+   boolean primitive_restart;
+   uint32_t prim;
+   uint32_t restart_index;
+};
+
+static INLINE unsigned
+prim_restart_search_i08(uint8_t *elts, unsigned push, uint8_t index)
+{
+   unsigned i;
+   for (i = 0; i < push; ++i)
+      if (elts[i] == index)
+         break;
+   return i;
+}
+
+static INLINE unsigned
+prim_restart_search_i16(uint16_t *elts, unsigned push, uint16_t index)
+{
+   unsigned i;
+   for (i = 0; i < push; ++i)
+      if (elts[i] == index)
+         break;
+   return i;
+}
+
+static INLINE unsigned
+prim_restart_search_i32(uint32_t *elts, unsigned push, uint32_t index)
+{
+   unsigned i;
+   for (i = 0; i < push; ++i)
+      if (elts[i] == index)
+         break;
+   return i;
+}
+
+static void
+emit_vertices_i08(struct push_context *ctx, unsigned start, unsigned count)
+{
+   uint8_t *elts = (uint8_t *)ctx->idxbuf + start;
+
+   while (count) {
+      unsigned push = MIN2(count, ctx->packet_vertex_limit);
+      unsigned size, nr;
+
+      nr = push;
+      if (ctx->primitive_restart)
+         nr = prim_restart_search_i08(elts, push, ctx->restart_index);
+
+      size = ctx->vertex_words * nr;
+
+      BEGIN_RING_NI(ctx->chan, RING_3D(VERTEX_DATA), size);
+
+      ctx->translate->run_elts8(ctx->translate, elts, push, 0, ctx->chan->cur);
+
+      ctx->chan->cur += size;
+      count -= push;
+      elts += push;
+
+      if (nr != push) {
+         BEGIN_RING(ctx->chan, RING_3D(VERTEX_END_GL), 2);
+         OUT_RING  (ctx->chan, 0);
+         OUT_RING  (ctx->chan, ctx->prim);
+      }
+   }
+}
+
+static void
+emit_vertices_i16(struct push_context *ctx, unsigned start, unsigned count)
+{
+   uint16_t *elts = (uint16_t *)ctx->idxbuf + start;
+
+   while (count) {
+      unsigned push = MIN2(count, ctx->packet_vertex_limit);
+      unsigned size, nr;
+
+      nr = push;
+      if (ctx->primitive_restart)
+         nr = prim_restart_search_i16(elts, push, ctx->restart_index);
+
+      size = ctx->vertex_words * nr;
+
+      BEGIN_RING_NI(ctx->chan, RING_3D(VERTEX_DATA), size);
+
+      ctx->translate->run_elts16(ctx->translate, elts, push, 0, ctx->chan->cur);
+
+      ctx->chan->cur += size;
+      count -= push;
+      elts += push;
+
+      if (nr != push) {
+         BEGIN_RING(ctx->chan, RING_3D(VERTEX_END_GL), 2);
+         OUT_RING  (ctx->chan, 0);
+         OUT_RING  (ctx->chan, ctx->prim);
+      }
+   }
+}
+
+static void
+emit_vertices_i32(struct push_context *ctx, unsigned start, unsigned count)
+{
+   uint32_t *elts = (uint32_t *)ctx->idxbuf + start;
+
+   while (count) {
+      unsigned push = MIN2(count, ctx->packet_vertex_limit);
+      unsigned size, nr;
+
+      nr = push;
+      if (ctx->primitive_restart)
+         nr = prim_restart_search_i32(elts, push, ctx->restart_index);
+
+      size = ctx->vertex_words * nr;
+
+      BEGIN_RING_NI(ctx->chan, RING_3D(VERTEX_DATA), size);
+
+      ctx->translate->run_elts(ctx->translate, elts, push, 0, ctx->chan->cur);
+
+      ctx->chan->cur += size;
+      count -= push;
+      elts += push;
+
+      if (nr != push) {
+         BEGIN_RING(ctx->chan, RING_3D(VERTEX_END_GL), 2);
+         OUT_RING  (ctx->chan, 0);
+         OUT_RING  (ctx->chan, ctx->prim);
+      }
+   }
+}
+
+static void
+emit_vertices_seq(struct push_context *ctx, unsigned start, unsigned count)
+{
+   while (count) {
+      unsigned push = MIN2(count, ctx->packet_vertex_limit);
+      unsigned size = ctx->vertex_words * push;
+
+      BEGIN_RING_NI(ctx->chan, RING_3D(VERTEX_DATA), size);
+
+      ctx->translate->run(ctx->translate, start, push, 0, ctx->chan->cur);
+      ctx->chan->cur += size;
+      count -= push;
+      start += push;
+   }
+}
+
+
+#define NVC0_PRIM_GL_CASE(n) \
+   case PIPE_PRIM_##n: return NVC0_3D_VERTEX_BEGIN_GL_PRIMITIVE_##n
+
+static INLINE unsigned
+nvc0_prim_gl(unsigned prim)
+{
+   switch (prim) {
+   NVC0_PRIM_GL_CASE(POINTS);
+   NVC0_PRIM_GL_CASE(LINES);
+   NVC0_PRIM_GL_CASE(LINE_LOOP);
+   NVC0_PRIM_GL_CASE(LINE_STRIP);
+   NVC0_PRIM_GL_CASE(TRIANGLES);
+   NVC0_PRIM_GL_CASE(TRIANGLE_STRIP);
+   NVC0_PRIM_GL_CASE(TRIANGLE_FAN);
+   NVC0_PRIM_GL_CASE(QUADS);
+   NVC0_PRIM_GL_CASE(QUAD_STRIP);
+   NVC0_PRIM_GL_CASE(POLYGON);
+   NVC0_PRIM_GL_CASE(LINES_ADJACENCY);
+   NVC0_PRIM_GL_CASE(LINE_STRIP_ADJACENCY);
+   NVC0_PRIM_GL_CASE(TRIANGLES_ADJACENCY);
+   NVC0_PRIM_GL_CASE(TRIANGLE_STRIP_ADJACENCY);
+   /*
+   NVC0_PRIM_GL_CASE(PATCHES); */
+   default:
+      return NVC0_3D_VERTEX_BEGIN_GL_PRIMITIVE_POINTS;
+      break;
+   }
+}
+
+void
+nvc0_push_vbo(struct nvc0_context *nvc0, const struct pipe_draw_info *info)
+{
+   struct push_context ctx;
+   unsigned i, index_size;
+   unsigned inst = info->instance_count;
+
+   ctx.chan = nvc0->screen->base.channel;
+   ctx.translate = nvc0->vertex->translate;
+   ctx.packet_vertex_limit = nvc0->vertex->vtx_per_packet_max;
+   ctx.vertex_words = nvc0->vertex->vtx_size;
+
+   for (i = 0; i < nvc0->num_vtxbufs; ++i) {
+      uint8_t *data;
+      struct pipe_vertex_buffer *vb = &nvc0->vtxbuf[i];
+      struct nvc0_resource *res = nvc0_resource(vb->buffer);
+
+      data = nvc0_resource_map_offset(nvc0, res,
+                                      vb->buffer_offset, NOUVEAU_BO_RD);
+      if (info->indexed)
+         data += info->index_bias * vb->stride;
+
+      ctx.translate->set_buffer(ctx.translate, i, data, vb->stride, ~0);
+   }
+
+   if (info->indexed) {
+      ctx.idxbuf = nvc0_resource_map_offset(nvc0,
+                                            nvc0_resource(nvc0->idxbuf.buffer),
+                                            nvc0->idxbuf.offset, NOUVEAU_BO_RD);
+      if (!ctx.idxbuf)
+         return;
+      index_size = nvc0->idxbuf.index_size;
+      ctx.primitive_restart = info->primitive_restart;
+      ctx.restart_index = info->restart_index;
+   } else {
+      ctx.idxbuf = NULL;
+      index_size = 0;
+      ctx.primitive_restart = FALSE;
+      ctx.restart_index = 0;
+   }
+
+   ctx.prim = nvc0_prim_gl(info->mode);
+
+   while (inst--) {
+      BEGIN_RING(ctx.chan, RING_3D(VERTEX_BEGIN_GL), 1);
+      OUT_RING  (ctx.chan, ctx.prim);
+      switch (index_size) {
+      case 0:
+         emit_vertices_seq(&ctx, info->start, info->count);
+         break;
+      case 1:
+         emit_vertices_i08(&ctx, info->start, info->count);
+         break;
+      case 2:
+         emit_vertices_i16(&ctx, info->start, info->count);
+         break;
+      case 4:
+         emit_vertices_i32(&ctx, info->start, info->count);
+         break;
+      default:
+         assert(0);
+         break;
+      }
+      IMMED_RING(ctx.chan, RING_3D(VERTEX_END_GL), 0);
+
+      ctx.prim |= NVC0_3D_VERTEX_BEGIN_GL_INSTANCE_NEXT;
+   }
+
+   if (info->indexed)
+      nvc0_resource_unmap(nvc0_resource(nvc0->idxbuf.buffer));
+
+   for (i = 0; i < nvc0->num_vtxbufs; ++i)
+      nvc0_resource_unmap(nvc0_resource(nvc0->vtxbuf[i].buffer));
+}
diff --git a/src/gallium/drivers/nvc0/nvc0_push2.c b/src/gallium/drivers/nvc0/nvc0_push2.c
new file mode 100644
index 00000000000..6f51600558a
--- /dev/null
+++ b/src/gallium/drivers/nvc0/nvc0_push2.c
@@ -0,0 +1,333 @@
+
+#if 0 /* not used, kept for now to compare with util/translate */
+
+#include "pipe/p_context.h"
+#include "pipe/p_state.h"
+#include "util/u_inlines.h"
+#include "util/u_format.h"
+#include "translate/translate.h"
+
+#include "nvc0_context.h"
+#include "nvc0_resource.h"
+
+#include "nvc0_3d.xml.h"
+
+struct push_context {
+   struct nvc0_context *nvc0;
+
+   uint vertex_size;
+
+   void *idxbuf;
+   uint idxsize;
+
+   float edgeflag;
+   int edgeflag_input;
+
+   struct {
+      void *map;
+      void (*push)(struct nouveau_channel *, void *);
+      uint32_t stride;
+      uint32_t divisor;
+      uint32_t step;
+   } attr[32];
+   int num_attrs;
+};
+
+static void
+emit_b32_1(struct nouveau_channel *chan, void *data)
+{
+   uint32_t *v = data;
+
+   OUT_RING(chan, v[0]);
+}
+
+static void
+emit_b32_2(struct nouveau_channel *chan, void *data)
+{
+   uint32_t *v = data;
+
+   OUT_RING(chan, v[0]);
+   OUT_RING(chan, v[1]);
+}
+
+static void
+emit_b32_3(struct nouveau_channel *chan, void *data)
+{
+   uint32_t *v = data;
+
+   OUT_RING(chan, v[0]);
+   OUT_RING(chan, v[1]);
+   OUT_RING(chan, v[2]);
+}
+
+static void
+emit_b32_4(struct nouveau_channel *chan, void *data)
+{
+   uint32_t *v = data;
+
+   OUT_RING(chan, v[0]);
+   OUT_RING(chan, v[1]);
+   OUT_RING(chan, v[2]);
+   OUT_RING(chan, v[3]);
+}
+
+static void
+emit_b16_1(struct nouveau_channel *chan, void *data)
+{
+   uint16_t *v = data;
+
+   OUT_RING(chan, v[0]);
+}
+
+static void
+emit_b16_3(struct nouveau_channel *chan, void *data)
+{
+   uint16_t *v = data;
+
+   OUT_RING(chan, (v[1] << 16) | v[0]);
+   OUT_RING(chan, v[2]);
+}
+
+static void
+emit_b08_1(struct nouveau_channel *chan, void *data)
+{
+   uint8_t *v = data;
+
+   OUT_RING(chan, v[0]);
+}
+
+static void
+emit_b08_3(struct nouveau_channel *chan, void *data)
+{
+   uint8_t *v = data;
+
+   OUT_RING(chan, (v[2] << 16) | (v[1] << 8) | v[0]);
+}
+
+static void
+emit_b64_1(struct nouveau_channel *chan, void *data)
+{
+   double *v = data;
+
+   OUT_RINGf(chan, v[0]);
+}
+
+static void
+emit_b64_2(struct nouveau_channel *chan, void *data)
+{
+   double *v = data;
+
+   OUT_RINGf(chan, v[0]);
+   OUT_RINGf(chan, v[1]);
+}
+
+static void
+emit_b64_3(struct nouveau_channel *chan, void *data)
+{
+   double *v = data;
+
+   OUT_RINGf(chan, v[0]);
+   OUT_RINGf(chan, v[1]);
+   OUT_RINGf(chan, v[2]);
+}
+
+static void
+emit_b64_4(struct nouveau_channel *chan, void *data)
+{
+   double *v = data;
+
+   OUT_RINGf(chan, v[0]);
+   OUT_RINGf(chan, v[1]);
+   OUT_RINGf(chan, v[2]);
+   OUT_RINGf(chan, v[3]);   
+}
+
+static INLINE void
+emit_vertex(struct push_context *ctx, unsigned n)
+{
+   struct nouveau_channel *chan = ctx->nvc0->screen->base.channel;
+   int i;
+
+   if (ctx->edgeflag_input < 32) {
+      /* TODO */
+   }
+
+   BEGIN_RING_NI(chan, RING_3D(VERTEX_DATA), ctx->vertex_size);
+   for (i = 0; i < ctx->num_attrs; ++i)
+      ctx->attr[i].push(chan,
+                        (uint8_t *)ctx->attr[i].map + n * ctx->attr[i].stride);
+}
+
+static void
+emit_edgeflag(struct push_context *ctx, boolean enabled)
+{
+   struct nouveau_channel *chan = ctx->nvc0->screen->base.channel;
+   
+   IMMED_RING(chan, RING_3D(EDGEFLAG_ENABLE), enabled);
+}
+
+static void
+emit_elt08(struct push_context *ctx, unsigned start, unsigned count)
+{
+   uint8_t *idxbuf = ctx->idxbuf;
+
+   while (count--)
+      emit_vertex(ctx, idxbuf[start++]);
+}
+
+static void
+emit_elt16(struct push_context *ctx, unsigned start, unsigned count)
+{
+   uint16_t *idxbuf = ctx->idxbuf;
+
+   while (count--)
+      emit_vertex(ctx, idxbuf[start++]);
+}
+
+static void
+emit_elt32(struct push_context *ctx, unsigned start, unsigned count)
+{
+   uint32_t *idxbuf = ctx->idxbuf;
+
+   while (count--)
+      emit_vertex(ctx, idxbuf[start++]);
+}
+
+static void
+emit_seq(struct push_context *ctx, unsigned start, unsigned count)
+{
+   while (count--)
+      emit_vertex(ctx, start++);
+}
+
+#define NVC0_PRIM_GL_CASE(n) \
+   case PIPE_PRIM_##n: return NVC0_3D_VERTEX_BEGIN_GL_PRIMITIVE_##n
+
+static INLINE unsigned
+nvc0_prim_gl(unsigned prim)
+{
+   switch (prim) {
+   NVC0_PRIM_GL_CASE(POINTS);
+   NVC0_PRIM_GL_CASE(LINES);
+   NVC0_PRIM_GL_CASE(LINE_LOOP);
+   NVC0_PRIM_GL_CASE(LINE_STRIP);
+   NVC0_PRIM_GL_CASE(TRIANGLES);
+   NVC0_PRIM_GL_CASE(TRIANGLE_STRIP);
+   NVC0_PRIM_GL_CASE(TRIANGLE_FAN);
+   NVC0_PRIM_GL_CASE(QUADS);
+   NVC0_PRIM_GL_CASE(QUAD_STRIP);
+   NVC0_PRIM_GL_CASE(POLYGON);
+   NVC0_PRIM_GL_CASE(LINES_ADJACENCY);
+   NVC0_PRIM_GL_CASE(LINE_STRIP_ADJACENCY);
+   NVC0_PRIM_GL_CASE(TRIANGLES_ADJACENCY);
+   NVC0_PRIM_GL_CASE(TRIANGLE_STRIP_ADJACENCY);
+   /*
+   NVC0_PRIM_GL_CASE(PATCHES); */
+   default:
+      return NVC0_3D_VERTEX_BEGIN_GL_PRIMITIVE_POINTS;
+      break;
+   }
+}
+
+void
+nvc0_push_vbo2(struct nvc0_context *nvc0, const struct pipe_draw_info *info)
+{
+   struct push_context ctx;
+   unsigned i, n;
+   unsigned inst = info->instance_count;
+   unsigned prim = nvc0_prim_gl(info->mode);
+
+   ctx.nvc0 = nvc0;
+   ctx.vertex_size = nvc0->vertex->vtx_size;
+   ctx.idxbuf = NULL;
+   ctx.num_attrs = 0;
+   ctx.edgeflag = 0.5f;
+   ctx.edgeflag_input = 32;
+
+   for (i = 0; i < nvc0->vertex->num_elements; ++i) {
+      struct pipe_vertex_element *ve = &nvc0->vertex->element[i].pipe;
+      struct pipe_vertex_buffer *vb = &nvc0->vtxbuf[ve->vertex_buffer_index];
+      struct nouveau_bo *bo = nvc0_resource(vb->buffer)->bo;
+      unsigned nr_components;
+
+      if (!(nvc0->vbo_fifo & (1 << i)))
+         continue;
+      n = ctx.num_attrs++;
+
+      if (nouveau_bo_map(bo, NOUVEAU_BO_RD))
+         return;
+      ctx.attr[n].map = (uint8_t *)bo->map + vb->buffer_offset + ve->src_offset;
+
+      nouveau_bo_unmap(bo);
+
+      ctx.attr[n].stride = vb->stride;
+      ctx.attr[n].divisor = ve->instance_divisor;
+
+      nr_components = util_format_get_nr_components(ve->src_format);
+      switch (util_format_get_component_bits(ve->src_format,
+                                             UTIL_FORMAT_COLORSPACE_RGB, 0)) {
+      case 8:
+         switch (nr_components) {
+         case 1: ctx.attr[n].push = emit_b08_1; break;
+         case 2: ctx.attr[n].push = emit_b16_1; break;
+         case 3: ctx.attr[n].push = emit_b08_3; break;
+         case 4: ctx.attr[n].push = emit_b32_1; break;
+         }
+         break;
+      case 16:
+         switch (nr_components) {
+         case 1: ctx.attr[n].push = emit_b16_1; break;
+         case 2: ctx.attr[n].push = emit_b32_1; break;
+         case 3: ctx.attr[n].push = emit_b16_3; break;
+         case 4: ctx.attr[n].push = emit_b32_2; break;
+         }
+         break;
+      case 32:
+         switch (nr_components) {
+         case 1: ctx.attr[n].push = emit_b32_1; break;
+         case 2: ctx.attr[n].push = emit_b32_2; break;
+         case 3: ctx.attr[n].push = emit_b32_3; break;
+         case 4: ctx.attr[n].push = emit_b32_4; break;
+         }
+         break;
+      default:
+         assert(0);
+         break;
+      }
+   }
+
+   if (info->indexed) {
+      struct nvc0_resource *res = nvc0_resource(nvc0->idxbuf.buffer);
+      if (!res || nouveau_bo_map(res->bo, NOUVEAU_BO_RD))
+         return;
+      ctx.idxbuf = (uint8_t *)res->bo->map + nvc0->idxbuf.offset + res->offset;
+      nouveau_bo_unmap(res->bo);
+      ctx.idxsize = nvc0->idxbuf.index_size;
+   } else {
+      ctx.idxsize = 0;
+   }
+
+   while (inst--) {
+      BEGIN_RING(nvc0->screen->base.channel, RING_3D(VERTEX_BEGIN_GL), 1);
+      OUT_RING  (nvc0->screen->base.channel, prim);
+      switch (ctx.idxsize) {
+      case 0:
+         emit_seq(&ctx, info->start, info->count);
+         break;
+      case 1:
+         emit_elt08(&ctx, info->start, info->count);
+         break;
+      case 2:
+         emit_elt16(&ctx, info->start, info->count);
+         break;
+      case 4:
+         emit_elt32(&ctx, info->start, info->count);
+         break;
+      }
+      IMMED_RING(nvc0->screen->base.channel, RING_3D(VERTEX_END_GL), 0);
+
+      prim |= NVC0_3D_VERTEX_BEGIN_GL_INSTANCE_NEXT;
+   }
+}
+
+#endif
diff --git a/src/gallium/drivers/nvc0/nvc0_resource.c b/src/gallium/drivers/nvc0/nvc0_resource.c
new file mode 100644
index 00000000000..7e42cedd163
--- /dev/null
+++ b/src/gallium/drivers/nvc0/nvc0_resource.c
@@ -0,0 +1,71 @@
+
+#include "pipe/p_context.h"
+#include "nvc0_resource.h"
+#include "nouveau/nouveau_screen.h"
+
+static unsigned
+nvc0_resource_is_referenced(struct pipe_context *pipe,
+                            struct pipe_resource *resource,
+                            unsigned face, int layer)
+{
+   struct nvc0_resource *res = nvc0_resource(resource);
+   unsigned flags = 0;
+
+#ifdef NOUVEAU_USERSPACE_MM
+   flags = res->status;
+#else
+   unsigned bo_flags = nouveau_bo_pending(res->bo);
+   if (bo_flags & NOUVEAU_BO_RD)
+      flags = PIPE_REFERENCED_FOR_READ;
+   if (bo_flags & NOUVEAU_BO_WR)
+      flags |= PIPE_REFERENCED_FOR_WRITE;
+#endif
+   return flags;
+}
+
+static struct pipe_resource *
+nvc0_resource_create(struct pipe_screen *screen,
+                     const struct pipe_resource *templ)
+{
+   switch (templ->target) {
+   case PIPE_BUFFER:
+      return nvc0_buffer_create(screen, templ);
+   default:
+      return nvc0_miptree_create(screen, templ);
+   }
+}
+
+static struct pipe_resource *
+nvc0_resource_from_handle(struct pipe_screen * screen,
+                          const struct pipe_resource *templ,
+                          struct winsys_handle *whandle)
+{
+   if (templ->target == PIPE_BUFFER)
+      return NULL;
+   else
+      return nvc0_miptree_from_handle(screen, templ, whandle);
+}
+
+void
+nvc0_init_resource_functions(struct pipe_context *pcontext)
+{
+   pcontext->get_transfer = u_get_transfer_vtbl;
+   pcontext->transfer_map = u_transfer_map_vtbl;
+   pcontext->transfer_flush_region = u_transfer_flush_region_vtbl;
+   pcontext->transfer_unmap = u_transfer_unmap_vtbl;
+   pcontext->transfer_destroy = u_transfer_destroy_vtbl;
+   pcontext->transfer_inline_write = u_transfer_inline_write_vtbl;
+   pcontext->is_resource_referenced = nvc0_resource_is_referenced;
+   pcontext->create_surface = nvc0_miptree_surface_new;
+   pcontext->surface_destroy = nvc0_miptree_surface_del;
+}
+
+void
+nvc0_screen_init_resource_functions(struct pipe_screen *pscreen)
+{
+   pscreen->resource_create = nvc0_resource_create;
+   pscreen->resource_from_handle = nvc0_resource_from_handle;
+   pscreen->resource_get_handle = u_resource_get_handle_vtbl;
+   pscreen->resource_destroy = u_resource_destroy_vtbl;
+   pscreen->user_buffer_create = nvc0_user_buffer_create;
+}
diff --git a/src/gallium/drivers/nvc0/nvc0_resource.h b/src/gallium/drivers/nvc0/nvc0_resource.h
new file mode 100644
index 00000000000..d33e2f0ed0a
--- /dev/null
+++ b/src/gallium/drivers/nvc0/nvc0_resource.h
@@ -0,0 +1,194 @@
+
+#ifndef __NVC0_RESOURCE_H__
+#define __NVC0_RESOURCE_H__
+
+#include "util/u_transfer.h"
+#include "util/u_double_list.h"
+#define NOUVEAU_NVC0
+#include "nouveau/nouveau_winsys.h"
+#undef NOUVEAU_NVC0
+
+#include "nvc0_fence.h"
+
+struct pipe_resource;
+struct nouveau_bo;
+struct nvc0_context;
+
+#define NVC0_BUFFER_SCORE_MIN -25000
+#define NVC0_BUFFER_SCORE_MAX  25000
+#define NVC0_BUFFER_SCORE_VRAM_THRESHOLD 20000
+
+#define NVC0_BUFFER_STATUS_DIRTY       (1 << 0)
+#define NVC0_BUFFER_STATUS_USER_MEMORY (1 << 7)
+
+/* Resources, if mapped into the GPU's address space, are guaranteed to
+ * have constant virtual addresses.
+ * The address of a resource will lie within the nouveau_bo referenced,
+ * and this bo should be added to the memory manager's validation list.
+ */
+struct nvc0_resource {
+   struct pipe_resource base;
+   const struct u_resource_vtbl *vtbl;
+
+   uint8_t *data;
+   struct nouveau_bo *bo;
+   uint32_t offset;
+
+   uint8_t status;
+   uint8_t domain;
+
+   int16_t score; /* low if mapped very often, if high can move to VRAM */
+
+   struct nvc0_fence *fence;
+   struct nvc0_fence *fence_wr;
+
+   struct nvc0_mm_allocation *mm;
+};
+
+boolean
+nvc0_buffer_download(struct nvc0_context *, struct nvc0_resource *,
+                     unsigned start, unsigned size);
+
+boolean
+nvc0_buffer_migrate(struct nvc0_context *,
+                    struct nvc0_resource *, unsigned domain);
+
+static INLINE void
+nvc0_buffer_adjust_score(struct nvc0_context *nvc0, struct nvc0_resource *res,
+                         int16_t score)
+{
+   if (score < 0) {
+      if (res->score > NVC0_BUFFER_SCORE_MIN)
+         res->score += score;
+   } else
+   if (score > 0){
+      if (res->score < NVC0_BUFFER_SCORE_MAX)
+         res->score += score;
+      if (res->domain == NOUVEAU_BO_GART &&
+          res->score > NVC0_BUFFER_SCORE_VRAM_THRESHOLD)
+         nvc0_buffer_migrate(nvc0, res, NOUVEAU_BO_VRAM);
+   }
+}
+
+/* XXX: wait for fence (atm only using this for vertex push) */
+static INLINE void *
+nvc0_resource_map_offset(struct nvc0_context *nvc0,
+                         struct nvc0_resource *res, uint32_t offset,
+                         uint32_t flags)
+{
+   void *map;
+
+   nvc0_buffer_adjust_score(nvc0, res, -250);
+
+   if ((res->domain == NOUVEAU_BO_VRAM) &&
+       (res->status & NVC0_BUFFER_STATUS_DIRTY))
+      nvc0_buffer_download(nvc0, res, 0, res->base.width0);
+
+   if (res->domain != NOUVEAU_BO_GART)
+      return res->data + offset;
+
+   if (res->mm)
+      flags |= NOUVEAU_BO_NOSYNC;
+
+   if (nouveau_bo_map_range(res->bo, res->offset + offset,
+                            res->base.width0, flags))
+      return NULL;
+
+   map = res->bo->map;
+   nouveau_bo_unmap(res->bo);
+   return map;
+}
+
+static INLINE void
+nvc0_resource_unmap(struct nvc0_resource *res)
+{
+   /* no-op */
+}
+
+#define NVC0_TILE_DIM_SHIFT(m, d) (((m) >> (d * 4)) & 0xf)
+
+#define NVC0_TILE_PITCH(m)  (64 << NVC0_TILE_DIM_SHIFT(m, 0))
+#define NVC0_TILE_HEIGHT(m) ( 8 << NVC0_TILE_DIM_SHIFT(m, 1))
+#define NVC0_TILE_DEPTH(m)  ( 1 << NVC0_TILE_DIM_SHIFT(m, 2))
+
+#define NVC0_TILE_SIZE_2D(m) (((64 * 8) <<                     \
+                               NVC0_TILE_DIM_SHIFT(m, 0)) <<   \
+                              NVC0_TILE_DIM_SHIFT(m, 1))
+
+#define NVC0_TILE_SIZE(m) (NVC0_TILE_SIZE_2D(m) << NVC0_TILE_DIM_SHIFT(m, 2))
+
+struct nvc0_miptree_level {
+   uint32_t offset;
+   uint32_t pitch;
+   uint32_t tile_mode;
+};
+
+#define NVC0_MAX_TEXTURE_LEVELS 16
+
+struct nvc0_miptree {
+   struct nvc0_resource base;
+   struct nvc0_miptree_level level[NVC0_MAX_TEXTURE_LEVELS];
+   uint32_t total_size;
+   uint32_t layer_stride;
+   boolean layout_3d; /* TRUE if layer count varies with mip level */
+};
+
+static INLINE struct nvc0_miptree *
+nvc0_miptree(struct pipe_resource *pt)
+{
+   return (struct nvc0_miptree *)pt;
+}
+
+static INLINE struct nvc0_resource *
+nvc0_resource(struct pipe_resource *resource)
+{
+   return (struct nvc0_resource *)resource;
+}
+
+/* is resource mapped into the GPU's address space (i.e. VRAM or GART) ? */
+static INLINE boolean
+nvc0_resource_mapped_by_gpu(struct pipe_resource *resource)
+{
+   return nvc0_resource(resource)->domain != 0;
+}
+
+void
+nvc0_init_resource_functions(struct pipe_context *pcontext);
+
+void
+nvc0_screen_init_resource_functions(struct pipe_screen *pscreen);
+
+/* Internal functions:
+ */
+struct pipe_resource *
+nvc0_miptree_create(struct pipe_screen *pscreen,
+                    const struct pipe_resource *tmp);
+
+struct pipe_resource *
+nvc0_miptree_from_handle(struct pipe_screen *pscreen,
+                         const struct pipe_resource *template,
+                         struct winsys_handle *whandle);
+
+struct pipe_resource *
+nvc0_buffer_create(struct pipe_screen *pscreen,
+                   const struct pipe_resource *templ);
+
+struct pipe_resource *
+nvc0_user_buffer_create(struct pipe_screen *screen,
+                        void *ptr,
+                        unsigned bytes,
+                        unsigned usage);
+
+
+struct pipe_surface *
+nvc0_miptree_surface_new(struct pipe_context *,
+                         struct pipe_resource *,
+                         const struct pipe_surface *templ);
+
+void
+nvc0_miptree_surface_del(struct pipe_context *, struct pipe_surface *);
+
+boolean
+nvc0_migrate_vertices(struct nvc0_resource *buf, unsigned base, unsigned size);
+
+#endif
diff --git a/src/gallium/drivers/nvc0/nvc0_screen.c b/src/gallium/drivers/nvc0/nvc0_screen.c
new file mode 100644
index 00000000000..e149b907314
--- /dev/null
+++ b/src/gallium/drivers/nvc0/nvc0_screen.c
@@ -0,0 +1,665 @@
+/*
+ * Copyright 2010 Christoph Bumiller
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF
+ * OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "util/u_format_s3tc.h"
+#include "pipe/p_screen.h"
+
+#include "nvc0_fence.h"
+#include "nvc0_context.h"
+#include "nvc0_screen.h"
+
+#include "nouveau/nv_object.xml.h"
+#include "nvc0_graph_macros.h"
+
+static boolean
+nvc0_screen_is_format_supported(struct pipe_screen *pscreen,
+                                enum pipe_format format,
+                                enum pipe_texture_target target,
+                                unsigned sample_count,
+                                unsigned bindings, unsigned geom_flags)
+{
+   if (sample_count > 1)
+      return FALSE;
+
+   if (!util_format_s3tc_enabled) {
+      switch (format) {
+      case PIPE_FORMAT_DXT1_RGB:
+      case PIPE_FORMAT_DXT1_RGBA:
+      case PIPE_FORMAT_DXT3_RGBA:
+      case PIPE_FORMAT_DXT5_RGBA:
+         return FALSE;
+      default:
+         break;
+      }
+   }
+
+   /* transfers & shared are always supported */
+   bindings &= ~(PIPE_BIND_TRANSFER_READ |
+                 PIPE_BIND_TRANSFER_WRITE |
+                 PIPE_BIND_SHARED);
+
+   return (nvc0_format_table[format].usage & bindings) == bindings;
+}
+
+static int
+nvc0_screen_get_param(struct pipe_screen *pscreen, enum pipe_cap param)
+{
+   switch (param) {
+   case PIPE_CAP_MAX_TEXTURE_IMAGE_UNITS:
+   case PIPE_CAP_MAX_VERTEX_TEXTURE_UNITS:
+      return 32;
+   case PIPE_CAP_MAX_COMBINED_SAMPLERS:
+      return 64;
+   case PIPE_CAP_MAX_TEXTURE_2D_LEVELS:
+      return 13;
+   case PIPE_CAP_MAX_TEXTURE_3D_LEVELS:
+      return 10;
+   case PIPE_CAP_MAX_TEXTURE_CUBE_LEVELS:
+      return 13;
+   case PIPE_CAP_TEXTURE_MIRROR_CLAMP:
+   case PIPE_CAP_TEXTURE_MIRROR_REPEAT:
+   case PIPE_CAP_TEXTURE_SWIZZLE:
+   case PIPE_CAP_TEXTURE_SHADOW_MAP:
+   case PIPE_CAP_NPOT_TEXTURES:
+   case PIPE_CAP_ANISOTROPIC_FILTER:
+      return 1;
+   case PIPE_CAP_TWO_SIDED_STENCIL:
+   case PIPE_CAP_DEPTH_CLAMP:
+   case PIPE_CAP_DEPTHSTENCIL_CLEAR_SEPARATE:
+   case PIPE_CAP_POINT_SPRITE:
+      return 1;
+   case PIPE_CAP_GLSL:
+   case PIPE_CAP_SM3:
+      return 1;
+   case PIPE_CAP_MAX_RENDER_TARGETS:
+      return 8;
+   case PIPE_CAP_OCCLUSION_QUERY:
+      return 1;
+   case PIPE_CAP_TIMER_QUERY:
+   case PIPE_CAP_STREAM_OUTPUT:
+      return 0;
+   case PIPE_CAP_BLEND_EQUATION_SEPARATE:
+   case PIPE_CAP_INDEP_BLEND_ENABLE:
+   case PIPE_CAP_INDEP_BLEND_FUNC:
+      return 1;
+   case PIPE_CAP_TGSI_FS_COORD_ORIGIN_UPPER_LEFT:
+   case PIPE_CAP_TGSI_FS_COORD_PIXEL_CENTER_HALF_INTEGER:
+      return 1;
+   case PIPE_CAP_TGSI_FS_COORD_ORIGIN_LOWER_LEFT:
+   case PIPE_CAP_TGSI_FS_COORD_PIXEL_CENTER_INTEGER:
+      return 0;
+   case PIPE_CAP_SHADER_STENCIL_EXPORT:
+      return 0;
+   case PIPE_CAP_PRIMITIVE_RESTART:
+      return 1;
+   default:
+      NOUVEAU_ERR("unknown PIPE_CAP %d\n", param);
+      return 0;
+   }
+}
+
+static int
+nvc0_screen_get_shader_param(struct pipe_screen *pscreen, unsigned shader,
+                             enum pipe_shader_cap param)
+{
+   switch (shader) {
+   case PIPE_SHADER_VERTEX:
+      /*
+   case PIPE_SHADER_TESSELLATION_CONTROL:
+   case PIPE_SHADER_TESSELLATION_EVALUATION:
+      */
+   case PIPE_SHADER_GEOMETRY:
+   case PIPE_SHADER_FRAGMENT:
+      break;
+   default:
+      return 0;
+   }
+   
+   switch (param) {
+   case PIPE_SHADER_CAP_MAX_INSTRUCTIONS:
+   case PIPE_SHADER_CAP_MAX_ALU_INSTRUCTIONS:
+   case PIPE_SHADER_CAP_MAX_TEX_INSTRUCTIONS:
+   case PIPE_SHADER_CAP_MAX_TEX_INDIRECTIONS:
+      return 16384;
+   case PIPE_SHADER_CAP_MAX_CONTROL_FLOW_DEPTH:
+      return 4;
+   case PIPE_SHADER_CAP_MAX_INPUTS:
+      if (shader == PIPE_SHADER_VERTEX)
+         return 32;
+      return 0x300 / 16;
+   case PIPE_SHADER_CAP_MAX_CONSTS:
+      return 65536 / 16;
+   case PIPE_SHADER_CAP_MAX_CONST_BUFFERS:
+      return 14;
+   case PIPE_SHADER_CAP_MAX_ADDRS:
+      return 1;
+   case PIPE_SHADER_CAP_INDIRECT_INPUT_ADDR:
+   case PIPE_SHADER_CAP_INDIRECT_OUTPUT_ADDR:
+      return shader != PIPE_SHADER_FRAGMENT;
+   case PIPE_SHADER_CAP_INDIRECT_TEMP_ADDR:
+   case PIPE_SHADER_CAP_INDIRECT_CONST_ADDR:
+      return 1;
+   case PIPE_SHADER_CAP_MAX_PREDS:
+      return 0;
+   case PIPE_SHADER_CAP_MAX_TEMPS:
+      return NVC0_CAP_MAX_PROGRAM_TEMPS;
+   case PIPE_SHADER_CAP_TGSI_CONT_SUPPORTED:
+      return 1;
+   case PIPE_SHADER_CAP_SUBROUTINES:
+      return 0; /* please inline, or provide function declarations */
+   default:
+      NOUVEAU_ERR("unknown PIPE_SHADER_CAP %d\n", param);
+      return 0;
+   }
+}
+
+static float
+nvc0_screen_get_paramf(struct pipe_screen *pscreen, enum pipe_cap param)
+{
+   switch (param) {
+   case PIPE_CAP_MAX_LINE_WIDTH:
+   case PIPE_CAP_MAX_LINE_WIDTH_AA:
+      return 10.0f;
+   case PIPE_CAP_MAX_POINT_WIDTH:
+   case PIPE_CAP_MAX_POINT_WIDTH_AA:
+      return 64.0f;
+   case PIPE_CAP_MAX_TEXTURE_ANISOTROPY:
+      return 16.0f;
+   case PIPE_CAP_MAX_TEXTURE_LOD_BIAS:
+      return 4.0f;
+   default:
+      NOUVEAU_ERR("unknown PIPE_CAP %d\n", param);
+      return 0.0f;
+   }
+}
+
+static void
+nvc0_screen_destroy(struct pipe_screen *pscreen)
+{
+   struct nvc0_screen *screen = nvc0_screen(pscreen);
+
+   nvc0_fence_wait(screen->fence.current);
+   nvc0_fence_reference(&screen->fence.current, NULL);
+
+   nouveau_bo_ref(NULL, &screen->text);
+   nouveau_bo_ref(NULL, &screen->tls);
+   nouveau_bo_ref(NULL, &screen->txc);
+   nouveau_bo_ref(NULL, &screen->fence.bo);
+   nouveau_bo_ref(NULL, &screen->mp_stack_bo);
+
+   nouveau_resource_destroy(&screen->text_heap);
+
+   if (screen->tic.entries)
+      FREE(screen->tic.entries);
+
+   nvc0_mm_destroy(screen->mm_GART);
+   nvc0_mm_destroy(screen->mm_VRAM);
+   nvc0_mm_destroy(screen->mm_VRAM_fe0);
+
+   nouveau_grobj_free(&screen->fermi);
+   nouveau_grobj_free(&screen->eng2d);
+   nouveau_grobj_free(&screen->m2mf);
+
+   nouveau_screen_fini(&screen->base);
+
+   FREE(screen);
+}
+
+static int
+nvc0_graph_set_macro(struct nvc0_screen *screen, uint32_t m, unsigned pos,
+                     unsigned size, const uint32_t *data)
+{
+   struct nouveau_channel *chan = screen->base.channel;
+
+   size /= 4;
+
+   BEGIN_RING(chan, RING_3D_(NVC0_GRAPH_MACRO_ID), 2);
+   OUT_RING  (chan, (m - 0x3800) / 8);
+   OUT_RING  (chan, pos);
+   BEGIN_RING_1I(chan, RING_3D_(NVC0_GRAPH_MACRO_UPLOAD_POS), size + 1);
+   OUT_RING  (chan, pos);
+   OUT_RINGp (chan, data, size);
+
+   return pos + size;
+}
+
+static void
+nvc0_screen_fence_reference(struct pipe_screen *pscreen,
+                            struct pipe_fence_handle **ptr,
+                            struct pipe_fence_handle *fence)
+{
+   nvc0_fence_reference((struct nvc0_fence **)ptr, nvc0_fence(fence));
+}
+
+static int
+nvc0_screen_fence_signalled(struct pipe_screen *pscreen,
+                            struct pipe_fence_handle *fence,
+                            unsigned flags)
+{
+   return !(nvc0_fence_signalled(nvc0_fence(fence)));
+}
+
+static int
+nvc0_screen_fence_finish(struct pipe_screen *pscreen,
+                         struct pipe_fence_handle *fence,
+                         unsigned flags)
+{
+   return nvc0_fence_wait((struct nvc0_fence *)fence) != TRUE;
+}
+
+static void
+nvc0_magic_3d_init(struct nouveau_channel *chan)
+{
+   BEGIN_RING(chan, RING_3D_(0x10cc), 1);
+   OUT_RING  (chan, 0xff);
+   BEGIN_RING(chan, RING_3D_(0x10e0), 2);
+   OUT_RING(chan, 0xff);
+   OUT_RING(chan, 0xff);
+   BEGIN_RING(chan, RING_3D_(0x10ec), 2);
+   OUT_RING(chan, 0xff);
+   OUT_RING(chan, 0xff);
+   BEGIN_RING(chan, RING_3D_(0x074c), 1);
+   OUT_RING  (chan, 0x3f);
+
+   BEGIN_RING(chan, RING_3D_(0x10f8), 1);
+   OUT_RING  (chan, 0x0101);
+
+   BEGIN_RING(chan, RING_3D_(0x16a8), 1);
+   OUT_RING  (chan, (3 << 16) | 3);
+   BEGIN_RING(chan, RING_3D_(0x1794), 1);
+   OUT_RING  (chan, (2 << 16) | 2);
+   BEGIN_RING(chan, RING_3D_(0x0de8), 1);
+   OUT_RING  (chan, 1);
+   BEGIN_RING(chan, RING_3D_(0x165c), 1);
+   OUT_RING  (chan, 0);
+
+   BEGIN_RING(chan, RING_3D_(0x1528), 1); /* MP poke */
+   OUT_RING  (chan, 0);
+
+   BEGIN_RING(chan, RING_3D_(0x12ac), 1);
+   OUT_RING  (chan, 0);
+   BEGIN_RING(chan, RING_3D_(0x0218), 1);
+   OUT_RING  (chan, 0x10);
+   BEGIN_RING(chan, RING_3D_(0x10fc), 1);
+   OUT_RING  (chan, 0x10);
+   BEGIN_RING(chan, RING_3D_(0x1290), 1);
+   OUT_RING  (chan, 0x10);
+   BEGIN_RING(chan, RING_3D_(0x12d8), 2);
+   OUT_RING  (chan, 0x10);
+   OUT_RING  (chan, 0x10);
+   BEGIN_RING(chan, RING_3D_(0x06d4), 1);
+   OUT_RING  (chan, 8);
+   BEGIN_RING(chan, RING_3D_(0x1140), 1);
+   OUT_RING  (chan, 0x10);
+   BEGIN_RING(chan, RING_3D_(0x1610), 1);
+   OUT_RING  (chan, 0xe);
+
+   BEGIN_RING(chan, RING_3D_(0x164c), 1);
+   OUT_RING  (chan, 1 << 12);
+   BEGIN_RING(chan, RING_3D_(0x151c), 1);
+   OUT_RING  (chan, 1);
+   BEGIN_RING(chan, RING_3D_(0x020c), 1);
+   OUT_RING  (chan, 1);
+   BEGIN_RING(chan, RING_3D_(0x030c), 1);
+   OUT_RING  (chan, 0);
+   BEGIN_RING(chan, RING_3D_(0x0300), 1);
+   OUT_RING  (chan, 3);
+   BEGIN_RING(chan, RING_3D_(0x1280), 1); /* PGRAPH poke */
+   OUT_RING  (chan, 0);
+   BEGIN_RING(chan, RING_3D_(0x02d0), 1);
+   OUT_RING  (chan, 0x1f40);
+   BEGIN_RING(chan, RING_3D_(0x00fdc), 1);
+   OUT_RING  (chan, 1);
+   BEGIN_RING(chan, RING_3D_(0x19c0), 1);
+   OUT_RING  (chan, 1);
+   BEGIN_RING(chan, RING_3D_(0x075c), 1);
+   OUT_RING  (chan, 3);
+
+   BEGIN_RING(chan, RING_3D_(0x0fac), 1);
+   OUT_RING  (chan, 0);
+   BEGIN_RING(chan, RING_3D_(0x0f90), 1);
+   OUT_RING  (chan, 0);
+}
+
+#define FAIL_SCREEN_INIT(str, err)                    \
+   do {                                               \
+      NOUVEAU_ERR(str, err);                          \
+      nvc0_screen_destroy(pscreen);                   \
+      return NULL;                                    \
+   } while(0)
+
+struct pipe_screen *
+nvc0_screen_create(struct pipe_winsys *ws, struct nouveau_device *dev)
+{
+   struct nvc0_screen *screen;
+   struct nouveau_channel *chan;
+   struct pipe_screen *pscreen;
+   int ret;
+   unsigned i;
+
+   screen = CALLOC_STRUCT(nvc0_screen);
+   if (!screen)
+      return NULL;
+   pscreen = &screen->base.base;
+
+   ret = nouveau_screen_init(&screen->base, dev);
+   if (ret) {
+      nvc0_screen_destroy(pscreen);
+      return NULL;
+   }
+   chan = screen->base.channel;
+
+   pscreen->winsys = ws;
+   pscreen->destroy = nvc0_screen_destroy;
+   pscreen->context_create = nvc0_create;
+   pscreen->is_format_supported = nvc0_screen_is_format_supported;
+   pscreen->get_param = nvc0_screen_get_param;
+   pscreen->get_shader_param = nvc0_screen_get_shader_param;
+   pscreen->get_paramf = nvc0_screen_get_paramf;
+   pscreen->fence_reference = nvc0_screen_fence_reference;
+   pscreen->fence_signalled = nvc0_screen_fence_signalled;
+   pscreen->fence_finish = nvc0_screen_fence_finish;
+
+   nvc0_screen_init_resource_functions(pscreen);
+
+   screen->base.vertex_buffer_flags = NOUVEAU_BO_GART;
+   screen->base.index_buffer_flags = 0;
+
+   ret = nouveau_bo_new(dev, NOUVEAU_BO_GART | NOUVEAU_BO_MAP, 0, 4096,
+                        &screen->fence.bo);
+   if (ret)
+      goto fail;
+   nouveau_bo_map(screen->fence.bo, NOUVEAU_BO_RDWR);
+   screen->fence.map = screen->fence.bo->map;
+   nouveau_bo_unmap(screen->fence.bo);
+
+   for (i = 0; i < NVC0_SCRATCH_NR_BUFFERS; ++i) {
+      ret = nouveau_bo_new(dev, NOUVEAU_BO_GART, 0, NVC0_SCRATCH_SIZE,
+                           &screen->scratch.bo[i]);
+      if (ret)
+         goto fail;
+   }
+
+   ret = nouveau_grobj_alloc(chan, 0xbeef9039, NVC0_M2MF, &screen->m2mf);
+   if (ret)
+      FAIL_SCREEN_INIT("Error allocating PGRAPH context for M2MF: %d\n", ret);
+
+   BIND_RING (chan, screen->m2mf, NVC0_SUBCH_MF);
+   BEGIN_RING(chan, RING_MF(NOTIFY_ADDRESS_HIGH), 3);
+   OUT_RELOCh(chan, screen->fence.bo, 16, NOUVEAU_BO_GART | NOUVEAU_BO_RDWR);
+   OUT_RELOCl(chan, screen->fence.bo, 16, NOUVEAU_BO_GART | NOUVEAU_BO_RDWR);
+   OUT_RING  (chan, 0);
+
+   ret = nouveau_grobj_alloc(chan, 0xbeef902d, NVC0_2D, &screen->eng2d);
+   if (ret)
+      FAIL_SCREEN_INIT("Error allocating PGRAPH context for 2D: %d\n", ret);
+
+   BIND_RING (chan, screen->eng2d, NVC0_SUBCH_2D);
+   BEGIN_RING(chan, RING_2D(OPERATION), 1);
+   OUT_RING  (chan, NVC0_2D_OPERATION_SRCCOPY);
+   BEGIN_RING(chan, RING_2D(CLIP_ENABLE), 1);
+   OUT_RING  (chan, 0);
+   BEGIN_RING(chan, RING_2D(COLOR_KEY_ENABLE), 1);
+   OUT_RING  (chan, 0);
+   BEGIN_RING(chan, RING_2D_(0x0884), 1);
+   OUT_RING  (chan, 0x3f);
+   BEGIN_RING(chan, RING_2D_(0x0888), 1);
+   OUT_RING  (chan, 1);
+
+   ret = nouveau_grobj_alloc(chan, 0xbeef9097, NVC0_3D, &screen->fermi);
+   if (ret)
+      FAIL_SCREEN_INIT("Error allocating PGRAPH context for 3D: %d\n", ret);
+
+   BIND_RING (chan, screen->fermi, NVC0_SUBCH_3D);
+   BEGIN_RING(chan, RING_3D(NOTIFY_ADDRESS_HIGH), 3);
+   OUT_RELOCh(chan, screen->fence.bo, 32, NOUVEAU_BO_GART | NOUVEAU_BO_RDWR);
+   OUT_RELOCl(chan, screen->fence.bo, 32, NOUVEAU_BO_GART | NOUVEAU_BO_RDWR);
+   OUT_RING  (chan, 0);
+
+   BEGIN_RING(chan, RING_3D(COND_MODE), 1);
+   OUT_RING  (chan, NVC0_3D_COND_MODE_ALWAYS);
+
+   BEGIN_RING(chan, RING_3D(RT_CONTROL), 1);
+   OUT_RING  (chan, 1);
+
+   BEGIN_RING(chan, RING_3D(MULTISAMPLE_ZETA_ENABLE), 1);
+   OUT_RING  (chan, 0);
+   BEGIN_RING(chan, RING_3D(MULTISAMPLE_COLOR_ENABLE), 1);
+   OUT_RING  (chan, 0);
+   BEGIN_RING(chan, RING_3D(MULTISAMPLE_MODE), 1);
+   OUT_RING  (chan, NVC0_3D_MULTISAMPLE_MODE_1X);
+   BEGIN_RING(chan, RING_3D(MULTISAMPLE_CTRL), 1);
+   OUT_RING  (chan, 0);
+
+   nvc0_magic_3d_init(chan);
+
+   ret = nouveau_bo_new(dev, NOUVEAU_BO_VRAM, 1 << 17, 1 << 20, &screen->text);
+   if (ret)
+      goto fail;
+
+   nouveau_resource_init(&screen->text_heap, 0, 1 << 20);
+
+   ret = nouveau_bo_new(dev, NOUVEAU_BO_VRAM, 1 << 12, 6 << 16,
+                        &screen->uniforms);
+   if (ret)
+      goto fail;
+
+   /* auxiliary constants (6 user clip planes, base instance id) */
+   BEGIN_RING(chan, RING_3D(CB_SIZE), 3);
+   OUT_RING  (chan, 256);
+   OUT_RELOCh(chan, screen->uniforms, 5 << 16, NOUVEAU_BO_VRAM | NOUVEAU_BO_RD);
+   OUT_RELOCl(chan, screen->uniforms, 5 << 16, NOUVEAU_BO_VRAM | NOUVEAU_BO_RD);
+   for (i = 0; i < 5; ++i) {
+      BEGIN_RING(chan, RING_3D(CB_BIND(i)), 1);
+      OUT_RING  (chan, (15 << 4) | 1);
+   }
+
+   screen->tls_size = 4 * 4 * 32 * 128 * 4;
+   ret = nouveau_bo_new(dev, NOUVEAU_BO_VRAM, 1 << 17,
+                        screen->tls_size, &screen->tls);
+   if (ret)
+      goto fail;
+
+   BEGIN_RING(chan, RING_3D(CODE_ADDRESS_HIGH), 2);
+   OUT_RELOCh(chan, screen->text, 0, NOUVEAU_BO_VRAM | NOUVEAU_BO_RD);
+   OUT_RELOCl(chan, screen->text, 0, NOUVEAU_BO_VRAM | NOUVEAU_BO_RD);
+   BEGIN_RING(chan, RING_3D(LOCAL_ADDRESS_HIGH), 4);
+   OUT_RELOCh(chan, screen->tls, 0, NOUVEAU_BO_VRAM | NOUVEAU_BO_RDWR);
+   OUT_RELOCl(chan, screen->tls, 0, NOUVEAU_BO_VRAM | NOUVEAU_BO_RDWR);
+   OUT_RING  (chan, screen->tls_size >> 32);
+   OUT_RING  (chan, screen->tls_size);
+   BEGIN_RING(chan, RING_3D(LOCAL_BASE), 1);
+   OUT_RING  (chan, 0);
+
+   for (i = 0; i < 5; ++i) {
+      BEGIN_RING(chan, RING_3D(TEX_LIMITS(i)), 1);
+      OUT_RING  (chan, 0x54);
+   }
+   BEGIN_RING(chan, RING_3D(LINKED_TSC), 1);
+   OUT_RING  (chan, 0);
+
+   ret = nouveau_bo_new(dev, NOUVEAU_BO_VRAM, 1 << 17, 1 << 20,
+                        &screen->mp_stack_bo);
+   if (ret)
+      goto fail;
+
+   BEGIN_RING(chan, RING_3D_(0x17bc), 3);
+   OUT_RELOCh(chan, screen->mp_stack_bo, 0, NOUVEAU_BO_GART | NOUVEAU_BO_RDWR);
+   OUT_RELOCl(chan, screen->mp_stack_bo, 0, NOUVEAU_BO_GART | NOUVEAU_BO_RDWR);
+   OUT_RING  (chan, 1);
+
+   ret = nouveau_bo_new(dev, NOUVEAU_BO_VRAM, 1 << 17, 1 << 17, &screen->txc);
+   if (ret)
+      goto fail;
+
+   BEGIN_RING(chan, RING_3D(TIC_ADDRESS_HIGH), 3);
+   OUT_RELOCh(chan, screen->txc, 0, NOUVEAU_BO_VRAM | NOUVEAU_BO_RD);
+   OUT_RELOCl(chan, screen->txc, 0, NOUVEAU_BO_VRAM | NOUVEAU_BO_RD);
+   OUT_RING  (chan, NVC0_TIC_MAX_ENTRIES - 1);
+
+   BEGIN_RING(chan, RING_3D(TSC_ADDRESS_HIGH), 3);
+   OUT_RELOCh(chan, screen->txc, 65536, NOUVEAU_BO_VRAM | NOUVEAU_BO_RD);
+   OUT_RELOCl(chan, screen->txc, 65536, NOUVEAU_BO_VRAM | NOUVEAU_BO_RD);
+   OUT_RING  (chan, NVC0_TSC_MAX_ENTRIES - 1);
+
+   BEGIN_RING(chan, RING_3D(Y_ORIGIN_BOTTOM), 1);
+   OUT_RING  (chan, 0);
+   BEGIN_RING(chan, RING_3D(WINDOW_OFFSET_X), 2);
+   OUT_RING  (chan, 0);
+   OUT_RING  (chan, 0);
+   BEGIN_RING(chan, RING_3D_(0x1590), 1); /* deactivate ZCULL */
+   OUT_RING  (chan, 0x3f);
+
+   BEGIN_RING(chan, RING_3D(VIEWPORT_CLIP_RECTS_EN), 1);
+   OUT_RING  (chan, 0);
+   BEGIN_RING(chan, RING_3D(CLIPID_ENABLE), 1);
+   OUT_RING  (chan, 0);
+
+   BEGIN_RING(chan, RING_3D(VIEWPORT_TRANSFORM_EN), 1);
+   OUT_RING  (chan, 1);
+   BEGIN_RING(chan, RING_3D(DEPTH_RANGE_NEAR(0)), 2);
+   OUT_RINGf (chan, 0.0f);
+   OUT_RINGf (chan, 1.0f);
+
+   /* We use scissors instead of exact view volume clipping,
+    * so they're always enabled.
+    */
+   BEGIN_RING(chan, RING_3D(SCISSOR_ENABLE(0)), 3);
+   OUT_RING  (chan, 1);
+   OUT_RING  (chan, 8192 << 16);
+   OUT_RING  (chan, 8192 << 16);
+
+   BEGIN_RING(chan, RING_3D_(0x0fac), 1);
+   OUT_RING  (chan, 0);
+   BEGIN_RING(chan, RING_3D_(0x3484), 1);
+   OUT_RING  (chan, 0);
+   BEGIN_RING(chan, RING_3D_(0x0dbc), 1);
+   OUT_RING  (chan, 0x00010000);
+   BEGIN_RING(chan, RING_3D_(0x0dd8), 1);
+   OUT_RING  (chan, 0xff800006);
+   BEGIN_RING(chan, RING_3D_(0x3488), 1);
+   OUT_RING  (chan, 0);
+
+#define MK_MACRO(m, n) i = nvc0_graph_set_macro(screen, m, i, sizeof(n), n);
+
+   i = 0;
+   MK_MACRO(NVC0_3D_BLEND_ENABLES, nvc0_9097_blend_enables);
+   MK_MACRO(NVC0_3D_VERTEX_ARRAY_SELECT, nvc0_9097_vertex_array_select);
+   MK_MACRO(NVC0_3D_TEP_SELECT, nvc0_9097_tep_select);
+   MK_MACRO(NVC0_3D_GP_SELECT, nvc0_9097_gp_select);
+   MK_MACRO(NVC0_3D_POLYGON_MODE_FRONT, nvc0_9097_poly_mode_front);
+   MK_MACRO(NVC0_3D_POLYGON_MODE_BACK, nvc0_9097_poly_mode_back);
+   MK_MACRO(NVC0_3D_COLOR_MASK_BROADCAST, nvc0_9097_color_mask_brdc);
+
+   BEGIN_RING(chan, RING_3D(RASTERIZE_ENABLE), 1);
+   OUT_RING  (chan, 1);
+   BEGIN_RING(chan, RING_3D(GP_SELECT), 1);
+   OUT_RING  (chan, 0x40);
+   BEGIN_RING(chan, RING_3D(GP_BUILTIN_RESULT_EN), 1);
+   OUT_RING  (chan, 0);
+   BEGIN_RING(chan, RING_3D(TEP_SELECT), 1);
+   OUT_RING  (chan, 0x30);
+   BEGIN_RING(chan, RING_3D(PATCH_VERTICES), 1);
+   OUT_RING  (chan, 3);
+   BEGIN_RING(chan, RING_3D(SP_SELECT(2)), 1);
+   OUT_RING  (chan, 0x20);
+   BEGIN_RING(chan, RING_3D(SP_SELECT(0)), 1);
+   OUT_RING  (chan, 0x00);
+
+   BEGIN_RING(chan, RING_3D(POINT_COORD_REPLACE), 1);
+   OUT_RING  (chan, 0);
+
+   BEGIN_RING(chan, RING_3D(FRAG_COLOR_CLAMP_EN), 1);
+   OUT_RING  (chan, 0x11111111);
+   BEGIN_RING(chan, RING_3D(EDGEFLAG_ENABLE), 1);
+   OUT_RING  (chan, 1);
+
+   BEGIN_RING(chan, RING_3D(VERTEX_RUNOUT_ADDRESS_HIGH), 2);
+   OUT_RING  (chan, 0xab);
+   OUT_RING  (chan, 0x00000000);
+
+   FIRE_RING (chan);
+
+   screen->tic.entries = CALLOC(4096, sizeof(void *));
+   screen->tsc.entries = screen->tic.entries + 2048;
+
+   screen->mm_GART = nvc0_mm_create(dev, NOUVEAU_BO_GART | NOUVEAU_BO_MAP,
+                                    0x000);
+   screen->mm_VRAM = nvc0_mm_create(dev, NOUVEAU_BO_VRAM, 0x000);
+   screen->mm_VRAM_fe0 = nvc0_mm_create(dev, NOUVEAU_BO_VRAM, 0xfe0);
+
+   nvc0_screen_fence_new(screen, &screen->fence.current, FALSE);
+
+   return pscreen;
+
+fail:
+   nvc0_screen_destroy(pscreen);
+   return NULL;
+}
+
+void
+nvc0_screen_make_buffers_resident(struct nvc0_screen *screen)
+{
+   struct nouveau_channel *chan = screen->base.channel;
+
+   const unsigned flags = NOUVEAU_BO_VRAM | NOUVEAU_BO_RD;
+
+   nouveau_bo_validate(chan, screen->text, flags);
+   nouveau_bo_validate(chan, screen->uniforms, flags);
+   nouveau_bo_validate(chan, screen->txc, flags);
+   nouveau_bo_validate(chan, screen->tls, flags);
+   nouveau_bo_validate(chan, screen->mp_stack_bo, flags);
+}
+
+int
+nvc0_screen_tic_alloc(struct nvc0_screen *screen, void *entry)
+{
+   int i = screen->tic.next;
+
+   while (screen->tic.lock[i / 32] & (1 << (i % 32)))
+      i = (i + 1) & (NVC0_TIC_MAX_ENTRIES - 1);
+
+   screen->tic.next = (i + 1) & (NVC0_TIC_MAX_ENTRIES - 1);
+
+   if (screen->tic.entries[i])
+      nvc0_tic_entry(screen->tic.entries[i])->id = -1;
+
+   screen->tic.entries[i] = entry;
+   return i;
+}
+
+int
+nvc0_screen_tsc_alloc(struct nvc0_screen *screen, void *entry)
+{
+   int i = screen->tsc.next;
+
+   while (screen->tsc.lock[i / 32] & (1 << (i % 32)))
+      i = (i + 1) & (NVC0_TSC_MAX_ENTRIES - 1);
+
+   screen->tsc.next = (i + 1) & (NVC0_TSC_MAX_ENTRIES - 1);
+
+   if (screen->tsc.entries[i])
+      nvc0_tsc_entry(screen->tsc.entries[i])->id = -1;
+
+   screen->tsc.entries[i] = entry;
+   return i;
+}
diff --git a/src/gallium/drivers/nvc0/nvc0_screen.h b/src/gallium/drivers/nvc0/nvc0_screen.h
new file mode 100644
index 00000000000..5c6482be96b
--- /dev/null
+++ b/src/gallium/drivers/nvc0/nvc0_screen.h
@@ -0,0 +1,185 @@
+#ifndef __NVC0_SCREEN_H__
+#define __NVC0_SCREEN_H__
+
+#define NOUVEAU_NVC0
+#include "nouveau/nouveau_screen.h"
+#undef NOUVEAU_NVC0
+#include "nvc0_winsys.h"
+#include "nvc0_stateobj.h"
+
+#define NVC0_TIC_MAX_ENTRIES 2048
+#define NVC0_TSC_MAX_ENTRIES 2048
+
+struct nvc0_mman;
+struct nvc0_context;
+struct nvc0_fence;
+
+#define NVC0_SCRATCH_SIZE (2 << 20)
+#define NVC0_SCRATCH_NR_BUFFERS 2
+
+struct nvc0_screen {
+   struct nouveau_screen base;
+   struct nouveau_winsys *nvws;
+
+   struct nvc0_context *cur_ctx;
+
+   struct nouveau_bo *text;
+   struct nouveau_bo *uniforms;
+   struct nouveau_bo *tls;
+   struct nouveau_bo *txc; /* TIC (offset 0) and TSC (65536) */
+   struct nouveau_bo *mp_stack_bo;
+
+   uint64_t tls_size;
+
+   struct nouveau_resource *text_heap;
+
+   struct {
+      struct nouveau_bo *bo[NVC0_SCRATCH_NR_BUFFERS];
+      uint8_t *buf;
+      int index;
+      uint32_t offset;
+   } scratch;
+
+   struct {
+      void **entries;
+      int next;
+      uint32_t lock[NVC0_TIC_MAX_ENTRIES / 32];
+   } tic;
+   
+   struct {
+      void **entries;
+      int next;
+      uint32_t lock[NVC0_TSC_MAX_ENTRIES / 32];
+   } tsc;
+
+   struct {
+      uint32_t *map;
+      struct nvc0_fence *head;
+      struct nvc0_fence *tail;
+      struct nvc0_fence *current;
+      uint32_t sequence;
+      uint32_t sequence_ack;
+      struct nouveau_bo *bo;
+   } fence;
+
+   struct nvc0_mman *mm_GART;
+   struct nvc0_mman *mm_VRAM;
+   struct nvc0_mman *mm_VRAM_fe0;
+
+   struct nouveau_grobj *fermi;
+   struct nouveau_grobj *eng2d;
+   struct nouveau_grobj *m2mf;
+};
+
+static INLINE struct nvc0_screen *
+nvc0_screen(struct pipe_screen *screen)
+{
+   return (struct nvc0_screen *)screen;
+}
+
+/* Since a resource can be migrated, we need to decouple allocations from
+ * them. This struct is linked with fences for delayed freeing of allocs.
+ */
+struct nvc0_mm_allocation {
+   struct nvc0_mm_allocation *next;
+   void *priv;
+   uint32_t offset;
+};
+
+extern struct nvc0_mman *
+nvc0_mm_create(struct nouveau_device *, uint32_t domain, uint32_t storage_type);
+
+extern void
+nvc0_mm_destroy(struct nvc0_mman *);
+
+extern struct nvc0_mm_allocation *
+nvc0_mm_allocate(struct nvc0_mman *,
+                 uint32_t size, struct nouveau_bo **, uint32_t *offset);
+extern void
+nvc0_mm_free(struct nvc0_mm_allocation *);
+
+void nvc0_screen_make_buffers_resident(struct nvc0_screen *);
+
+int nvc0_screen_tic_alloc(struct nvc0_screen *, void *);
+int nvc0_screen_tsc_alloc(struct nvc0_screen *, void *);
+
+static INLINE void
+nvc0_resource_fence(struct nvc0_resource *res, uint32_t flags)
+{
+   struct nvc0_screen *screen = nvc0_screen(res->base.screen);
+
+   if (res->mm) {
+      nvc0_fence_reference(&res->fence, screen->fence.current);
+
+      if (flags & NOUVEAU_BO_WR)
+         nvc0_fence_reference(&res->fence_wr, screen->fence.current);
+   }
+}
+
+static INLINE void
+nvc0_resource_validate(struct nvc0_resource *res, uint32_t flags)
+{
+   struct nvc0_screen *screen = nvc0_screen(res->base.screen);
+
+   nouveau_bo_validate(screen->base.channel, res->bo, flags);
+
+   nvc0_resource_fence(res, flags);
+}
+
+
+boolean
+nvc0_screen_fence_new(struct nvc0_screen *, struct nvc0_fence **, boolean emit);
+
+void
+nvc0_screen_fence_next(struct nvc0_screen *);
+
+static INLINE boolean
+nvc0_screen_fence_emit(struct nvc0_screen *screen)
+{
+   nvc0_fence_emit(screen->fence.current);
+
+   return nvc0_screen_fence_new(screen, &screen->fence.current, FALSE);
+}
+
+struct nvc0_format {
+   uint32_t rt;
+   uint32_t tic;
+   uint32_t vtx;
+   uint32_t usage;
+};
+
+extern const struct nvc0_format nvc0_format_table[];
+
+static INLINE void
+nvc0_screen_tic_unlock(struct nvc0_screen *screen, struct nvc0_tic_entry *tic)
+{
+   if (tic->id >= 0)
+      screen->tic.lock[tic->id / 32] &= ~(1 << (tic->id % 32));
+}
+
+static INLINE void
+nvc0_screen_tsc_unlock(struct nvc0_screen *screen, struct nvc0_tsc_entry *tsc)
+{
+   if (tsc->id >= 0)
+      screen->tsc.lock[tsc->id / 32] &= ~(1 << (tsc->id % 32));
+}
+
+static INLINE void
+nvc0_screen_tic_free(struct nvc0_screen *screen, struct nvc0_tic_entry *tic)
+{
+   if (tic->id >= 0) {
+      screen->tic.entries[tic->id] = NULL;
+      screen->tic.lock[tic->id / 32] &= ~(1 << (tic->id % 32));
+   }
+}
+
+static INLINE void
+nvc0_screen_tsc_free(struct nvc0_screen *screen, struct nvc0_tsc_entry *tsc)
+{
+   if (tsc->id >= 0) {
+      screen->tsc.entries[tsc->id] = NULL;
+      screen->tsc.lock[tsc->id / 32] &= ~(1 << (tsc->id % 32));
+   }
+}
+
+#endif
diff --git a/src/gallium/drivers/nvc0/nvc0_shader_state.c b/src/gallium/drivers/nvc0/nvc0_shader_state.c
new file mode 100644
index 00000000000..a6595c56106
--- /dev/null
+++ b/src/gallium/drivers/nvc0/nvc0_shader_state.c
@@ -0,0 +1,180 @@
+/*
+ * Copyright 2010 Christoph Bumiller
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF
+ * OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "pipe/p_context.h"
+#include "pipe/p_defines.h"
+#include "pipe/p_state.h"
+#include "util/u_inlines.h"
+
+#include "nvc0_context.h"
+
+static boolean
+nvc0_program_validate(struct nvc0_context *nvc0, struct nvc0_program *prog)
+{
+   int ret;
+   unsigned size;
+
+   if (prog->translated)
+      return TRUE;
+
+   prog->translated = nvc0_program_translate(prog);
+   if (!prog->translated)
+      return FALSE;
+
+   size = align(prog->code_size + NVC0_SHADER_HEADER_SIZE, 0x100);
+
+   ret = nouveau_resource_alloc(nvc0->screen->text_heap, size, prog,
+                                &prog->res);
+   if (ret)
+      return FALSE;
+
+   prog->code_base = prog->res->start;
+
+   nvc0_m2mf_push_linear(nvc0, nvc0->screen->text, NOUVEAU_BO_VRAM,
+                         prog->code_base, NVC0_SHADER_HEADER_SIZE, prog->hdr);
+   nvc0_m2mf_push_linear(nvc0, nvc0->screen->text, NOUVEAU_BO_VRAM,
+                         prog->code_base + NVC0_SHADER_HEADER_SIZE,
+                         prog->code_size, prog->code);
+
+   BEGIN_RING(nvc0->screen->base.channel, RING_3D_(0x021c), 1);
+   OUT_RING  (nvc0->screen->base.channel, 0x1111);
+
+   return TRUE;
+}
+
+void
+nvc0_vertprog_validate(struct nvc0_context *nvc0)
+{
+   struct nouveau_channel *chan = nvc0->screen->base.channel;
+   struct nvc0_program *vp = nvc0->vertprog;
+
+   if (nvc0->clip.nr > vp->vp.num_ucps) {
+      assert(nvc0->clip.nr <= 6);
+      vp->vp.num_ucps = 6;
+
+      if (vp->translated)
+         nvc0_program_destroy(nvc0, vp);
+   }
+
+   if (!nvc0_program_validate(nvc0, vp))
+         return;
+
+   BEGIN_RING(chan, RING_3D(SP_SELECT(1)), 2);
+   OUT_RING  (chan, 0x11);
+   OUT_RING  (chan, vp->code_base);
+   BEGIN_RING(chan, RING_3D(SP_GPR_ALLOC(1)), 1);
+   OUT_RING  (chan, vp->max_gpr);
+
+   // BEGIN_RING(chan, RING_3D_(0x163c), 1);
+   // OUT_RING  (chan, 0);
+   BEGIN_RING(chan, RING_3D(VERT_COLOR_CLAMP_EN), 1);
+   OUT_RING  (chan, 1);
+}
+
+void
+nvc0_fragprog_validate(struct nvc0_context *nvc0)
+{
+   struct nouveau_channel *chan = nvc0->screen->base.channel;
+   struct nvc0_program *fp = nvc0->fragprog;
+
+   if (!nvc0_program_validate(nvc0, fp))
+         return;
+
+   BEGIN_RING(chan, RING_3D(EARLY_FRAGMENT_TESTS), 1);
+   OUT_RING  (chan, 0);
+   BEGIN_RING(chan, RING_3D(SP_SELECT(5)), 2);
+   OUT_RING  (chan, 0x51);
+   OUT_RING  (chan, fp->code_base);
+   BEGIN_RING(chan, RING_3D(SP_GPR_ALLOC(5)), 1);
+   OUT_RING  (chan, fp->max_gpr);
+
+   BEGIN_RING(chan, RING_3D_(0x0360), 2);
+   OUT_RING  (chan, 0x20164010);
+   OUT_RING  (chan, 0x20);
+   BEGIN_RING(chan, RING_3D_(0x196c), 1);
+   OUT_RING  (chan, fp->flags[0]);
+}
+
+void
+nvc0_tctlprog_validate(struct nvc0_context *nvc0)
+{
+   struct nouveau_channel *chan = nvc0->screen->base.channel;
+   struct nvc0_program *tp = nvc0->tctlprog;
+
+   if (!tp) {
+      BEGIN_RING(chan, RING_3D(SP_SELECT(2)), 1);
+      OUT_RING  (chan, 0x20);
+      return;
+   }
+   if (!nvc0_program_validate(nvc0, tp))
+         return;
+
+   BEGIN_RING(chan, RING_3D(SP_SELECT(2)), 2);
+   OUT_RING  (chan, 0x21);
+   OUT_RING  (chan, tp->code_base);
+   BEGIN_RING(chan, RING_3D(SP_GPR_ALLOC(2)), 1);
+   OUT_RING  (chan, tp->max_gpr);   
+}
+
+void
+nvc0_tevlprog_validate(struct nvc0_context *nvc0)
+{
+   struct nouveau_channel *chan = nvc0->screen->base.channel;
+   struct nvc0_program *tp = nvc0->tevlprog;
+
+   if (!tp) {
+      BEGIN_RING(chan, RING_3D(TEP_SELECT), 1);
+      OUT_RING  (chan, 0x30);
+      return;
+   }
+   if (!nvc0_program_validate(nvc0, tp))
+         return;
+
+   BEGIN_RING(chan, RING_3D(TEP_SELECT), 1);
+   OUT_RING  (chan, 0x31);
+   BEGIN_RING(chan, RING_3D(SP_START_ID(3)), 1);
+   OUT_RING  (chan, tp->code_base);
+   BEGIN_RING(chan, RING_3D(SP_GPR_ALLOC(3)), 1);
+   OUT_RING  (chan, tp->max_gpr);   
+}
+
+void
+nvc0_gmtyprog_validate(struct nvc0_context *nvc0)
+{
+   struct nouveau_channel *chan = nvc0->screen->base.channel;
+   struct nvc0_program *gp = nvc0->gmtyprog;
+
+   if (!gp) {
+      BEGIN_RING(chan, RING_3D(GP_SELECT), 1);
+      OUT_RING  (chan, 0x40);
+      return;
+   }
+   if (!nvc0_program_validate(nvc0, gp))
+         return;
+
+   BEGIN_RING(chan, RING_3D(GP_SELECT), 1);
+   OUT_RING  (chan, 0x41);
+   BEGIN_RING(chan, RING_3D(SP_START_ID(4)), 1);
+   OUT_RING  (chan, gp->code_base);
+   BEGIN_RING(chan, RING_3D(SP_GPR_ALLOC(4)), 1);
+   OUT_RING  (chan, gp->max_gpr);   
+}
diff --git a/src/gallium/drivers/nvc0/nvc0_state.c b/src/gallium/drivers/nvc0/nvc0_state.c
new file mode 100644
index 00000000000..5a9b1c28509
--- /dev/null
+++ b/src/gallium/drivers/nvc0/nvc0_state.c
@@ -0,0 +1,864 @@
+/*
+ * Copyright 2010 Christoph Bumiller
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF
+ * OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "pipe/p_defines.h"
+#include "util/u_inlines.h"
+
+#include "tgsi/tgsi_parse.h"
+
+#include "nvc0_stateobj.h"
+#include "nvc0_context.h"
+
+#include "nvc0_3d.xml.h"
+#include "nv50_texture.xml.h"
+
+#include "nouveau/nouveau_gldefs.h"
+
+static INLINE uint32_t
+nvc0_colormask(unsigned mask)
+{
+    uint32_t ret = 0;
+
+    if (mask & PIPE_MASK_R)
+        ret |= 0x0001;
+    if (mask & PIPE_MASK_G)
+        ret |= 0x0010;
+    if (mask & PIPE_MASK_B)
+        ret |= 0x0100;
+    if (mask & PIPE_MASK_A)
+        ret |= 0x1000;
+
+    return ret;
+}
+
+static INLINE uint32_t
+nvc0_blend_fac(unsigned factor)
+{
+    static const uint16_t bf[] = {
+        NV50_3D_BLEND_FACTOR_ZERO, /* 0x00 */
+        NV50_3D_BLEND_FACTOR_ONE,
+        NV50_3D_BLEND_FACTOR_SRC_COLOR,
+        NV50_3D_BLEND_FACTOR_SRC_ALPHA,
+        NV50_3D_BLEND_FACTOR_DST_ALPHA,
+        NV50_3D_BLEND_FACTOR_DST_COLOR,
+        NV50_3D_BLEND_FACTOR_SRC_ALPHA_SATURATE,
+        NV50_3D_BLEND_FACTOR_CONSTANT_COLOR,
+        NV50_3D_BLEND_FACTOR_CONSTANT_ALPHA,
+        NV50_3D_BLEND_FACTOR_SRC1_COLOR,
+        NV50_3D_BLEND_FACTOR_SRC1_ALPHA,
+        NV50_3D_BLEND_FACTOR_ZERO, /* 0x0b */
+        NV50_3D_BLEND_FACTOR_ZERO, /* 0x0c */
+        NV50_3D_BLEND_FACTOR_ZERO, /* 0x0d */
+        NV50_3D_BLEND_FACTOR_ZERO, /* 0x0e */
+        NV50_3D_BLEND_FACTOR_ZERO, /* 0x0f */
+        NV50_3D_BLEND_FACTOR_ZERO, /* 0x10 */
+        NV50_3D_BLEND_FACTOR_ZERO, /* 0x11 */
+        NV50_3D_BLEND_FACTOR_ONE_MINUS_SRC_COLOR,
+        NV50_3D_BLEND_FACTOR_ONE_MINUS_SRC_ALPHA,
+        NV50_3D_BLEND_FACTOR_ONE_MINUS_DST_ALPHA,
+        NV50_3D_BLEND_FACTOR_ONE_MINUS_DST_COLOR,
+        NV50_3D_BLEND_FACTOR_ONE_MINUS_CONSTANT_COLOR,
+        NV50_3D_BLEND_FACTOR_ONE_MINUS_CONSTANT_ALPHA,
+        NV50_3D_BLEND_FACTOR_ONE_MINUS_SRC1_COLOR,
+        NV50_3D_BLEND_FACTOR_ONE_MINUS_SRC1_ALPHA
+    };
+
+    assert(factor < (sizeof(bf) / sizeof(bf[0])));
+    return bf[factor];
+}
+
+static void *
+nvc0_blend_state_create(struct pipe_context *pipe,
+                        const struct pipe_blend_state *cso)
+{
+    struct nvc0_blend_stateobj *so = CALLOC_STRUCT(nvc0_blend_stateobj);
+    int i;
+
+    so->pipe = *cso;
+
+    SB_IMMED_3D(so, BLEND_INDEPENDENT, cso->independent_blend_enable);
+
+    if (!cso->independent_blend_enable) {
+        SB_BEGIN_3D(so, BLEND_ENABLES, 1);
+        SB_DATA    (so, cso->rt[0].blend_enable ? 0xff : 0);
+
+        if (cso->rt[0].blend_enable) {
+            SB_BEGIN_3D(so, BLEND_EQUATION_RGB, 5);
+            SB_DATA    (so, nvgl_blend_eqn(cso->rt[0].rgb_func));
+            SB_DATA    (so, nvc0_blend_fac(cso->rt[0].rgb_src_factor));
+            SB_DATA    (so, nvc0_blend_fac(cso->rt[0].rgb_dst_factor));
+            SB_DATA    (so, nvgl_blend_eqn(cso->rt[0].alpha_func));
+            SB_DATA    (so, nvc0_blend_fac(cso->rt[0].alpha_src_factor));
+            SB_BEGIN_3D(so, BLEND_FUNC_DST_ALPHA, 1);
+            SB_DATA    (so, nvc0_blend_fac(cso->rt[0].alpha_dst_factor));
+        }
+
+        SB_BEGIN_3D(so, COLOR_MASK_BROADCAST, 1);
+        SB_DATA    (so, nvc0_colormask(cso->rt[0].colormask));
+    } else {
+        uint8_t en = 0;
+
+        for (i = 0; i < 8; ++i) {
+            if (!cso->rt[i].blend_enable)
+                continue;
+            en |= 1 << i;
+
+            SB_BEGIN_3D(so, IBLEND_EQUATION_RGB(i), 6);
+            SB_DATA    (so, nvgl_blend_eqn(cso->rt[i].rgb_func));
+            SB_DATA    (so, nvc0_blend_fac(cso->rt[i].rgb_src_factor));
+            SB_DATA    (so, nvc0_blend_fac(cso->rt[i].rgb_dst_factor));
+            SB_DATA    (so, nvgl_blend_eqn(cso->rt[i].alpha_func));
+            SB_DATA    (so, nvc0_blend_fac(cso->rt[i].alpha_src_factor));
+            SB_DATA    (so, nvc0_blend_fac(cso->rt[i].alpha_dst_factor));
+        }
+        SB_BEGIN_3D(so, BLEND_ENABLES, 1);
+        SB_DATA    (so, en);
+
+        SB_BEGIN_3D(so, COLOR_MASK(0), 8);
+        for (i = 0; i < 8; ++i)
+            SB_DATA(so, nvc0_colormask(cso->rt[i].colormask));
+    }
+
+    if (cso->logicop_enable) {
+       SB_BEGIN_3D(so, LOGIC_OP_ENABLE, 2);
+       SB_DATA    (so, 1);
+       SB_DATA    (so, nvgl_logicop_func(cso->logicop_func));
+    } else {
+       SB_IMMED_3D(so, LOGIC_OP_ENABLE, 0);
+    }
+
+    assert(so->size < (sizeof(so->state) / sizeof(so->state[0])));
+    return so;
+}
+
+static void
+nvc0_blend_state_bind(struct pipe_context *pipe, void *hwcso)
+{
+    struct nvc0_context *nvc0 = nvc0_context(pipe);
+
+    nvc0->blend = hwcso;
+    nvc0->dirty |= NVC0_NEW_BLEND;
+}
+
+static void
+nvc0_blend_state_delete(struct pipe_context *pipe, void *hwcso)
+{
+    FREE(hwcso);
+}
+
+static void *
+nvc0_rasterizer_state_create(struct pipe_context *pipe,
+                             const struct pipe_rasterizer_state *cso)
+{
+    struct nvc0_rasterizer_stateobj *so;
+
+    so = CALLOC_STRUCT(nvc0_rasterizer_stateobj);
+    if (!so)
+        return NULL;
+    so->pipe = *cso;
+
+#ifndef NVC0_SCISSORS_CLIPPING
+    SB_IMMED_3D(so, SCISSOR_ENABLE(0), cso->scissor);
+#endif
+    
+    SB_BEGIN_3D(so, SHADE_MODEL, 1);
+    SB_DATA    (so, cso->flatshade ? NVC0_3D_SHADE_MODEL_FLAT :
+                                     NVC0_3D_SHADE_MODEL_SMOOTH);
+    SB_IMMED_3D(so, PROVOKING_VERTEX_LAST, !cso->flatshade_first);
+    SB_IMMED_3D(so, VERTEX_TWO_SIDE_ENABLE, cso->light_twoside);
+
+    SB_BEGIN_3D(so, LINE_WIDTH, 1);
+    SB_DATA    (so, fui(cso->line_width));
+    SB_IMMED_3D(so, LINE_SMOOTH_ENABLE, cso->line_smooth);
+
+    SB_BEGIN_3D(so, LINE_STIPPLE_ENABLE, 1);
+    if (cso->line_stipple_enable) {
+        SB_DATA    (so, 1);
+        SB_BEGIN_3D(so, LINE_STIPPLE_PATTERN, 1);
+        SB_DATA    (so, (cso->line_stipple_pattern << 8) |
+                         cso->line_stipple_factor);
+                    
+    } else {
+        SB_DATA    (so, 0);
+    }
+
+    SB_IMMED_3D(so, VP_POINT_SIZE_EN, cso->point_size_per_vertex);
+    if (!cso->point_size_per_vertex) {
+       SB_BEGIN_3D(so, POINT_SIZE, 1);
+       SB_DATA    (so, fui(cso->point_size));
+    }
+    SB_IMMED_3D(so, POINT_SPRITE_ENABLE, cso->point_quad_rasterization);
+
+    SB_BEGIN_3D(so, POLYGON_MODE_FRONT, 1);
+    SB_DATA    (so, nvgl_polygon_mode(cso->fill_front));
+    SB_BEGIN_3D(so, POLYGON_MODE_BACK, 1);
+    SB_DATA    (so, nvgl_polygon_mode(cso->fill_back));
+    SB_IMMED_3D(so, POLYGON_SMOOTH_ENABLE, cso->poly_smooth);
+
+    SB_BEGIN_3D(so, CULL_FACE_ENABLE, 3);
+    SB_DATA    (so, cso->cull_face != PIPE_FACE_NONE);
+    SB_DATA    (so, cso->front_ccw ? NVC0_3D_FRONT_FACE_CCW :
+                                     NVC0_3D_FRONT_FACE_CW);
+    switch (cso->cull_face) {
+    case PIPE_FACE_FRONT_AND_BACK:
+       SB_DATA(so, NVC0_3D_CULL_FACE_FRONT_AND_BACK);
+       break;
+    case PIPE_FACE_FRONT:
+       SB_DATA(so, NVC0_3D_CULL_FACE_FRONT);
+       break;
+    case PIPE_FACE_BACK:
+    default:
+       SB_DATA(so, NVC0_3D_CULL_FACE_BACK);
+       break;
+    }
+
+    SB_IMMED_3D(so, POLYGON_STIPPLE_ENABLE, cso->poly_stipple_enable);
+    SB_BEGIN_3D(so, POLYGON_OFFSET_POINT_ENABLE, 3);
+    SB_DATA    (so, cso->offset_point);
+    SB_DATA    (so, cso->offset_line);
+    SB_DATA    (so, cso->offset_tri);
+
+    if (cso->offset_point || cso->offset_line || cso->offset_tri) {
+        SB_BEGIN_3D(so, POLYGON_OFFSET_FACTOR, 1);
+        SB_DATA    (so, fui(cso->offset_scale));
+        SB_BEGIN_3D(so, POLYGON_OFFSET_UNITS, 1);
+        SB_DATA    (so, fui(cso->offset_units)); /* XXX: multiply by 2 ? */
+    }
+
+    assert(so->size < (sizeof(so->state) / sizeof(so->state[0])));
+    return (void *)so;
+}
+
+static void
+nvc0_rasterizer_state_bind(struct pipe_context *pipe, void *hwcso)
+{
+   struct nvc0_context *nvc0 = nvc0_context(pipe);
+
+   nvc0->rast = hwcso;
+   nvc0->dirty |= NVC0_NEW_RASTERIZER;
+}
+
+static void
+nvc0_rasterizer_state_delete(struct pipe_context *pipe, void *hwcso)
+{
+   FREE(hwcso);
+}
+
+static void *
+nvc0_zsa_state_create(struct pipe_context *pipe,
+                      const struct pipe_depth_stencil_alpha_state *cso)
+{
+   struct nvc0_zsa_stateobj *so = CALLOC_STRUCT(nvc0_zsa_stateobj);
+
+   so->pipe = *cso;
+
+   SB_IMMED_3D(so, DEPTH_WRITE_ENABLE, cso->depth.writemask);
+   SB_BEGIN_3D(so, DEPTH_TEST_ENABLE, 1);
+   if (cso->depth.enabled) {
+      SB_DATA    (so, 1);
+      SB_BEGIN_3D(so, DEPTH_TEST_FUNC, 1);
+      SB_DATA    (so, nvgl_comparison_op(cso->depth.func));
+   } else {
+      SB_DATA    (so, 0);
+   }
+
+   if (cso->stencil[0].enabled) {
+      SB_BEGIN_3D(so, STENCIL_FRONT_ENABLE, 5);
+      SB_DATA    (so, 1);
+      SB_DATA    (so, nvgl_stencil_op(cso->stencil[0].fail_op));
+      SB_DATA    (so, nvgl_stencil_op(cso->stencil[0].zfail_op));
+      SB_DATA    (so, nvgl_stencil_op(cso->stencil[0].zpass_op));
+      SB_DATA    (so, nvgl_comparison_op(cso->stencil[0].func));
+      SB_BEGIN_3D(so, STENCIL_FRONT_MASK, 2);
+      SB_DATA    (so, cso->stencil[0].writemask);
+      SB_DATA    (so, cso->stencil[0].valuemask);
+   } else {
+      SB_IMMED_3D(so, STENCIL_FRONT_ENABLE, 0);
+   }
+
+   if (cso->stencil[1].enabled) {
+      SB_BEGIN_3D(so, STENCIL_TWO_SIDE_ENABLE, 5);
+      SB_DATA    (so, 1);
+      SB_DATA    (so, nvgl_stencil_op(cso->stencil[1].fail_op));
+      SB_DATA    (so, nvgl_stencil_op(cso->stencil[1].zfail_op));
+      SB_DATA    (so, nvgl_stencil_op(cso->stencil[1].zpass_op));
+      SB_DATA    (so, nvgl_comparison_op(cso->stencil[1].func));
+      SB_BEGIN_3D(so, STENCIL_BACK_MASK, 2);
+      SB_DATA    (so, cso->stencil[1].writemask);
+      SB_DATA    (so, cso->stencil[1].valuemask);
+   } else {
+      SB_IMMED_3D(so, STENCIL_TWO_SIDE_ENABLE, 0);
+   }
+    
+   SB_BEGIN_3D(so, ALPHA_TEST_ENABLE, 1);
+   if (cso->alpha.enabled) {
+      SB_DATA    (so, 1);
+      SB_BEGIN_3D(so, ALPHA_TEST_REF, 2);
+      SB_DATA    (so, fui(cso->alpha.ref_value));
+      SB_DATA    (so, nvgl_comparison_op(cso->alpha.func));
+   } else {
+      SB_DATA    (so, 0);
+   }
+
+   assert(so->size < (sizeof(so->state) / sizeof(so->state[0])));
+   return (void *)so;
+}
+
+static void
+nvc0_zsa_state_bind(struct pipe_context *pipe, void *hwcso)
+{
+   struct nvc0_context *nvc0 = nvc0_context(pipe);
+
+   nvc0->zsa = hwcso;
+   nvc0->dirty |= NVC0_NEW_ZSA;
+}
+
+static void
+nvc0_zsa_state_delete(struct pipe_context *pipe, void *hwcso)
+{
+   FREE(hwcso);
+}
+
+/* ====================== SAMPLERS AND TEXTURES ================================
+ */
+
+#define NV50_TSC_WRAP_CASE(n) \
+    case PIPE_TEX_WRAP_##n: return NV50_TSC_WRAP_##n
+
+static INLINE unsigned
+nv50_tsc_wrap_mode(unsigned wrap)
+{
+   switch (wrap) {
+   NV50_TSC_WRAP_CASE(REPEAT);
+   NV50_TSC_WRAP_CASE(MIRROR_REPEAT);
+   NV50_TSC_WRAP_CASE(CLAMP_TO_EDGE);
+   NV50_TSC_WRAP_CASE(CLAMP_TO_BORDER);
+   NV50_TSC_WRAP_CASE(CLAMP);
+   NV50_TSC_WRAP_CASE(MIRROR_CLAMP_TO_EDGE);
+   NV50_TSC_WRAP_CASE(MIRROR_CLAMP_TO_BORDER);
+   NV50_TSC_WRAP_CASE(MIRROR_CLAMP);
+   default:
+       NOUVEAU_ERR("unknown wrap mode: %d\n", wrap);
+       return NV50_TSC_WRAP_REPEAT;
+   }
+}
+
+static void *
+nvc0_sampler_state_create(struct pipe_context *pipe,
+                          const struct pipe_sampler_state *cso)
+{
+   struct nvc0_tsc_entry *so = CALLOC_STRUCT(nvc0_tsc_entry);
+   float f[2];
+
+   so->id = -1;
+
+   so->tsc[0] = (0x00026000 |
+                 (nv50_tsc_wrap_mode(cso->wrap_s) << 0) |
+                 (nv50_tsc_wrap_mode(cso->wrap_t) << 3) |
+                 (nv50_tsc_wrap_mode(cso->wrap_r) << 6));
+
+   switch (cso->mag_img_filter) {
+   case PIPE_TEX_FILTER_LINEAR:
+      so->tsc[1] |= NV50_TSC_1_MAGF_LINEAR;
+      break;
+   case PIPE_TEX_FILTER_NEAREST:
+   default:
+      so->tsc[1] |= NV50_TSC_1_MAGF_NEAREST;
+      break;
+   }
+
+   switch (cso->min_img_filter) {
+   case PIPE_TEX_FILTER_LINEAR:
+      so->tsc[1] |= NV50_TSC_1_MINF_LINEAR;
+      break;
+   case PIPE_TEX_FILTER_NEAREST:
+   default:
+      so->tsc[1] |= NV50_TSC_1_MINF_NEAREST;
+      break;
+   }
+
+   switch (cso->min_mip_filter) {
+   case PIPE_TEX_MIPFILTER_LINEAR:
+      so->tsc[1] |= NV50_TSC_1_MIPF_LINEAR;
+      break;
+   case PIPE_TEX_MIPFILTER_NEAREST:
+      so->tsc[1] |= NV50_TSC_1_MIPF_NEAREST;
+      break;
+   case PIPE_TEX_MIPFILTER_NONE:
+   default:
+      so->tsc[1] |= NV50_TSC_1_MIPF_NONE;
+      break;
+   }
+
+   if (cso->max_anisotropy >= 16)
+      so->tsc[0] |= (7 << 20);
+   else
+   if (cso->max_anisotropy >= 12)
+      so->tsc[0] |= (6 << 20);
+   else {
+      so->tsc[0] |= (cso->max_anisotropy >> 1) << 20;
+
+      if (cso->max_anisotropy >= 4)
+         so->tsc[1] |= NV50_TSC_1_UNKN_ANISO_35;
+      else
+      if (cso->max_anisotropy >= 2)
+         so->tsc[1] |= NV50_TSC_1_UNKN_ANISO_15;
+   }
+
+   if (cso->compare_mode == PIPE_TEX_COMPARE_R_TO_TEXTURE) {
+      /* NOTE: must be deactivated for non-shadow textures */
+      so->tsc[0] |= (1 << 9);
+      so->tsc[0] |= (nvgl_comparison_op(cso->compare_func) & 0x7) << 10;
+   }
+
+   f[0] = CLAMP(cso->lod_bias, -16.0f, 15.0f);
+   so->tsc[1] |= ((int)(f[0] * 256.0f) & 0x1fff) << 12;
+
+   f[0] = CLAMP(cso->min_lod, 0.0f, 15.0f);
+   f[1] = CLAMP(cso->max_lod, 0.0f, 15.0f);
+   so->tsc[2] |=
+      (((int)(f[1] * 256.0f) & 0xfff) << 12) | ((int)(f[0] * 256.0f) & 0xfff);
+
+   so->tsc[4] = fui(cso->border_color[0]);
+   so->tsc[5] = fui(cso->border_color[1]);
+   so->tsc[6] = fui(cso->border_color[2]);
+   so->tsc[7] = fui(cso->border_color[3]);
+
+   return (void *)so;
+}
+
+static void
+nvc0_sampler_state_delete(struct pipe_context *pipe, void *hwcso)
+{
+   unsigned s, i;
+
+   for (s = 0; s < 5; ++s)
+      for (i = 0; i < nvc0_context(pipe)->num_samplers[s]; ++i)
+         if (nvc0_context(pipe)->samplers[s][i] == hwcso)
+            nvc0_context(pipe)->samplers[s][i] = NULL;
+
+   nvc0_screen_tsc_free(nvc0_context(pipe)->screen, nvc0_tsc_entry(hwcso));
+
+   FREE(hwcso);
+}
+
+static INLINE void
+nvc0_stage_sampler_states_bind(struct nvc0_context *nvc0, int s,
+                               unsigned nr, void **hwcso)
+{
+   unsigned i;
+
+   for (i = 0; i < nr; ++i) {
+      struct nvc0_tsc_entry *old = nvc0->samplers[s][i];
+
+      nvc0->samplers[s][i] = nvc0_tsc_entry(hwcso[i]);
+      if (old)
+         nvc0_screen_tsc_unlock(nvc0->screen, old);
+   }
+   for (; i < nvc0->num_samplers[s]; ++i)
+      if (nvc0->samplers[s][i])
+         nvc0_screen_tsc_unlock(nvc0->screen, nvc0->samplers[s][i]);
+
+   nvc0->num_samplers[s] = nr;
+
+   nvc0->dirty |= NVC0_NEW_SAMPLERS;
+}
+
+static void
+nvc0_vp_sampler_states_bind(struct pipe_context *pipe, unsigned nr, void **s)
+{
+   nvc0_stage_sampler_states_bind(nvc0_context(pipe), 0, nr, s);
+}
+
+static void
+nvc0_fp_sampler_states_bind(struct pipe_context *pipe, unsigned nr, void **s)
+{
+   nvc0_stage_sampler_states_bind(nvc0_context(pipe), 4, nr, s);
+}
+
+static void
+nvc0_gp_sampler_states_bind(struct pipe_context *pipe, unsigned nr, void **s)
+{
+   nvc0_stage_sampler_states_bind(nvc0_context(pipe), 3, nr, s);
+}
+
+/* NOTE: only called when not referenced anywhere, won't be bound */
+static void
+nvc0_sampler_view_destroy(struct pipe_context *pipe,
+                          struct pipe_sampler_view *view)
+{
+   pipe_resource_reference(&view->texture, NULL);
+
+   nvc0_screen_tic_free(nvc0_context(pipe)->screen, nvc0_tic_entry(view));
+
+   FREE(nvc0_tic_entry(view));
+}
+
+static INLINE void
+nvc0_stage_set_sampler_views(struct nvc0_context *nvc0, int s,
+                             unsigned nr,
+                             struct pipe_sampler_view **views)
+{
+   unsigned i;
+
+   for (i = 0; i < nr; ++i) {
+      struct nvc0_tic_entry *old = nvc0_tic_entry(nvc0->textures[s][i]);
+      if (old)
+         nvc0_screen_tic_unlock(nvc0->screen, old);
+
+      pipe_sampler_view_reference(&nvc0->textures[s][i], views[i]);
+   }
+
+   for (i = nr; i < nvc0->num_textures[s]; ++i) {
+      struct nvc0_tic_entry *old = nvc0_tic_entry(nvc0->textures[s][i]);
+      if (!old)
+         continue;
+      nvc0_screen_tic_unlock(nvc0->screen, old);
+
+      pipe_sampler_view_reference(&nvc0->textures[s][i], NULL);
+   }
+
+   nvc0->num_textures[s] = nr;
+
+   nvc0_bufctx_reset(nvc0, NVC0_BUFCTX_TEXTURES);
+
+   nvc0->dirty |= NVC0_NEW_TEXTURES;
+}
+
+static void
+nvc0_vp_set_sampler_views(struct pipe_context *pipe,
+                          unsigned nr,
+                          struct pipe_sampler_view **views)
+{
+   nvc0_stage_set_sampler_views(nvc0_context(pipe), 0, nr, views);
+}
+
+static void
+nvc0_fp_set_sampler_views(struct pipe_context *pipe,
+                          unsigned nr,
+                          struct pipe_sampler_view **views)
+{
+   nvc0_stage_set_sampler_views(nvc0_context(pipe), 4, nr, views);
+}
+
+static void
+nvc0_gp_set_sampler_views(struct pipe_context *pipe,
+                          unsigned nr,
+                          struct pipe_sampler_view **views)
+{
+   nvc0_stage_set_sampler_views(nvc0_context(pipe), 3, nr, views);
+}
+
+/* ============================= SHADERS =======================================
+ */
+
+static void *
+nvc0_sp_state_create(struct pipe_context *pipe,
+                     const struct pipe_shader_state *cso, unsigned type)
+{
+   struct nvc0_program *prog;
+
+   prog = CALLOC_STRUCT(nvc0_program);
+   if (!prog)
+      return NULL;
+
+   prog->type = type;
+   prog->pipe.tokens = tgsi_dup_tokens(cso->tokens);
+
+   return (void *)prog;
+}
+
+static void
+nvc0_sp_state_delete(struct pipe_context *pipe, void *hwcso)
+{
+   struct nvc0_program *prog = (struct nvc0_program *)hwcso;
+
+   nvc0_program_destroy(nvc0_context(pipe), prog);
+
+   FREE((void *)prog->pipe.tokens);
+   FREE(prog);
+}
+
+static void *
+nvc0_vp_state_create(struct pipe_context *pipe,
+                     const struct pipe_shader_state *cso)
+{
+   return nvc0_sp_state_create(pipe, cso, PIPE_SHADER_VERTEX);
+}
+
+static void
+nvc0_vp_state_bind(struct pipe_context *pipe, void *hwcso)
+{
+    struct nvc0_context *nvc0 = nvc0_context(pipe);
+
+    nvc0->vertprog = hwcso;
+    nvc0->dirty |= NVC0_NEW_VERTPROG;
+}
+
+static void *
+nvc0_fp_state_create(struct pipe_context *pipe,
+                     const struct pipe_shader_state *cso)
+{
+   return nvc0_sp_state_create(pipe, cso, PIPE_SHADER_FRAGMENT);
+}
+
+static void
+nvc0_fp_state_bind(struct pipe_context *pipe, void *hwcso)
+{
+    struct nvc0_context *nvc0 = nvc0_context(pipe);
+
+    nvc0->fragprog = hwcso;
+    nvc0->dirty |= NVC0_NEW_FRAGPROG;
+}
+
+static void *
+nvc0_gp_state_create(struct pipe_context *pipe,
+                     const struct pipe_shader_state *cso)
+{
+   return nvc0_sp_state_create(pipe, cso, PIPE_SHADER_GEOMETRY);
+}
+
+static void
+nvc0_gp_state_bind(struct pipe_context *pipe, void *hwcso)
+{
+    struct nvc0_context *nvc0 = nvc0_context(pipe);
+
+    nvc0->gmtyprog = hwcso;
+    nvc0->dirty |= NVC0_NEW_GMTYPROG;
+}
+
+static void
+nvc0_set_constant_buffer(struct pipe_context *pipe, uint shader, uint index,
+                         struct pipe_resource *res)
+{
+   struct nvc0_context *nvc0 = nvc0_context(pipe);
+
+   switch (shader) {
+   case PIPE_SHADER_VERTEX: shader = 0; break;
+      /*
+   case PIPE_SHADER_TESSELLATION_CONTROL: shader = 1; break;
+   case PIPE_SHADER_TESSELLATION_EVALUATION: shader = 2; break;
+      */
+   case PIPE_SHADER_GEOMETRY: shader = 3; break;
+   case PIPE_SHADER_FRAGMENT: shader = 4; break;
+   default:
+      assert(0);
+      break;
+   }
+
+   if (nvc0->constbuf[shader][index])
+      nvc0_bufctx_del_resident(nvc0, NVC0_BUFCTX_CONSTANT,
+			       nvc0_resource(
+				       nvc0->constbuf[shader][index]));
+
+   pipe_resource_reference(&nvc0->constbuf[shader][index], res);
+
+   nvc0->constbuf_dirty[shader] |= 1 << index;
+
+   nvc0->dirty |= NVC0_NEW_CONSTBUF;
+}
+
+/* =============================================================================
+ */
+
+static void
+nvc0_set_blend_color(struct pipe_context *pipe,
+                     const struct pipe_blend_color *bcol)
+{
+    struct nvc0_context *nvc0 = nvc0_context(pipe);
+
+    nvc0->blend_colour = *bcol;
+    nvc0->dirty |= NVC0_NEW_BLEND_COLOUR;
+}
+
+static void
+nvc0_set_stencil_ref(struct pipe_context *pipe,
+                     const struct pipe_stencil_ref *sr)
+{
+    struct nvc0_context *nvc0 = nvc0_context(pipe);
+
+    nvc0->stencil_ref = *sr;
+    nvc0->dirty |= NVC0_NEW_STENCIL_REF;
+}
+
+static void
+nvc0_set_clip_state(struct pipe_context *pipe,
+                    const struct pipe_clip_state *clip)
+{
+    struct nvc0_context *nvc0 = nvc0_context(pipe);
+    const unsigned size = clip->nr * sizeof(clip->ucp[0]);
+
+    memcpy(&nvc0->clip.ucp[0][0], &clip->ucp[0][0], size);
+    nvc0->clip.nr = clip->nr;
+
+    nvc0->clip.depth_clamp = clip->depth_clamp;
+
+    nvc0->dirty |= NVC0_NEW_CLIP;
+}
+
+static void
+nvc0_set_sample_mask(struct pipe_context *pipe, unsigned sample_mask)
+{
+    struct nvc0_context *nvc0 = nvc0_context(pipe);
+
+    nvc0->sample_mask = sample_mask;
+    nvc0->dirty |= NVC0_NEW_SAMPLE_MASK;
+}
+
+
+static void
+nvc0_set_framebuffer_state(struct pipe_context *pipe,
+                           const struct pipe_framebuffer_state *fb)
+{
+    struct nvc0_context *nvc0 = nvc0_context(pipe);
+
+    nvc0->framebuffer = *fb;
+    nvc0->dirty |= NVC0_NEW_FRAMEBUFFER;
+}
+
+static void
+nvc0_set_polygon_stipple(struct pipe_context *pipe,
+                         const struct pipe_poly_stipple *stipple)
+{
+    struct nvc0_context *nvc0 = nvc0_context(pipe);
+
+    nvc0->stipple = *stipple;
+    nvc0->dirty |= NVC0_NEW_STIPPLE;
+}
+
+static void
+nvc0_set_scissor_state(struct pipe_context *pipe,
+                       const struct pipe_scissor_state *scissor)
+{
+    struct nvc0_context *nvc0 = nvc0_context(pipe);
+
+    nvc0->scissor = *scissor;
+    nvc0->dirty |= NVC0_NEW_SCISSOR;
+}
+
+static void
+nvc0_set_viewport_state(struct pipe_context *pipe,
+                        const struct pipe_viewport_state *vpt)
+{
+    struct nvc0_context *nvc0 = nvc0_context(pipe);
+
+    nvc0->viewport = *vpt;
+    nvc0->dirty |= NVC0_NEW_VIEWPORT;
+}
+
+static void
+nvc0_set_vertex_buffers(struct pipe_context *pipe,
+                        unsigned count,
+                        const struct pipe_vertex_buffer *vb)
+{
+    struct nvc0_context *nvc0 = nvc0_context(pipe);
+    unsigned i;
+
+    for (i = 0; i < count; ++i)
+       pipe_resource_reference(&nvc0->vtxbuf[i].buffer, vb[i].buffer);
+    for (; i < nvc0->num_vtxbufs; ++i)
+       pipe_resource_reference(&nvc0->vtxbuf[i].buffer, NULL);
+
+    memcpy(nvc0->vtxbuf, vb, sizeof(*vb) * count);
+    nvc0->num_vtxbufs = count;
+
+    nvc0_bufctx_reset(nvc0, NVC0_BUFCTX_VERTEX);
+
+    nvc0->dirty |= NVC0_NEW_ARRAYS;
+}
+
+static void
+nvc0_set_index_buffer(struct pipe_context *pipe,
+                      const struct pipe_index_buffer *ib)
+{
+    struct nvc0_context *nvc0 = nvc0_context(pipe);
+
+    if (ib)
+        memcpy(&nvc0->idxbuf, ib, sizeof(nvc0->idxbuf));
+    else
+        nvc0->idxbuf.buffer = NULL;
+}
+
+static void
+nvc0_vertex_state_bind(struct pipe_context *pipe, void *hwcso)
+{
+    struct nvc0_context *nvc0 = nvc0_context(pipe);
+
+    nvc0->vertex = hwcso;
+    nvc0->dirty |= NVC0_NEW_VERTEX;
+}
+
+void
+nvc0_init_state_functions(struct nvc0_context *nvc0)
+{
+    nvc0->pipe.create_blend_state = nvc0_blend_state_create;
+    nvc0->pipe.bind_blend_state = nvc0_blend_state_bind;
+    nvc0->pipe.delete_blend_state = nvc0_blend_state_delete;
+
+    nvc0->pipe.create_rasterizer_state = nvc0_rasterizer_state_create;
+    nvc0->pipe.bind_rasterizer_state = nvc0_rasterizer_state_bind;
+    nvc0->pipe.delete_rasterizer_state = nvc0_rasterizer_state_delete;
+
+    nvc0->pipe.create_depth_stencil_alpha_state = nvc0_zsa_state_create;
+    nvc0->pipe.bind_depth_stencil_alpha_state = nvc0_zsa_state_bind;
+    nvc0->pipe.delete_depth_stencil_alpha_state = nvc0_zsa_state_delete;
+
+    nvc0->pipe.create_sampler_state = nvc0_sampler_state_create;
+    nvc0->pipe.delete_sampler_state = nvc0_sampler_state_delete;
+    nvc0->pipe.bind_vertex_sampler_states   = nvc0_vp_sampler_states_bind;
+    nvc0->pipe.bind_fragment_sampler_states = nvc0_fp_sampler_states_bind;
+    nvc0->pipe.bind_geometry_sampler_states = nvc0_gp_sampler_states_bind;
+
+    nvc0->pipe.create_sampler_view = nvc0_create_sampler_view;
+    nvc0->pipe.sampler_view_destroy = nvc0_sampler_view_destroy;
+    nvc0->pipe.set_vertex_sampler_views   = nvc0_vp_set_sampler_views;
+    nvc0->pipe.set_fragment_sampler_views = nvc0_fp_set_sampler_views;
+    nvc0->pipe.set_geometry_sampler_views = nvc0_gp_set_sampler_views;
+
+    nvc0->pipe.create_vs_state = nvc0_vp_state_create;
+    nvc0->pipe.create_fs_state = nvc0_fp_state_create;
+    nvc0->pipe.create_gs_state = nvc0_gp_state_create;
+    nvc0->pipe.bind_vs_state = nvc0_vp_state_bind;
+    nvc0->pipe.bind_fs_state = nvc0_fp_state_bind;
+    nvc0->pipe.bind_gs_state = nvc0_gp_state_bind;
+    nvc0->pipe.delete_vs_state = nvc0_sp_state_delete;
+    nvc0->pipe.delete_fs_state = nvc0_sp_state_delete;
+    nvc0->pipe.delete_gs_state = nvc0_sp_state_delete;
+
+    nvc0->pipe.set_blend_color = nvc0_set_blend_color;
+    nvc0->pipe.set_stencil_ref = nvc0_set_stencil_ref;
+    nvc0->pipe.set_clip_state = nvc0_set_clip_state;
+    nvc0->pipe.set_sample_mask = nvc0_set_sample_mask;
+    nvc0->pipe.set_constant_buffer = nvc0_set_constant_buffer;
+    nvc0->pipe.set_framebuffer_state = nvc0_set_framebuffer_state;
+    nvc0->pipe.set_polygon_stipple = nvc0_set_polygon_stipple;
+    nvc0->pipe.set_scissor_state = nvc0_set_scissor_state;
+    nvc0->pipe.set_viewport_state = nvc0_set_viewport_state;
+
+    nvc0->pipe.create_vertex_elements_state = nvc0_vertex_state_create;
+    nvc0->pipe.delete_vertex_elements_state = nvc0_vertex_state_delete;
+    nvc0->pipe.bind_vertex_elements_state = nvc0_vertex_state_bind;
+
+    nvc0->pipe.set_vertex_buffers = nvc0_set_vertex_buffers;
+    nvc0->pipe.set_index_buffer = nvc0_set_index_buffer;
+}
+
diff --git a/src/gallium/drivers/nvc0/nvc0_state_validate.c b/src/gallium/drivers/nvc0/nvc0_state_validate.c
new file mode 100644
index 00000000000..25aec0244db
--- /dev/null
+++ b/src/gallium/drivers/nvc0/nvc0_state_validate.c
@@ -0,0 +1,430 @@
+
+#include "nvc0_context.h"
+#include "os/os_time.h"
+
+static void
+nvc0_validate_zcull(struct nvc0_context *nvc0)
+{
+    struct nouveau_channel *chan = nvc0->screen->base.channel;
+    struct pipe_framebuffer_state *fb = &nvc0->framebuffer;
+    struct nvc0_surface *sf = nvc0_surface(fb->zsbuf);
+    struct nvc0_miptree *mt = nvc0_miptree(sf->base.texture);
+    struct nouveau_bo *bo = mt->base.bo;
+    uint32_t size;
+    uint32_t offset = align(mt->total_size, 1 << 17);
+    unsigned width, height;
+
+    assert(mt->base.base.depth0 == 1 && mt->base.base.array_size < 2);
+
+    size = mt->total_size * 2;
+
+    height = align(fb->height, 32);
+    width = fb->width % 224;
+    if (width)
+       width = fb->width + (224 - width);
+    else
+       width = fb->width;
+
+    BEGIN_RING(chan, RING_3D_(0x1590), 1); /* ZCULL_REGION_INDEX (bits 0x3f) */
+    OUT_RING  (chan, 0);
+    BEGIN_RING(chan, RING_3D_(0x07e8), 2); /* ZCULL_ADDRESS_A_HIGH */
+    OUT_RELOCh(chan, bo, offset, NOUVEAU_BO_VRAM | NOUVEAU_BO_RDWR);
+    OUT_RELOCl(chan, bo, offset, NOUVEAU_BO_VRAM | NOUVEAU_BO_RDWR);
+    offset += 1 << 17;
+    BEGIN_RING(chan, RING_3D_(0x07f0), 2); /* ZCULL_ADDRESS_B_HIGH */
+    OUT_RELOCh(chan, bo, offset, NOUVEAU_BO_VRAM | NOUVEAU_BO_RDWR);
+    OUT_RELOCl(chan, bo, offset, NOUVEAU_BO_VRAM | NOUVEAU_BO_RDWR);
+    BEGIN_RING(chan, RING_3D_(0x07e0), 2);
+    OUT_RING  (chan, size);
+    OUT_RING  (chan, size >> 16);
+    BEGIN_RING(chan, RING_3D_(0x15c8), 1); /* bits 0x3 */
+    OUT_RING  (chan, 2);
+    BEGIN_RING(chan, RING_3D_(0x07c0), 4); /* ZCULL dimensions */
+    OUT_RING  (chan, width);
+    OUT_RING  (chan, height);
+    OUT_RING  (chan, 1);
+    OUT_RING  (chan, 0);
+    BEGIN_RING(chan, RING_3D_(0x15fc), 2);
+    OUT_RING  (chan, 0); /* bits 0xffff */
+    OUT_RING  (chan, 0); /* bits 0xffff */
+    BEGIN_RING(chan, RING_3D_(0x1958), 1);
+    OUT_RING  (chan, 0); /* bits ~0 */
+}
+
+static void
+nvc0_validate_fb(struct nvc0_context *nvc0)
+{
+    struct nouveau_channel *chan = nvc0->screen->base.channel;
+    struct pipe_framebuffer_state *fb = &nvc0->framebuffer;
+    unsigned i;
+
+    nvc0_bufctx_reset(nvc0, NVC0_BUFCTX_FRAME);
+
+    BEGIN_RING(chan, RING_3D(RT_CONTROL), 1);
+    OUT_RING  (chan, (076543210 << 4) | fb->nr_cbufs);
+    BEGIN_RING(chan, RING_3D(SCREEN_SCISSOR_HORIZ), 2);
+    OUT_RING  (chan, fb->width << 16);
+    OUT_RING  (chan, fb->height << 16);
+
+    for (i = 0; i < fb->nr_cbufs; ++i) {
+        struct nvc0_miptree *mt = nvc0_miptree(fb->cbufs[i]->texture);
+        struct nvc0_surface *sf = nvc0_surface(fb->cbufs[i]);
+        struct nouveau_bo *bo = mt->base.bo;
+        uint32_t offset = sf->offset;
+        
+        BEGIN_RING(chan, RING_3D(RT_ADDRESS_HIGH(i)), 8);
+        OUT_RELOCh(chan, bo, offset, NOUVEAU_BO_VRAM | NOUVEAU_BO_RDWR);
+        OUT_RELOCl(chan, bo, offset, NOUVEAU_BO_VRAM | NOUVEAU_BO_RDWR);
+        OUT_RING  (chan, sf->width);
+        OUT_RING  (chan, sf->height);
+        OUT_RING  (chan, nvc0_format_table[sf->base.format].rt);
+        OUT_RING  (chan, (mt->layout_3d << 16) |
+                   mt->level[sf->base.u.tex.level].tile_mode);
+        OUT_RING  (chan, sf->depth);
+        OUT_RING  (chan, mt->layer_stride >> 2);
+
+        nvc0_bufctx_add_resident(nvc0, NVC0_BUFCTX_FRAME, &mt->base,
+                                 NOUVEAU_BO_VRAM | NOUVEAU_BO_RDWR);
+    }
+
+    if (fb->zsbuf) {
+        struct nvc0_miptree *mt = nvc0_miptree(fb->zsbuf->texture);
+        struct nvc0_surface *sf = nvc0_surface(fb->zsbuf);
+        struct nouveau_bo *bo = mt->base.bo;
+        int unk = mt->base.base.target == PIPE_TEXTURE_2D;
+        uint32_t offset = sf->offset;
+        
+        BEGIN_RING(chan, RING_3D(ZETA_ADDRESS_HIGH), 5);
+        OUT_RELOCh(chan, bo, offset, NOUVEAU_BO_VRAM | NOUVEAU_BO_RDWR);
+        OUT_RELOCl(chan, bo, offset, NOUVEAU_BO_VRAM | NOUVEAU_BO_RDWR);
+        OUT_RING  (chan, nvc0_format_table[fb->zsbuf->format].rt);
+        OUT_RING  (chan, mt->level[sf->base.u.tex.level].tile_mode);
+        OUT_RING  (chan, mt->layer_stride >> 2);
+        BEGIN_RING(chan, RING_3D(ZETA_ENABLE), 1);
+        OUT_RING  (chan, 1);
+        BEGIN_RING(chan, RING_3D(ZETA_HORIZ), 3);
+        OUT_RING  (chan, sf->width);
+        OUT_RING  (chan, sf->height);
+        OUT_RING  (chan, (unk << 16) | sf->depth);
+
+        nvc0_bufctx_add_resident(nvc0, NVC0_BUFCTX_FRAME, &mt->base,
+                                 NOUVEAU_BO_VRAM | NOUVEAU_BO_RDWR);
+    } else {
+        BEGIN_RING(chan, RING_3D(ZETA_ENABLE), 1);
+        OUT_RING  (chan, 0);
+    }
+
+#ifndef NVC0_SCISSORS_CLIPPING
+    BEGIN_RING(chan, RING_3D(VIEWPORT_HORIZ(0)), 2);
+    OUT_RING  (chan, fb->width << 16);
+    OUT_RING  (chan, fb->height << 16);
+#endif
+}
+
+static void
+nvc0_validate_blend_colour(struct nvc0_context *nvc0)
+{
+    struct nouveau_channel *chan = nvc0->screen->base.channel;
+
+    BEGIN_RING(chan, RING_3D(BLEND_COLOR(0)), 4);
+    OUT_RINGf (chan, nvc0->blend_colour.color[0]);
+    OUT_RINGf (chan, nvc0->blend_colour.color[1]);
+    OUT_RINGf (chan, nvc0->blend_colour.color[2]);
+    OUT_RINGf (chan, nvc0->blend_colour.color[3]);    
+}
+
+static void
+nvc0_validate_stencil_ref(struct nvc0_context *nvc0)
+{
+    struct nouveau_channel *chan = nvc0->screen->base.channel;
+
+    BEGIN_RING(chan, RING_3D(STENCIL_FRONT_FUNC_REF), 1);
+    OUT_RING  (chan, nvc0->stencil_ref.ref_value[0]);
+    BEGIN_RING(chan, RING_3D(STENCIL_BACK_FUNC_REF), 1);
+    OUT_RING  (chan, nvc0->stencil_ref.ref_value[1]);
+}
+
+static void
+nvc0_validate_stipple(struct nvc0_context *nvc0)
+{
+    struct nouveau_channel *chan = nvc0->screen->base.channel;
+    unsigned i;
+
+    BEGIN_RING(chan, RING_3D(POLYGON_STIPPLE_PATTERN(0)), 32);
+    for (i = 0; i < 32; ++i)
+        OUT_RING(chan, util_bswap32(nvc0->stipple.stipple[i]));
+}
+
+static void
+nvc0_validate_scissor(struct nvc0_context *nvc0)
+{
+    struct nouveau_channel *chan = nvc0->screen->base.channel;
+    struct pipe_scissor_state *s = &nvc0->scissor;
+#ifdef NVC0_SCISSORS_CLIPPING
+    struct pipe_viewport_state *vp = &nvc0->viewport;
+    int minx, maxx, miny, maxy;
+
+    if (!(nvc0->dirty &
+          (NVC0_NEW_SCISSOR | NVC0_NEW_VIEWPORT | NVC0_NEW_FRAMEBUFFER)) &&
+        nvc0->state.scissor == nvc0->rast->pipe.scissor)
+       return;
+    nvc0->state.scissor = nvc0->rast->pipe.scissor;
+
+    if (nvc0->state.scissor) {
+       minx = s->minx;
+       maxx = s->maxx;
+       miny = s->miny;
+       maxy = s->maxy;
+    } else {
+       minx = 0;
+       maxx = nvc0->framebuffer.width;
+       miny = 0;
+       maxy = nvc0->framebuffer.height;
+    }
+
+    minx = MAX2(minx, (int)(vp->translate[0] - fabsf(vp->scale[0])));
+    maxx = MIN2(maxx, (int)(vp->translate[0] + fabsf(vp->scale[0])));
+    miny = MAX2(miny, (int)(vp->translate[1] - fabsf(vp->scale[1])));
+    maxy = MIN2(maxy, (int)(vp->translate[1] + fabsf(vp->scale[1])));
+
+    BEGIN_RING(chan, RING_3D(SCISSOR_HORIZ(0)), 2);
+    OUT_RING  (chan, (maxx << 16) | minx);
+    OUT_RING  (chan, (maxy << 16) | miny);
+    BEGIN_RING(chan, RING_3D(VIEWPORT_HORIZ(0)), 2);
+    OUT_RING  (chan, ((maxx - minx) << 16) | minx);
+    OUT_RING  (chan, ((maxy - miny) << 16) | miny);
+#else
+    BEGIN_RING(chan, RING_3D(SCISSOR_HORIZ(0)), 2);
+    OUT_RING  (chan, (s->maxx << 16) | s->minx);
+    OUT_RING  (chan, (s->maxy << 16) | s->miny);
+#endif
+}
+
+static void
+nvc0_validate_viewport(struct nvc0_context *nvc0)
+{
+    struct nouveau_channel *chan = nvc0->screen->base.channel;
+
+    BEGIN_RING(chan, RING_3D(VIEWPORT_TRANSLATE_X(0)), 3);
+    OUT_RINGf (chan, nvc0->viewport.translate[0]);
+    OUT_RINGf (chan, nvc0->viewport.translate[1]);
+    OUT_RINGf (chan, nvc0->viewport.translate[2]);
+    BEGIN_RING(chan, RING_3D(VIEWPORT_SCALE_X(0)), 3);
+    OUT_RINGf (chan, nvc0->viewport.scale[0]);
+    OUT_RINGf (chan, nvc0->viewport.scale[1]);
+    OUT_RINGf (chan, nvc0->viewport.scale[2]);
+
+#ifdef NVC0_SCISSORS_CLIPPING
+    BEGIN_RING(chan, RING_3D(DEPTH_RANGE_NEAR(0)), 2);
+    OUT_RINGf (chan, nvc0->viewport.translate[2] - nvc0->viewport.scale[2]);
+    OUT_RINGf (chan, nvc0->viewport.translate[2] + nvc0->viewport.scale[2]);
+#endif
+}
+
+static void
+nvc0_validate_clip(struct nvc0_context *nvc0)
+{
+   struct nouveau_channel *chan = nvc0->screen->base.channel;
+   uint32_t clip;
+
+   clip = nvc0->clip.depth_clamp ? 0x201a : 0x0002;
+#ifndef NVC0_SCISSORS_CLIPPING
+   clip |= 0x1080;
+#endif
+
+   BEGIN_RING(chan, RING_3D(VIEW_VOLUME_CLIP_CTRL), 1);
+   OUT_RING  (chan, clip);
+
+   if (nvc0->clip.nr) {
+      struct nouveau_bo *bo = nvc0->screen->uniforms;
+
+      BEGIN_RING(chan, RING_3D(CB_SIZE), 3);
+      OUT_RING  (chan, 256);
+      OUT_RELOCh(chan, bo, 5 << 16, NOUVEAU_BO_VRAM | NOUVEAU_BO_RD);
+      OUT_RELOCl(chan, bo, 5 << 16, NOUVEAU_BO_VRAM | NOUVEAU_BO_RD);
+      BEGIN_RING_1I(chan, RING_3D(CB_POS), nvc0->clip.nr * 4 + 1);
+      OUT_RING  (chan, 0);
+      OUT_RINGp (chan, &nvc0->clip.ucp[0][0], nvc0->clip.nr * 4);
+
+      BEGIN_RING(chan, RING_3D(VP_CLIP_DISTANCE_ENABLE), 1);
+      OUT_RING  (chan, (1 << nvc0->clip.nr) - 1);
+   } else {
+      IMMED_RING(chan, RING_3D(VP_CLIP_DISTANCE_ENABLE), 0);
+   }
+}
+
+static void
+nvc0_validate_blend(struct nvc0_context *nvc0)
+{
+   struct nouveau_channel *chan = nvc0->screen->base.channel;
+
+   WAIT_RING(chan, nvc0->blend->size);
+   OUT_RINGp(chan, nvc0->blend->state, nvc0->blend->size);
+}
+
+static void
+nvc0_validate_zsa(struct nvc0_context *nvc0)
+{
+   struct nouveau_channel *chan = nvc0->screen->base.channel;
+
+   WAIT_RING(chan, nvc0->zsa->size);
+   OUT_RINGp(chan, nvc0->zsa->state, nvc0->zsa->size);
+}
+
+static void
+nvc0_validate_rasterizer(struct nvc0_context *nvc0)
+{
+   struct nouveau_channel *chan = nvc0->screen->base.channel;
+
+   WAIT_RING(chan, nvc0->rast->size);
+   OUT_RINGp(chan, nvc0->rast->state, nvc0->rast->size);
+}
+
+static void
+nvc0_constbufs_validate(struct nvc0_context *nvc0)
+{
+   struct nouveau_channel *chan = nvc0->screen->base.channel;
+   struct nouveau_bo *bo;
+   unsigned s;
+
+   for (s = 0; s < 5; ++s) {
+      struct nvc0_resource *res;
+      int i;
+
+      while (nvc0->constbuf_dirty[s]) {
+         unsigned base = 0;
+         unsigned offset = 0, words = 0;
+         boolean rebind = TRUE;
+
+         i = ffs(nvc0->constbuf_dirty[s]) - 1;
+         nvc0->constbuf_dirty[s] &= ~(1 << i);
+
+         res = nvc0_resource(nvc0->constbuf[s][i]);
+         if (!res) {
+            BEGIN_RING(chan, RING_3D(CB_BIND(s)), 1);
+            OUT_RING  (chan, (i << 4) | 0);
+            if (i == 0)
+               nvc0->state.uniform_buffer_bound[s] = 0;
+            continue;
+         }
+
+         if (!nvc0_resource_mapped_by_gpu(&res->base)) {
+            if (i == 0) {
+               base = s << 16;
+               bo = nvc0->screen->uniforms;
+
+               if (nvc0->state.uniform_buffer_bound[s] >= res->base.width0)
+                  rebind = FALSE;
+               else
+                  nvc0->state.uniform_buffer_bound[s] =
+                     align(res->base.width0, 0x100);
+            } else {
+               bo = res->bo;
+            }
+#if 0
+            nvc0_m2mf_push_linear(nvc0, bo, NOUVEAU_BO_VRAM,
+                                  base, res->base.width0, res->data);
+            BEGIN_RING(chan, RING_3D_(0x021c), 1);
+            OUT_RING  (chan, 0x1111);
+#else
+            words = res->base.width0 / 4;
+#endif
+         } else {
+            bo = res->bo;
+            if (i == 0)
+               nvc0->state.uniform_buffer_bound[s] = 0;
+         }
+
+         if (bo != nvc0->screen->uniforms)
+            nvc0_bufctx_add_resident(nvc0, NVC0_BUFCTX_CONSTANT, res,
+                                     NOUVEAU_BO_VRAM | NOUVEAU_BO_RD);
+
+         if (rebind) {
+            BEGIN_RING(chan, RING_3D(CB_SIZE), 3);
+            OUT_RING  (chan, align(res->base.width0, 0x100));
+            OUT_RELOCh(chan, bo, base, NOUVEAU_BO_VRAM | NOUVEAU_BO_RD);
+            OUT_RELOCl(chan, bo, base, NOUVEAU_BO_VRAM | NOUVEAU_BO_RD);
+            BEGIN_RING(chan, RING_3D(CB_BIND(s)), 1);
+            OUT_RING  (chan, (i << 4) | 1);
+         }
+
+         while (words) {
+            unsigned nr = AVAIL_RING(chan);
+
+            if (nr < 16) {
+               FIRE_RING(chan);
+               continue;
+            }
+            nr = MIN2(MIN2(nr - 6, words), NV04_PFIFO_MAX_PACKET_LEN - 1);
+
+            BEGIN_RING(chan, RING_3D(CB_SIZE), 3);
+            OUT_RING  (chan, align(res->base.width0, 0x100));
+            OUT_RELOCh(chan, bo, base, NOUVEAU_BO_VRAM | NOUVEAU_BO_RD);
+            OUT_RELOCl(chan, bo, base, NOUVEAU_BO_VRAM | NOUVEAU_BO_RD);
+            BEGIN_RING_1I(chan, RING_3D(CB_POS), nr + 1);
+            OUT_RING  (chan, offset);
+            OUT_RINGp (chan, &res->data[offset], nr);
+
+            offset += nr * 4;
+            words -= nr;
+         }
+      }
+   }
+}
+
+static struct state_validate {
+    void (*func)(struct nvc0_context *);
+    uint32_t states;
+} validate_list[] = {
+    { nvc0_validate_fb,            NVC0_NEW_FRAMEBUFFER },
+    { nvc0_validate_blend,         NVC0_NEW_BLEND },
+    { nvc0_validate_zsa,           NVC0_NEW_ZSA },
+    { nvc0_validate_rasterizer,    NVC0_NEW_RASTERIZER },
+    { nvc0_validate_blend_colour,  NVC0_NEW_BLEND_COLOUR },
+    { nvc0_validate_stencil_ref,   NVC0_NEW_STENCIL_REF },
+    { nvc0_validate_stipple,       NVC0_NEW_STIPPLE },
+#ifdef NVC0_SCISSORS_CLIPPING
+    { nvc0_validate_scissor,       NVC0_NEW_SCISSOR | NVC0_NEW_VIEWPORT |
+                                   NVC0_NEW_RASTERIZER |
+                                   NVC0_NEW_FRAMEBUFFER },
+#else
+    { nvc0_validate_scissor,       NVC0_NEW_SCISSOR },
+#endif
+    { nvc0_validate_viewport,      NVC0_NEW_VIEWPORT },
+    { nvc0_validate_clip,          NVC0_NEW_CLIP },
+    { nvc0_vertprog_validate,      NVC0_NEW_VERTPROG },
+    { nvc0_tctlprog_validate,      NVC0_NEW_TCTLPROG },
+    { nvc0_tevlprog_validate,      NVC0_NEW_TEVLPROG },
+    { nvc0_gmtyprog_validate,      NVC0_NEW_GMTYPROG },
+    { nvc0_fragprog_validate,      NVC0_NEW_FRAGPROG },
+    { nvc0_constbufs_validate,     NVC0_NEW_CONSTBUF },
+    { nvc0_validate_textures,      NVC0_NEW_TEXTURES },
+    { nvc0_validate_samplers,      NVC0_NEW_SAMPLERS },
+    { nvc0_vertex_arrays_validate, NVC0_NEW_VERTEX | NVC0_NEW_ARRAYS }
+};
+#define validate_list_len (sizeof(validate_list) / sizeof(validate_list[0]))
+
+boolean
+nvc0_state_validate(struct nvc0_context *nvc0)
+{
+   unsigned i;
+#if 0
+   if (nvc0->screen->cur_ctx != nvc0) /* FIXME: not everything is valid */
+      nvc0->dirty = 0xffffffff;
+#endif
+   nvc0->screen->cur_ctx = nvc0;
+
+   if (nvc0->dirty) {
+      for (i = 0; i < validate_list_len; ++i) {
+         struct state_validate *validate = &validate_list[i];
+
+         if (nvc0->dirty & validate->states)
+            validate->func(nvc0);
+      }
+      nvc0->dirty = 0;
+   }
+
+   nvc0_bufctx_emit_relocs(nvc0);
+
+   return TRUE;
+}
diff --git a/src/gallium/drivers/nvc0/nvc0_stateobj.h b/src/gallium/drivers/nvc0/nvc0_stateobj.h
new file mode 100644
index 00000000000..e7cd94800de
--- /dev/null
+++ b/src/gallium/drivers/nvc0/nvc0_stateobj.h
@@ -0,0 +1,81 @@
+
+#ifndef __NVC0_STATEOBJ_H__
+#define __NVC0_STATEOBJ_H__
+
+#include "pipe/p_state.h"
+
+#define NVC0_SCISSORS_CLIPPING
+
+#define SB_BEGIN_3D(so, m, s)                                                  \
+   (so)->state[(so)->size++] =                                                 \
+      (0x2 << 28) | ((s) << 16) | (NVC0_SUBCH_3D << 13) | ((NVC0_3D_##m) >> 2)
+
+#define SB_IMMED_3D(so, m, d)                                                  \
+   (so)->state[(so)->size++] =                                                 \
+      (0x8 << 28) | ((d) << 16) | (NVC0_SUBCH_3D << 13) | ((NVC0_3D_##m) >> 2)
+
+#define SB_DATA(so, u) (so)->state[(so)->size++] = (u)
+
+struct nvc0_blend_stateobj {
+   struct pipe_blend_state pipe;
+   int size;
+   uint32_t state[72];
+};
+
+struct nvc0_tsc_entry {
+   int id;
+   uint32_t tsc[8];
+};
+
+static INLINE struct nvc0_tsc_entry *
+nvc0_tsc_entry(void *hwcso)
+{
+   return (struct nvc0_tsc_entry *)hwcso;
+}
+
+struct nvc0_tic_entry {
+   struct pipe_sampler_view pipe;
+   int id;
+   uint32_t tic[8];
+};
+
+static INLINE struct nvc0_tic_entry *
+nvc0_tic_entry(struct pipe_sampler_view *view)
+{
+   return (struct nvc0_tic_entry *)view;
+}
+
+struct nvc0_rasterizer_stateobj {
+   struct pipe_rasterizer_state pipe;
+   int size;
+   uint32_t state[43];
+};
+
+struct nvc0_zsa_stateobj {
+   struct pipe_depth_stencil_alpha_state pipe;
+   int size;
+   uint32_t state[29];
+};
+
+struct nvc0_vertex_element {
+   struct pipe_vertex_element pipe;
+   uint32_t state;
+};
+
+struct nvc0_vertex_stateobj {
+   struct translate *translate;
+   unsigned num_elements;
+   uint32_t instance_bits;
+   unsigned vtx_size;
+   unsigned vtx_per_packet_max;
+   struct nvc0_vertex_element element[1];
+};
+
+/* will have to lookup index -> location qualifier from nvc0_program */
+struct nvc0_tfb_state {
+   uint8_t varying_count[4];
+   uint32_t stride[4];
+   uint8_t varying_indices[1];
+};
+
+#endif
diff --git a/src/gallium/drivers/nvc0/nvc0_surface.c b/src/gallium/drivers/nvc0/nvc0_surface.c
new file mode 100644
index 00000000000..cc0a65687dc
--- /dev/null
+++ b/src/gallium/drivers/nvc0/nvc0_surface.c
@@ -0,0 +1,377 @@
+/*
+ * Copyright 2008 Ben Skeggs
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF
+ * OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <stdint.h>
+
+#include "pipe/p_defines.h"
+
+#include "util/u_inlines.h"
+#include "util/u_pack_color.h"
+#include "util/u_format.h"
+
+#include "nvc0_context.h"
+#include "nvc0_resource.h"
+
+#include "nv50_defs.xml.h"
+
+/* return TRUE for formats that can be converted among each other by NVC0_2D */
+static INLINE boolean
+nvc0_2d_format_faithful(enum pipe_format format)
+{
+   switch (format) {
+   case PIPE_FORMAT_B8G8R8A8_UNORM:
+   case PIPE_FORMAT_B8G8R8X8_UNORM:
+   case PIPE_FORMAT_B8G8R8A8_SRGB:
+   case PIPE_FORMAT_B8G8R8X8_SRGB:
+   case PIPE_FORMAT_B5G6R5_UNORM:
+   case PIPE_FORMAT_B5G5R5A1_UNORM:
+   case PIPE_FORMAT_B10G10R10A2_UNORM:
+   case PIPE_FORMAT_R8_UNORM:
+   case PIPE_FORMAT_R32G32B32A32_FLOAT:
+   case PIPE_FORMAT_R32G32B32_FLOAT:
+      return TRUE;
+   default:
+      return FALSE;
+   }
+}
+
+static INLINE uint8_t
+nvc0_2d_format(enum pipe_format format)
+{
+   uint8_t id = nvc0_format_table[format].rt;
+
+   /* Hardware values for color formats range from 0xc0 to 0xff,
+    * but the 2D engine doesn't support all of them.
+    */
+   if ((id >= 0xc0) && (0xff0843e080608409ULL & (1ULL << (id - 0xc0))))
+      return id;
+
+   switch (util_format_get_blocksize(format)) {
+   case 1:
+      return NV50_SURFACE_FORMAT_R8_UNORM;
+   case 2:
+      return NV50_SURFACE_FORMAT_R16_UNORM;
+   case 4:
+      return NV50_SURFACE_FORMAT_A8R8G8B8_UNORM;
+   default:
+      return 0;
+   }
+}
+
+static int
+nvc0_2d_texture_set(struct nouveau_channel *chan, int dst,
+                    struct nvc0_miptree *mt, unsigned level, unsigned layer)
+{
+   struct nouveau_bo *bo = mt->base.bo;
+   uint32_t width, height, depth;
+   uint32_t format;
+   uint32_t mthd = dst ? NVC0_2D_DST_FORMAT : NVC0_2D_SRC_FORMAT;
+   uint32_t flags = mt->base.domain | (dst ? NOUVEAU_BO_WR : NOUVEAU_BO_RD);
+   uint32_t offset = mt->level[level].offset;
+
+   format = nvc0_2d_format(mt->base.base.format);
+   if (!format) {
+      NOUVEAU_ERR("invalid/unsupported surface format: %s\n",
+                  util_format_name(mt->base.base.format));
+      return 1;
+   }
+
+   width = u_minify(mt->base.base.width0, level);
+   height = u_minify(mt->base.base.height0, level);
+
+   offset = mt->level[level].offset;
+   if (!mt->layout_3d) {
+      offset += mt->layer_stride * layer;
+      depth = 1;
+      layer = 0;
+   } else {
+      depth = u_minify(mt->base.base.depth0, level);
+   }
+
+   if (!(bo->tile_flags & NOUVEAU_BO_TILE_LAYOUT_MASK)) {
+      BEGIN_RING(chan, RING_2D_(mthd), 2);
+      OUT_RING  (chan, format);
+      OUT_RING  (chan, 1);
+      BEGIN_RING(chan, RING_2D_(mthd + 0x14), 5);
+      OUT_RING  (chan, mt->level[level].pitch);
+      OUT_RING  (chan, width);
+      OUT_RING  (chan, height);
+      OUT_RELOCh(chan, bo, offset, flags);
+      OUT_RELOCl(chan, bo, offset, flags);
+   } else {
+      BEGIN_RING(chan, RING_2D_(mthd), 5);
+      OUT_RING  (chan, format);
+      OUT_RING  (chan, 0);
+      OUT_RING  (chan, mt->level[level].tile_mode);
+      OUT_RING  (chan, depth);
+      OUT_RING  (chan, layer);
+      BEGIN_RING(chan, RING_2D_(mthd + 0x18), 4);
+      OUT_RING  (chan, width);
+      OUT_RING  (chan, height);
+      OUT_RELOCh(chan, bo, offset, flags);
+      OUT_RELOCl(chan, bo, offset, flags);
+   }
+
+#if 0
+   if (dst) {
+      BEGIN_RING(chan, RING_2D_(NVC0_2D_CLIP_X), 4);
+      OUT_RING  (chan, 0);
+      OUT_RING  (chan, 0);
+      OUT_RING  (chan, width);
+      OUT_RING  (chan, height);
+   }
+#endif
+   return 0;
+}
+
+static int
+nvc0_2d_texture_do_copy(struct nouveau_channel *chan,
+                        struct nvc0_miptree *dst, unsigned dst_level,
+                        unsigned dx, unsigned dy, unsigned dz,
+                        struct nvc0_miptree *src, unsigned src_level,
+                        unsigned sx, unsigned sy, unsigned sz,
+                        unsigned w, unsigned h)
+{
+   int ret;
+
+   ret = MARK_RING(chan, 2 * 16 + 32, 4);
+   if (ret)
+      return ret;
+
+   ret = nvc0_2d_texture_set(chan, 1, dst, dst_level, dz);
+   if (ret)
+      return ret;
+
+   ret = nvc0_2d_texture_set(chan, 0, src, src_level, sz);
+   if (ret)
+      return ret;
+
+   /* 0/1 = CENTER/CORNER, 10/00 = POINT/BILINEAR */
+   BEGIN_RING(chan, RING_2D(BLIT_CONTROL), 1);
+   OUT_RING  (chan, 0);
+   BEGIN_RING(chan, RING_2D(BLIT_DST_X), 4);
+   OUT_RING  (chan, dx);
+   OUT_RING  (chan, dy);
+   OUT_RING  (chan, w);
+   OUT_RING  (chan, h);
+   BEGIN_RING(chan, RING_2D(BLIT_DU_DX_FRACT), 4);
+   OUT_RING  (chan, 0);
+   OUT_RING  (chan, 1);
+   OUT_RING  (chan, 0);
+   OUT_RING  (chan, 1);
+   BEGIN_RING(chan, RING_2D(BLIT_SRC_X_FRACT), 4);
+   OUT_RING  (chan, 0);
+   OUT_RING  (chan, sx);
+   OUT_RING  (chan, 0);
+   OUT_RING  (chan, sy);
+
+   return 0;
+}
+
+static void
+nvc0_resource_copy_region(struct pipe_context *pipe,
+                          struct pipe_resource *dst, unsigned dst_level,
+                          unsigned dstx, unsigned dsty, unsigned dstz,
+                          struct pipe_resource *src, unsigned src_level,
+                          const struct pipe_box *src_box)
+{
+   struct nvc0_screen *screen = nvc0_context(pipe)->screen;
+   int ret;
+   unsigned dst_layer = dstz, src_layer = src_box->z;
+
+   assert((src->format == dst->format) ||
+          (nvc0_2d_format_faithful(src->format) &&
+           nvc0_2d_format_faithful(dst->format)));
+
+   for (; dst_layer < dstz + src_box->depth; ++dst_layer, ++src_layer) {
+      ret = nvc0_2d_texture_do_copy(screen->base.channel,
+                                    nvc0_miptree(dst), dst_level,
+                                    dstx, dsty, dst_layer,
+                                    nvc0_miptree(src), src_level,
+                                    src_box->x, src_box->y, src_layer,
+                                    src_box->width, src_box->height);
+      if (ret)
+         return;
+   }
+}
+
+static void
+nvc0_clear_render_target(struct pipe_context *pipe,
+                         struct pipe_surface *dst,
+                         const float *rgba,
+                         unsigned dstx, unsigned dsty,
+                         unsigned width, unsigned height)
+{
+	struct nvc0_context *nv50 = nvc0_context(pipe);
+	struct nvc0_screen *screen = nv50->screen;
+	struct nouveau_channel *chan = screen->base.channel;
+	struct nvc0_miptree *mt = nvc0_miptree(dst->texture);
+	struct nvc0_surface *sf = nvc0_surface(dst);
+	struct nouveau_bo *bo = mt->base.bo;
+
+	BEGIN_RING(chan, RING_3D(CLEAR_COLOR(0)), 4);
+	OUT_RINGf (chan, rgba[0]);
+	OUT_RINGf (chan, rgba[1]);
+	OUT_RINGf (chan, rgba[2]);
+	OUT_RINGf (chan, rgba[3]);
+
+	if (MARK_RING(chan, 18, 2))
+		return;
+
+	BEGIN_RING(chan, RING_3D(RT_CONTROL), 1);
+	OUT_RING  (chan, 1);
+	BEGIN_RING(chan, RING_3D(RT_ADDRESS_HIGH(0)), 8);
+	OUT_RELOCh(chan, bo, sf->offset, NOUVEAU_BO_VRAM | NOUVEAU_BO_WR);
+	OUT_RELOCl(chan, bo, sf->offset, NOUVEAU_BO_VRAM | NOUVEAU_BO_WR);
+	OUT_RING  (chan, sf->width);
+	OUT_RING  (chan, sf->height);
+	OUT_RING  (chan, nvc0_format_table[dst->format].rt);
+	OUT_RING  (chan, mt->level[sf->base.u.tex.level].tile_mode);
+	OUT_RING  (chan, 1);
+	OUT_RING  (chan, 0);
+
+	/* NOTE: only works with D3D clear flag (5097/0x143c bit 4) */
+
+	BEGIN_RING(chan, RING_3D(VIEWPORT_HORIZ(0)), 2);
+	OUT_RING  (chan, (width << 16) | dstx);
+	OUT_RING  (chan, (height << 16) | dsty);
+
+	BEGIN_RING(chan, RING_3D(CLEAR_BUFFERS), 1);
+	OUT_RING  (chan, 0x3c);
+
+	nv50->dirty |= NVC0_NEW_FRAMEBUFFER;
+}
+
+static void
+nvc0_clear_depth_stencil(struct pipe_context *pipe,
+                         struct pipe_surface *dst,
+                         unsigned clear_flags,
+                         double depth,
+                         unsigned stencil,
+                         unsigned dstx, unsigned dsty,
+                         unsigned width, unsigned height)
+{
+	struct nvc0_context *nv50 = nvc0_context(pipe);
+	struct nvc0_screen *screen = nv50->screen;
+	struct nouveau_channel *chan = screen->base.channel;
+	struct nvc0_miptree *mt = nvc0_miptree(dst->texture);
+	struct nvc0_surface *sf = nvc0_surface(dst);
+	struct nouveau_bo *bo = mt->base.bo;
+	uint32_t mode = 0;
+
+	if (clear_flags & PIPE_CLEAR_DEPTH) {
+		BEGIN_RING(chan, RING_3D(CLEAR_DEPTH), 1);
+		OUT_RINGf (chan, depth);
+		mode |= NVC0_3D_CLEAR_BUFFERS_Z;
+	}
+
+	if (clear_flags & PIPE_CLEAR_STENCIL) {
+		BEGIN_RING(chan, RING_3D(CLEAR_STENCIL), 1);
+		OUT_RING  (chan, stencil & 0xff);
+		mode |= NVC0_3D_CLEAR_BUFFERS_S;
+	}
+
+	if (MARK_RING(chan, 17, 2))
+		return;
+
+	BEGIN_RING(chan, RING_3D(ZETA_ADDRESS_HIGH), 5);
+	OUT_RELOCh(chan, bo, sf->offset, NOUVEAU_BO_VRAM | NOUVEAU_BO_WR);
+	OUT_RELOCl(chan, bo, sf->offset, NOUVEAU_BO_VRAM | NOUVEAU_BO_WR);
+	OUT_RING  (chan, nvc0_format_table[dst->format].rt);
+	OUT_RING  (chan, mt->level[sf->base.u.tex.level].tile_mode);
+	OUT_RING  (chan, 0);
+	BEGIN_RING(chan, RING_3D(ZETA_ENABLE), 1);
+	OUT_RING  (chan, 1);
+	BEGIN_RING(chan, RING_3D(ZETA_HORIZ), 3);
+	OUT_RING  (chan, sf->width);
+	OUT_RING  (chan, sf->height);
+	OUT_RING  (chan, (1 << 16) | 1);
+
+	BEGIN_RING(chan, RING_3D(VIEWPORT_HORIZ(0)), 2);
+	OUT_RING  (chan, (width << 16) | dstx);
+	OUT_RING  (chan, (height << 16) | dsty);
+
+	BEGIN_RING(chan, RING_3D(CLEAR_BUFFERS), 1);
+	OUT_RING  (chan, mode);
+
+	nv50->dirty |= NVC0_NEW_FRAMEBUFFER;
+}
+
+void
+nvc0_clear(struct pipe_context *pipe, unsigned buffers,
+           const float *rgba, double depth, unsigned stencil)
+{
+   struct nvc0_context *nvc0 = nvc0_context(pipe);
+   struct nouveau_channel *chan = nvc0->screen->base.channel;
+   struct pipe_framebuffer_state *fb = &nvc0->framebuffer;
+   unsigned i;
+   const unsigned dirty = nvc0->dirty;
+   uint32_t mode = 0;
+
+   /* don't need NEW_BLEND, COLOR_MASK doesn't affect CLEAR_BUFFERS */
+   nvc0->dirty &= NVC0_NEW_FRAMEBUFFER;
+   if (!nvc0_state_validate(nvc0))
+      return;
+
+   if (buffers & PIPE_CLEAR_COLOR && fb->nr_cbufs) {
+      BEGIN_RING(chan, RING_3D(CLEAR_COLOR(0)), 4);
+      OUT_RINGf (chan, rgba[0]);
+      OUT_RINGf (chan, rgba[1]);
+      OUT_RINGf (chan, rgba[2]);
+      OUT_RINGf (chan, rgba[3]);
+      mode =
+         NVC0_3D_CLEAR_BUFFERS_R | NVC0_3D_CLEAR_BUFFERS_G |
+         NVC0_3D_CLEAR_BUFFERS_B | NVC0_3D_CLEAR_BUFFERS_A;
+   }
+
+   if (buffers & PIPE_CLEAR_DEPTH) {
+      BEGIN_RING(chan, RING_3D(CLEAR_DEPTH), 1);
+      OUT_RING  (chan, fui(depth));
+      mode |= NVC0_3D_CLEAR_BUFFERS_Z;
+   }
+
+   if (buffers & PIPE_CLEAR_STENCIL) {
+      BEGIN_RING(chan, RING_3D(CLEAR_STENCIL), 1);
+      OUT_RING  (chan, stencil & 0xff);
+      mode |= NVC0_3D_CLEAR_BUFFERS_S;
+   }
+
+   BEGIN_RING(chan, RING_3D(CLEAR_BUFFERS), 1);
+   OUT_RING  (chan, mode);
+
+   for (i = 1; i < fb->nr_cbufs; i++) {
+      BEGIN_RING(chan, RING_3D(CLEAR_BUFFERS), 1);
+      OUT_RING  (chan, (i << 6) | 0x3c);
+   }
+
+   nvc0->dirty = dirty & ~NVC0_NEW_FRAMEBUFFER;
+}
+
+void
+nvc0_init_surface_functions(struct nvc0_context *nvc0)
+{
+	nvc0->pipe.resource_copy_region = nvc0_resource_copy_region;
+	nvc0->pipe.clear_render_target = nvc0_clear_render_target;
+	nvc0->pipe.clear_depth_stencil = nvc0_clear_depth_stencil;
+}
+
+
diff --git a/src/gallium/drivers/nvc0/nvc0_tex.c b/src/gallium/drivers/nvc0/nvc0_tex.c
new file mode 100644
index 00000000000..b219f82c903
--- /dev/null
+++ b/src/gallium/drivers/nvc0/nvc0_tex.c
@@ -0,0 +1,277 @@
+/*
+ * Copyright 2008 Ben Skeggs
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF
+ * OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "nvc0_context.h"
+#include "nvc0_resource.h"
+#include "nv50_texture.xml.h"
+
+#include "util/u_format.h"
+
+static INLINE uint32_t
+nv50_tic_swizzle(uint32_t tc, unsigned swz)
+{
+   switch (swz) {
+   case PIPE_SWIZZLE_RED:
+      return (tc & NV50_TIC_0_MAPR__MASK) >> NV50_TIC_0_MAPR__SHIFT;
+   case PIPE_SWIZZLE_GREEN:
+      return (tc & NV50_TIC_0_MAPG__MASK) >> NV50_TIC_0_MAPG__SHIFT;
+   case PIPE_SWIZZLE_BLUE:
+      return (tc & NV50_TIC_0_MAPB__MASK) >> NV50_TIC_0_MAPB__SHIFT;
+   case PIPE_SWIZZLE_ALPHA:
+      return (tc & NV50_TIC_0_MAPA__MASK) >> NV50_TIC_0_MAPA__SHIFT;
+   case PIPE_SWIZZLE_ONE:
+      return NV50_TIC_MAP_ONE;
+   case PIPE_SWIZZLE_ZERO:
+   default:
+      return NV50_TIC_MAP_ZERO;
+   }
+}
+
+struct pipe_sampler_view *
+nvc0_create_sampler_view(struct pipe_context *pipe,
+                         struct pipe_resource *texture,
+                         const struct pipe_sampler_view *templ)
+{
+   const struct util_format_description *desc;
+   uint32_t *tic;
+   uint32_t swz[4];
+   uint32_t depth;
+   struct nvc0_tic_entry *view;
+   struct nvc0_miptree *mt = nvc0_miptree(texture);
+
+   view = MALLOC_STRUCT(nvc0_tic_entry);
+   if (!view)
+      return NULL;
+
+   view->pipe = *templ;
+   view->pipe.reference.count = 1;
+   view->pipe.texture = NULL;
+   view->pipe.context = pipe;
+
+   view->id = -1;
+
+   pipe_resource_reference(&view->pipe.texture, texture);
+
+   tic = &view->tic[0];
+
+   desc = util_format_description(mt->base.base.format);
+
+   /* TIC[0] */
+
+   tic[0] = nvc0_format_table[view->pipe.format].tic;
+
+   swz[0] = nv50_tic_swizzle(tic[0], view->pipe.swizzle_r);
+   swz[1] = nv50_tic_swizzle(tic[0], view->pipe.swizzle_g);
+   swz[2] = nv50_tic_swizzle(tic[0], view->pipe.swizzle_b);
+   swz[3] = nv50_tic_swizzle(tic[0], view->pipe.swizzle_a);
+   tic[0] = (tic[0] & ~NV50_TIC_0_SWIZZLE__MASK) |
+      (swz[0] << NV50_TIC_0_MAPR__SHIFT) |
+      (swz[1] << NV50_TIC_0_MAPG__SHIFT) |
+      (swz[2] << NV50_TIC_0_MAPB__SHIFT) |
+      (swz[3] << NV50_TIC_0_MAPA__SHIFT);
+
+   /* tic[1] = mt->base.bo->offset; */
+   tic[2] = /* mt->base.bo->offset >> 32 */ 0;
+
+   tic[2] |= 0x10001000 | /* NV50_TIC_2_NO_BORDER */ 0x40000000;
+
+   if (desc->colorspace == UTIL_FORMAT_COLORSPACE_SRGB)
+      tic[2] |= NV50_TIC_2_COLORSPACE_SRGB;
+
+   if (mt->base.base.target != PIPE_TEXTURE_RECT)
+      tic[2] |= NV50_TIC_2_NORMALIZED_COORDS;
+
+   tic[2] |=
+      ((mt->base.bo->tile_mode & 0x0f0) << (22 - 4)) |
+      ((mt->base.bo->tile_mode & 0xf00) << (25 - 8));
+
+   depth = MAX2(mt->base.base.array_size, mt->base.base.depth0);
+
+   switch (mt->base.base.target) {
+   case PIPE_TEXTURE_1D:
+      tic[2] |= NV50_TIC_2_TARGET_1D;
+      break;
+   case PIPE_TEXTURE_2D:
+      tic[2] |= NV50_TIC_2_TARGET_2D;
+      break;
+   case PIPE_TEXTURE_RECT:
+      tic[2] |= NV50_TIC_2_TARGET_RECT;
+      break;
+   case PIPE_TEXTURE_3D:
+      tic[2] |= NV50_TIC_2_TARGET_3D;
+      break;
+   case PIPE_TEXTURE_CUBE:
+      depth /= 6;
+      if (depth > 1)
+         tic[2] |= NV50_TIC_2_TARGET_CUBE_ARRAY;
+      else
+         tic[2] |= NV50_TIC_2_TARGET_CUBE;
+      break;
+   case PIPE_TEXTURE_1D_ARRAY:
+      tic[2] |= NV50_TIC_2_TARGET_1D_ARRAY;
+      break;
+   case PIPE_TEXTURE_2D_ARRAY:
+      tic[2] |= NV50_TIC_2_TARGET_2D_ARRAY;
+      break;
+   case PIPE_BUFFER:
+      tic[2] |= NV50_TIC_2_TARGET_BUFFER | /* NV50_TIC_2_LINEAR */ (1 << 18);
+   default:
+      NOUVEAU_ERR("invalid texture target: %d\n", mt->base.base.target);
+      return FALSE;
+   }
+
+   if (mt->base.base.target == PIPE_BUFFER)
+      tic[3] = mt->base.base.width0;
+   else
+      tic[3] = 0x00300000;
+
+   tic[4] = (1 << 31) | mt->base.base.width0;
+
+   tic[5] = mt->base.base.height0 & 0xffff;
+   tic[5] |= depth << 16;
+   tic[5] |= mt->base.base.last_level << 28;
+
+   tic[6] = 0x03000000;
+
+   tic[7] = (view->pipe.u.tex.last_level << 4) | view->pipe.u.tex.first_level;
+
+   return &view->pipe;
+}
+
+static boolean
+nvc0_validate_tic(struct nvc0_context *nvc0, int s)
+{
+   struct nouveau_channel *chan = nvc0->screen->base.channel;
+   struct nouveau_bo *txc = nvc0->screen->txc;
+   unsigned i;
+   boolean need_flush = FALSE;
+
+   for (i = 0; i < nvc0->num_textures[s]; ++i) {
+      struct nvc0_tic_entry *tic = nvc0_tic_entry(nvc0->textures[s][i]);
+      struct nvc0_resource *res;
+
+      if (!tic) {
+         BEGIN_RING(chan, RING_3D(BIND_TIC(s)), 1);
+         OUT_RING  (chan, (i << 1) | 0);
+         continue;
+      }
+      res = &nvc0_miptree(tic->pipe.texture)->base;
+
+      if (tic->id < 0) {
+         tic->id = nvc0_screen_tic_alloc(nvc0->screen, tic);
+
+         MARK_RING (chan, 9 + 8, 4);
+         BEGIN_RING(chan, RING_MF(OFFSET_OUT_HIGH), 2);
+         OUT_RELOCh(chan, txc, tic->id * 32, NOUVEAU_BO_VRAM | NOUVEAU_BO_WR);
+         OUT_RELOCl(chan, txc, tic->id * 32, NOUVEAU_BO_VRAM | NOUVEAU_BO_WR);
+         BEGIN_RING(chan, RING_MF(LINE_LENGTH_IN), 2);
+         OUT_RING  (chan, 32);
+         OUT_RING  (chan, 1);
+         BEGIN_RING(chan, RING_MF(EXEC), 1);
+         OUT_RING  (chan, 0x100111);
+         BEGIN_RING_NI(chan, RING_MF(DATA), 8);
+         OUT_RING  (chan, tic->tic[0]);
+         OUT_RELOCl(chan, res->bo, 0, NOUVEAU_BO_VRAM | NOUVEAU_BO_RD);
+         OUT_RELOC (chan, res->bo, 0, NOUVEAU_BO_VRAM | NOUVEAU_BO_RD |
+                    NOUVEAU_BO_HIGH | NOUVEAU_BO_OR, tic->tic[2], tic->tic[2]);
+         OUT_RINGp (chan, &tic->tic[3], 5);
+
+         need_flush = TRUE;
+      }
+      nvc0->screen->tic.lock[tic->id / 32] |= 1 << (tic->id % 32);
+
+      nvc0_bufctx_add_resident(nvc0, NVC0_BUFCTX_TEXTURES, res,
+                               NOUVEAU_BO_VRAM | NOUVEAU_BO_RD);
+
+      BEGIN_RING(chan, RING_3D(BIND_TIC(s)), 1);
+      OUT_RING  (chan, (tic->id << 9) | (i << 1) | 1);
+   }
+   for (; i < nvc0->state.num_textures[s]; ++i) {
+      BEGIN_RING(chan, RING_3D(BIND_TIC(s)), 1);
+      OUT_RING  (chan, (i << 1) | 0);
+   }
+   nvc0->state.num_textures[s] = nvc0->num_textures[s];
+
+   return need_flush;
+}
+
+void nvc0_validate_textures(struct nvc0_context *nvc0)
+{
+   boolean need_flush;
+
+   need_flush  = nvc0_validate_tic(nvc0, 0);
+   need_flush |= nvc0_validate_tic(nvc0, 4);
+
+   if (need_flush) {
+      BEGIN_RING(nvc0->screen->base.channel, RING_3D(TIC_FLUSH), 1);
+      OUT_RING  (nvc0->screen->base.channel, 0);
+   }
+}
+
+static boolean
+nvc0_validate_tsc(struct nvc0_context *nvc0, int s)
+{
+   struct nouveau_channel *chan = nvc0->screen->base.channel;
+   unsigned i;
+   boolean need_flush = FALSE;
+
+   for (i = 0; i < nvc0->num_samplers[s]; ++i) {
+      struct nvc0_tsc_entry *tsc = nvc0_tsc_entry(nvc0->samplers[s][i]);
+
+      if (!tsc) {
+         BEGIN_RING(chan, RING_3D(BIND_TSC(s)), 1);
+         OUT_RING  (chan, (i << 4) | 0);
+         continue;
+      }
+      if (tsc->id < 0) {
+         tsc->id = nvc0_screen_tsc_alloc(nvc0->screen, tsc);
+
+         nvc0_m2mf_push_linear(nvc0, nvc0->screen->txc, NOUVEAU_BO_VRAM,
+                               65536 + tsc->id * 32, 32, tsc->tsc);
+         need_flush = TRUE;
+      }
+      nvc0->screen->tsc.lock[tsc->id / 32] |= 1 << (tsc->id % 32);
+
+      BEGIN_RING(chan, RING_3D(BIND_TSC(s)), 1);
+      OUT_RING  (chan, (tsc->id << 12) | (i << 4) | 1);
+   }
+   for (; i < nvc0->state.num_samplers[s]; ++i) {
+      BEGIN_RING(chan, RING_3D(BIND_TSC(s)), 1);
+      OUT_RING  (chan, (i << 4) | 0);
+   }
+   nvc0->state.num_samplers[s] = nvc0->num_samplers[s];
+
+   return need_flush;
+}
+
+void nvc0_validate_samplers(struct nvc0_context *nvc0)
+{
+   boolean need_flush;
+
+   need_flush  = nvc0_validate_tsc(nvc0, 0);
+   need_flush |= nvc0_validate_tsc(nvc0, 4);
+
+   if (need_flush) {
+      BEGIN_RING(nvc0->screen->base.channel, RING_3D(TSC_FLUSH), 1);
+      OUT_RING  (nvc0->screen->base.channel, 0);
+   }
+}
diff --git a/src/gallium/drivers/nvc0/nvc0_tgsi_to_nc.c b/src/gallium/drivers/nvc0/nvc0_tgsi_to_nc.c
new file mode 100644
index 00000000000..fecfc76fb79
--- /dev/null
+++ b/src/gallium/drivers/nvc0/nvc0_tgsi_to_nc.c
@@ -0,0 +1,2004 @@
+/*
+ * Copyright 2010 Christoph Bumiller
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF
+ * OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <unistd.h>
+
+#define NOUVEAU_DEBUG 1
+
+#include "pipe/p_shader_tokens.h"
+#include "tgsi/tgsi_parse.h"
+#include "tgsi/tgsi_util.h"
+#include "tgsi/tgsi_dump.h"
+#include "util/u_dynarray.h"
+
+#include "nvc0_pc.h"
+#include "nvc0_program.h"
+
+/* Arbitrary internal limits. */
+#define BLD_MAX_TEMPS 64
+#define BLD_MAX_ADDRS 4
+#define BLD_MAX_PREDS 4
+#define BLD_MAX_IMMDS 128
+#define BLD_MAX_OUTPS PIPE_MAX_SHADER_OUTPUTS
+
+#define BLD_MAX_COND_NESTING 8
+#define BLD_MAX_LOOP_NESTING 4
+#define BLD_MAX_CALL_NESTING 2
+
+/* This structure represents a TGSI register. */
+struct bld_register {
+   struct nv_value *current;
+   /* collect all SSA values assigned to it */
+   struct util_dynarray vals;
+   /* 1 bit per loop level, indicates if used/defd, reset when loop ends */
+   uint16_t loop_use;
+   uint16_t loop_def;
+};
+
+static INLINE struct nv_value **
+bld_register_access(struct bld_register *reg, unsigned i)
+{
+   return util_dynarray_element(&reg->vals, struct nv_value *, i);
+}
+
+static INLINE void
+bld_register_add_val(struct bld_register *reg, struct nv_value *val)
+{
+   util_dynarray_append(&reg->vals, struct nv_value *, val);
+}
+
+static INLINE boolean
+bld_register_del_val(struct bld_register *reg, struct nv_value *val)
+{
+   unsigned i;
+
+   for (i = reg->vals.size / sizeof(struct nv_value *); i > 0; --i)
+      if (*bld_register_access(reg, i - 1) == val)
+         break;
+   if (!i)
+      return FALSE;
+
+   if (i != reg->vals.size / sizeof(struct nv_value *))
+      *bld_register_access(reg, i - 1) = util_dynarray_pop(&reg->vals,
+                                                           struct nv_value *);
+   else
+      reg->vals.size -= sizeof(struct nv_value *);
+
+   return TRUE;
+}
+
+struct bld_context {
+   struct nvc0_translation_info *ti;
+
+   struct nv_pc *pc;
+   struct nv_basic_block *b;
+
+   struct tgsi_parse_context parse[BLD_MAX_CALL_NESTING];
+   int call_lvl;
+
+   struct nv_basic_block *cond_bb[BLD_MAX_COND_NESTING];
+   struct nv_basic_block *join_bb[BLD_MAX_COND_NESTING];
+   struct nv_basic_block *else_bb[BLD_MAX_COND_NESTING];
+   int cond_lvl;
+   struct nv_basic_block *loop_bb[BLD_MAX_LOOP_NESTING];
+   struct nv_basic_block *brkt_bb[BLD_MAX_LOOP_NESTING];
+   int loop_lvl;
+
+   ubyte out_kind; /* CFG_EDGE_FORWARD, or FAKE in case of BREAK/CONT */
+
+   struct bld_register tvs[BLD_MAX_TEMPS][4]; /* TGSI_FILE_TEMPORARY */
+   struct bld_register avs[BLD_MAX_ADDRS][4]; /* TGSI_FILE_ADDRESS */
+   struct bld_register pvs[BLD_MAX_PREDS][4]; /* TGSI_FILE_PREDICATE */
+   struct bld_register ovs[BLD_MAX_OUTPS][4]; /* TGSI_FILE_OUTPUT, FP only */
+
+   uint32_t outputs_written[(PIPE_MAX_SHADER_OUTPUTS + 7) / 8];
+   int hpos_index;
+
+   struct nv_value *zero;
+   struct nv_value *frag_coord[4];
+
+   /* wipe on new BB */
+   struct nv_value *saved_sysvals[4];
+   struct nv_value *saved_addr[4][2];
+   struct nv_value *saved_inputs[PIPE_MAX_SHADER_INPUTS][4];
+   struct nv_value *saved_immd[BLD_MAX_IMMDS];
+   uint num_immds;
+};
+
+static INLINE ubyte
+bld_register_file(struct bld_context *bld, struct bld_register *reg)
+{
+   if (reg < &bld->avs[0][0]) return NV_FILE_GPR;
+   else
+   if (reg < &bld->pvs[0][0]) return NV_FILE_GPR;
+   else
+   if (reg < &bld->ovs[0][0]) return NV_FILE_PRED;
+   else
+      return NV_FILE_MEM_V;
+}
+
+static INLINE struct nv_value *
+bld_fetch(struct bld_context *bld, struct bld_register *regs, int i, int c)
+{
+   regs[i * 4 + c].loop_use |= 1 << bld->loop_lvl;
+   return regs[i * 4 + c].current;
+}
+
+static struct nv_value *
+bld_loop_phi(struct bld_context *, struct bld_register *, struct nv_value *);
+
+/* If a variable is defined in a loop without prior use, we don't need
+ * a phi in the loop header to account for backwards flow.
+ *
+ * However, if this variable is then also used outside the loop, we do
+ * need a phi after all. But we must not use this phi's def inside the
+ * loop, so we can eliminate the phi if it is unused later.
+ */
+static INLINE void
+bld_store(struct bld_context *bld,
+          struct bld_register *regs, int i, int c, struct nv_value *val)
+{
+   const uint16_t m = 1 << bld->loop_lvl;
+   struct bld_register *reg = &regs[i * 4 + c];
+
+   if (bld->loop_lvl && !(m & (reg->loop_def | reg->loop_use)))
+      bld_loop_phi(bld, reg, val);
+
+   reg->current = val;
+   bld_register_add_val(reg, reg->current);
+
+   reg->loop_def |= 1 << bld->loop_lvl;
+}
+
+#define FETCH_TEMP(i, c)    bld_fetch(bld, &bld->tvs[0][0], i, c)
+#define STORE_TEMP(i, c, v) bld_store(bld, &bld->tvs[0][0], i, c, (v))
+#define FETCH_ADDR(i, c)    bld_fetch(bld, &bld->avs[0][0], i, c)
+#define STORE_ADDR(i, c, v) bld_store(bld, &bld->avs[0][0], i, c, (v))
+#define FETCH_PRED(i, c)    bld_fetch(bld, &bld->pvs[0][0], i, c)
+#define STORE_PRED(i, c, v) bld_store(bld, &bld->pvs[0][0], i, c, (v))
+#define STORE_OUTP(i, c, v)                                         \
+   do {                                                             \
+      bld_store(bld, &bld->ovs[0][0], i, c, (v));                   \
+      bld->outputs_written[(i) / 8] |= 1 << (((i) * 4 + (c)) % 32); \
+   } while (0)
+
+static INLINE void
+bld_clear_def_use(struct bld_register *regs, int n, int lvl)
+{
+   int i;
+   const uint16_t mask = ~(1 << lvl);
+
+   for (i = 0; i < n * 4; ++i) {
+      regs[i].loop_def &= mask;
+      regs[i].loop_use &= mask;
+   }
+}
+
+static INLINE void
+bld_warn_uninitialized(struct bld_context *bld, int kind,
+                       struct bld_register *reg, struct nv_basic_block *b)
+{
+#ifdef NOUVEAU_DEBUG
+   long i = (reg - &bld->tvs[0][0]) / 4;
+   long c = (reg - &bld->tvs[0][0]) & 3;
+
+   if (c == 3)
+      c = -1;
+   debug_printf("WARNING: TEMP[%li].%c %s used uninitialized in BB:%i\n",
+                i, (int)('x' + c), kind ? "may be" : "is", b->id);
+#endif
+}
+
+static INLINE struct nv_value *
+bld_def(struct nv_instruction *i, int c, struct nv_value *value)
+{
+   i->def[c] = value;
+   value->insn = i;
+   return value;
+}
+
+static INLINE struct nv_value *
+find_by_bb(struct bld_register *reg, struct nv_basic_block *b)
+{
+   int i;
+
+   if (reg->current && reg->current->insn->bb == b)
+      return reg->current;
+
+   for (i = 0; i < reg->vals.size / sizeof(struct nv_value *); ++i)
+      if ((*bld_register_access(reg, i))->insn->bb == b)
+         return *bld_register_access(reg, i);
+   return NULL;
+}
+
+/* Fetch value from register that was defined in the specified BB,
+ * or search for first definitions in all of its predecessors.
+ */
+static void
+fetch_by_bb(struct bld_register *reg,
+            struct nv_value **vals, int *n,
+            struct nv_basic_block *b)
+{
+   int i;
+   struct nv_value *val;
+
+   assert(*n < 16); /* MAX_COND_NESTING */
+
+   val = find_by_bb(reg, b);
+   if (val) {
+      for (i = 0; i < *n; ++i)
+         if (vals[i] == val)
+            return;
+      vals[(*n)++] = val;
+      return;
+   }
+   for (i = 0; i < b->num_in; ++i)
+      if (!IS_WALL_EDGE(b->in_kind[i]))
+         fetch_by_bb(reg, vals, n, b->in[i]);
+}
+
+static INLINE struct nv_value *
+bld_load_imm_u32(struct bld_context *bld, uint32_t u);
+
+static INLINE struct nv_value *
+bld_undef(struct bld_context *bld, ubyte file)
+{
+   struct nv_instruction *nvi = new_instruction(bld->pc, NV_OP_UNDEF);
+
+   return bld_def(nvi, 0, new_value(bld->pc, file, 4));
+}
+
+static struct nv_value *
+bld_phi(struct bld_context *bld, struct nv_basic_block *b,
+        struct bld_register *reg)
+{
+   struct nv_basic_block *in;
+   struct nv_value *vals[16] = { NULL };
+   struct nv_value *val;
+   struct nv_instruction *phi;
+   int i, j, n;
+
+   do {
+      i = n = 0;
+      fetch_by_bb(reg, vals, &n, b);
+
+      if (!n) {
+         bld_warn_uninitialized(bld, 0, reg, b);
+         return NULL;
+      }
+
+      if (n == 1) {
+         if (nvc0_bblock_dominated_by(b, vals[0]->insn->bb))
+            break;
+
+         bld_warn_uninitialized(bld, 1, reg, b);
+
+         /* back-tracking to insert missing value of other path */
+         in = b;
+         while (in->in[0]) {
+            if (in->num_in == 1) {
+               in = in->in[0];
+            } else {
+               if (!nvc0_bblock_reachable_by(in->in[0], vals[0]->insn->bb, b))
+                  in = in->in[0];
+               else
+               if (!nvc0_bblock_reachable_by(in->in[1], vals[0]->insn->bb, b))
+                  in = in->in[1];
+               else
+                  in = in->in[0];
+            }
+         }
+         bld->pc->current_block = in;
+
+         /* should make this a no-op */
+         bld_register_add_val(reg, bld_undef(bld, vals[0]->reg.file));
+         continue;
+      }
+
+      for (i = 0; i < n; ++i) {
+         /* if value dominates b, continue to the redefinitions */
+         if (nvc0_bblock_dominated_by(b, vals[i]->insn->bb))
+            continue;
+
+         /* if value dominates any in-block, b should be the dom frontier */
+         for (j = 0; j < b->num_in; ++j)
+            if (nvc0_bblock_dominated_by(b->in[j], vals[i]->insn->bb))
+               break;
+         /* otherwise, find the dominance frontier and put the phi there */
+         if (j == b->num_in) {
+            in = nvc0_bblock_dom_frontier(vals[i]->insn->bb);
+            val = bld_phi(bld, in, reg);
+            bld_register_add_val(reg, val);
+            break;
+         }
+      }
+   } while(i < n);
+
+   bld->pc->current_block = b;
+
+   if (n == 1)
+      return vals[0];
+
+   phi = new_instruction(bld->pc, NV_OP_PHI);
+
+   bld_def(phi, 0, new_value(bld->pc, vals[0]->reg.file, vals[0]->reg.size));
+   for (i = 0; i < n; ++i)
+      nv_reference(bld->pc, phi, i, vals[i]);
+
+   return phi->def[0];
+}
+
+/* Insert a phi function in the loop header.
+ * For nested loops, we need to insert phi functions in all the outer
+ * loop headers if they don't have one yet.
+ *
+ * @def: redefinition from inside loop, or NULL if to be replaced later
+ */
+static struct nv_value *
+bld_loop_phi(struct bld_context *bld, struct bld_register *reg,
+             struct nv_value *def)
+{
+   struct nv_instruction *phi;
+   struct nv_basic_block *bb = bld->pc->current_block;
+   struct nv_value *val = NULL;
+
+   if (bld->loop_lvl > 1) {
+      --bld->loop_lvl;
+      if (!((reg->loop_def | reg->loop_use) & (1 << bld->loop_lvl)))
+         val = bld_loop_phi(bld, reg, NULL);
+      ++bld->loop_lvl;
+   }
+
+   if (!val)
+      val = bld_phi(bld, bld->pc->current_block, reg); /* old definition */
+   if (!val) {
+      bld->pc->current_block = bld->loop_bb[bld->loop_lvl - 1]->in[0];
+      val = bld_undef(bld, bld_register_file(bld, reg));
+   }
+
+   bld->pc->current_block = bld->loop_bb[bld->loop_lvl - 1];
+
+   phi = new_instruction(bld->pc, NV_OP_PHI);
+
+   bld_def(phi, 0, new_value_like(bld->pc, val));
+   if (!def)
+      def = phi->def[0];
+
+   bld_register_add_val(reg, phi->def[0]);
+
+   phi->target = (struct nv_basic_block *)reg; /* cheat */
+
+   nv_reference(bld->pc, phi, 0, val);
+   nv_reference(bld->pc, phi, 1, def);
+
+   bld->pc->current_block = bb;
+
+   return phi->def[0];
+}
+
+static INLINE struct nv_value *
+bld_fetch_global(struct bld_context *bld, struct bld_register *reg)
+{
+   const uint16_t m = 1 << bld->loop_lvl;
+   const uint16_t use = reg->loop_use;
+
+   reg->loop_use |= m;
+
+   /* If neither used nor def'd inside the loop, build a phi in foresight,
+    * so we don't have to replace stuff later on, which requires tracking.
+    */
+   if (bld->loop_lvl && !((use | reg->loop_def) & m))
+      return bld_loop_phi(bld, reg, NULL);
+
+   return bld_phi(bld, bld->pc->current_block, reg);
+}
+
+static INLINE struct nv_value *
+bld_imm_u32(struct bld_context *bld, uint32_t u)
+{
+   int i;
+   unsigned n = bld->num_immds;
+
+   for (i = 0; i < n; ++i)
+      if (bld->saved_immd[i]->reg.imm.u32 == u)
+         return bld->saved_immd[i];
+
+   assert(n < BLD_MAX_IMMDS);
+   bld->num_immds++;
+
+   bld->saved_immd[n] = new_value(bld->pc, NV_FILE_IMM, 4);
+   bld->saved_immd[n]->reg.imm.u32 = u;
+   return bld->saved_immd[n];
+}
+
+static void
+bld_replace_value(struct nv_pc *, struct nv_basic_block *, struct nv_value *,
+                  struct nv_value *);
+
+/* Replace the source of the phi in the loop header by the last assignment,
+ * or eliminate the phi function if there is no assignment inside the loop.
+ *
+ * Redundancy situation 1 - (used) but (not redefined) value:
+ *  %3 = phi %0, %3 = %3 is used
+ *  %3 = phi %0, %4 = is new definition
+ *
+ * Redundancy situation 2 - (not used) but (redefined) value:
+ *  %3 = phi %0, %2 = %2 is used, %3 could be used outside, deleted by DCE
+ */
+static void
+bld_loop_end(struct bld_context *bld, struct nv_basic_block *bb)
+{
+   struct nv_basic_block *save = bld->pc->current_block;
+   struct nv_instruction *phi, *next;
+   struct nv_value *val;
+   struct bld_register *reg;
+   int i, s, n;
+
+   for (phi = bb->phi; phi && phi->opcode == NV_OP_PHI; phi = next) {
+      next = phi->next;
+
+      reg = (struct bld_register *)phi->target;
+      phi->target = NULL;
+
+      for (s = 1, n = 0; n < bb->num_in; ++n) {
+         if (bb->in_kind[n] != CFG_EDGE_BACK)
+            continue;
+
+         assert(s < 4);
+         bld->pc->current_block = bb->in[n];
+         val = bld_fetch_global(bld, reg);
+
+         for (i = 0; i < 4; ++i)
+            if (phi->src[i] && phi->src[i]->value == val)
+               break;
+         if (i == 4)
+            nv_reference(bld->pc, phi, s++, val);
+      }
+      bld->pc->current_block = save;
+
+      if (phi->src[0]->value == phi->def[0] ||
+          phi->src[0]->value == phi->src[1]->value)
+         s = 1;
+      else
+      if (phi->src[1]->value == phi->def[0])
+         s = 0;
+      else
+         continue;
+
+      if (s >= 0) {
+         /* eliminate the phi */
+         bld_register_del_val(reg, phi->def[0]);
+
+         ++bld->pc->pass_seq;
+         bld_replace_value(bld->pc, bb, phi->def[0], phi->src[s]->value);
+
+         nvc0_insn_delete(phi);
+      }
+   }
+}
+
+static INLINE struct nv_value *
+bld_imm_f32(struct bld_context *bld, float f)
+{
+   return bld_imm_u32(bld, fui(f));
+}
+
+static struct nv_value *
+bld_insn_1(struct bld_context *bld, uint opcode, struct nv_value *src0)
+{
+   struct nv_instruction *insn = new_instruction(bld->pc, opcode);
+
+   nv_reference(bld->pc, insn, 0, src0);
+   
+   return bld_def(insn, 0, new_value(bld->pc, NV_FILE_GPR, src0->reg.size));
+}
+
+static struct nv_value *
+bld_insn_2(struct bld_context *bld, uint opcode,
+           struct nv_value *src0, struct nv_value *src1)
+{
+   struct nv_instruction *insn = new_instruction(bld->pc, opcode);
+
+   nv_reference(bld->pc, insn, 0, src0);
+   nv_reference(bld->pc, insn, 1, src1);
+
+   return bld_def(insn, 0, new_value(bld->pc, NV_FILE_GPR, src0->reg.size));
+}
+
+static struct nv_value *
+bld_insn_3(struct bld_context *bld, uint opcode,
+           struct nv_value *src0, struct nv_value *src1,
+           struct nv_value *src2)
+{
+   struct nv_instruction *insn = new_instruction(bld->pc, opcode);
+
+   nv_reference(bld->pc, insn, 0, src0);
+   nv_reference(bld->pc, insn, 1, src1);
+   nv_reference(bld->pc, insn, 2, src2);
+
+   return bld_def(insn, 0, new_value(bld->pc, NV_FILE_GPR, src0->reg.size));
+}
+
+static INLINE void
+bld_src_predicate(struct bld_context *bld,
+                  struct nv_instruction *nvi, int s, struct nv_value *val)
+{
+   nvi->predicate = s;
+   nv_reference(bld->pc, nvi, s, val);
+}
+
+static INLINE void
+bld_src_pointer(struct bld_context *bld,
+                struct nv_instruction *nvi, int s, struct nv_value *val)
+{
+   nvi->indirect = s;
+   nv_reference(bld->pc, nvi, s, val);
+}
+
+static void
+bld_lmem_store(struct bld_context *bld, struct nv_value *ptr, int ofst,
+               struct nv_value *val)
+{
+   struct nv_instruction *insn = new_instruction(bld->pc, NV_OP_ST);
+   struct nv_value *loc;
+
+   loc = new_value(bld->pc, NV_FILE_MEM_L, nv_type_sizeof(NV_TYPE_U32));
+
+   loc->reg.id = ofst * 4;
+
+   nv_reference(bld->pc, insn, 0, loc);
+   nv_reference(bld->pc, insn, 1, ptr);
+   nv_reference(bld->pc, insn, 2, val);
+}
+
+static struct nv_value *
+bld_lmem_load(struct bld_context *bld, struct nv_value *ptr, int ofst)
+{
+   struct nv_value *loc, *val;
+
+   loc = new_value(bld->pc, NV_FILE_MEM_L, nv_type_sizeof(NV_TYPE_U32));
+
+   loc->reg.address = ofst * 4;
+
+   val = bld_insn_2(bld, NV_OP_LD, loc, ptr);
+
+   return val;
+}
+
+static struct nv_value *
+bld_pow(struct bld_context *bld, struct nv_value *x, struct nv_value *e)
+{
+   struct nv_value *val;
+
+   val = bld_insn_1(bld, NV_OP_LG2, x);
+   val = bld_insn_2(bld, NV_OP_MUL_F32, e, val);
+
+   val = bld_insn_1(bld, NV_OP_PREEX2, val);
+   val = bld_insn_1(bld, NV_OP_EX2, val);
+
+   return val;
+}
+
+static INLINE struct nv_value *
+bld_load_imm_f32(struct bld_context *bld, float f)
+{
+   if (f == 0.0f)
+      return bld->zero;
+   return bld_insn_1(bld, NV_OP_MOV, bld_imm_f32(bld, f));
+}
+
+static INLINE struct nv_value *
+bld_load_imm_u32(struct bld_context *bld, uint32_t u)
+{
+   if (u == 0)
+      return bld->zero;
+   return bld_insn_1(bld, NV_OP_MOV, bld_imm_u32(bld, u));
+}
+
+static INLINE struct nv_value *
+bld_setp(struct bld_context *bld, uint op, uint8_t cc,
+         struct nv_value *src0, struct nv_value *src1)
+{
+   struct nv_value *val = bld_insn_2(bld, op, src0, src1);
+
+   val->reg.file = NV_FILE_PRED;
+   val->reg.size = 1;
+   val->insn->set_cond = cc & 0xf;
+   return val;
+}
+
+static INLINE struct nv_value *
+bld_cvt(struct bld_context *bld, uint8_t dt, uint8_t st, struct nv_value *src)
+{
+   struct nv_value *val = bld_insn_1(bld, NV_OP_CVT, src);
+   val->insn->ext.cvt.d = dt;
+   val->insn->ext.cvt.s = st;
+   return val;
+}
+
+static void
+bld_kil(struct bld_context *bld, struct nv_value *src)
+{
+   struct nv_instruction *nvi;
+
+   src = bld_setp(bld, NV_OP_SET_F32, NV_CC_LT, src, bld->zero);
+
+   nvi = new_instruction(bld->pc, NV_OP_KIL);
+   nvi->fixed = 1;
+
+   bld_src_predicate(bld, nvi, 0, src);
+}
+
+static void
+bld_flow(struct bld_context *bld, uint opcode,
+         struct nv_value *src, struct nv_basic_block *target,
+         boolean reconverge)
+{
+   struct nv_instruction *nvi;
+
+   if (reconverge)
+      new_instruction(bld->pc, NV_OP_JOINAT)->fixed = 1;
+
+   nvi = new_instruction(bld->pc, opcode);
+   nvi->target = target;
+   nvi->terminator = 1;
+   if (src)
+      bld_src_predicate(bld, nvi, 0, src);
+}
+
+static ubyte
+translate_setcc(unsigned opcode)
+{
+   switch (opcode) {
+   case TGSI_OPCODE_SLT: return NV_CC_LT;
+   case TGSI_OPCODE_SGE: return NV_CC_GE;
+   case TGSI_OPCODE_SEQ: return NV_CC_EQ;
+   case TGSI_OPCODE_SGT: return NV_CC_GT;
+   case TGSI_OPCODE_SLE: return NV_CC_LE;
+   case TGSI_OPCODE_SNE: return NV_CC_NE | NV_CC_U;
+   case TGSI_OPCODE_STR: return NV_CC_TR;
+   case TGSI_OPCODE_SFL: return NV_CC_FL;
+
+   case TGSI_OPCODE_ISLT: return NV_CC_LT;
+   case TGSI_OPCODE_ISGE: return NV_CC_GE;
+   case TGSI_OPCODE_USEQ: return NV_CC_EQ;
+   case TGSI_OPCODE_USGE: return NV_CC_GE;
+   case TGSI_OPCODE_USLT: return NV_CC_LT;
+   case TGSI_OPCODE_USNE: return NV_CC_NE;
+   default:
+      assert(0);
+      return NV_CC_FL;
+   }
+}
+
+static uint
+translate_opcode(uint opcode)
+{
+   switch (opcode) {
+   case TGSI_OPCODE_ABS: return NV_OP_ABS_F32;
+   case TGSI_OPCODE_ADD: return NV_OP_ADD_F32;
+   case TGSI_OPCODE_SUB: return NV_OP_SUB_F32;
+   case TGSI_OPCODE_UADD: return NV_OP_ADD_B32;
+   case TGSI_OPCODE_AND: return NV_OP_AND;
+   case TGSI_OPCODE_EX2: return NV_OP_EX2;
+   case TGSI_OPCODE_CEIL: return NV_OP_CEIL;
+   case TGSI_OPCODE_FLR: return NV_OP_FLOOR;
+   case TGSI_OPCODE_TRUNC: return NV_OP_TRUNC;
+   case TGSI_OPCODE_COS: return NV_OP_COS;
+   case TGSI_OPCODE_SIN: return NV_OP_SIN;
+   case TGSI_OPCODE_DDX: return NV_OP_DFDX;
+   case TGSI_OPCODE_DDY: return NV_OP_DFDY;
+   case TGSI_OPCODE_F2I:
+   case TGSI_OPCODE_F2U:
+   case TGSI_OPCODE_I2F:
+   case TGSI_OPCODE_U2F: return NV_OP_CVT;
+   case TGSI_OPCODE_INEG: return NV_OP_NEG_S32;
+   case TGSI_OPCODE_LG2: return NV_OP_LG2;
+   case TGSI_OPCODE_ISHR: return NV_OP_SAR;
+   case TGSI_OPCODE_USHR: return NV_OP_SHR;
+   case TGSI_OPCODE_MAD: return NV_OP_MAD_F32;
+   case TGSI_OPCODE_MAX: return NV_OP_MAX_F32;
+   case TGSI_OPCODE_IMAX: return NV_OP_MAX_S32;
+   case TGSI_OPCODE_UMAX: return NV_OP_MAX_U32;
+   case TGSI_OPCODE_MIN: return NV_OP_MIN_F32;
+   case TGSI_OPCODE_IMIN: return NV_OP_MIN_S32;
+   case TGSI_OPCODE_UMIN: return NV_OP_MIN_U32;
+   case TGSI_OPCODE_MUL: return NV_OP_MUL_F32;
+   case TGSI_OPCODE_UMUL: return NV_OP_MUL_B32;
+   case TGSI_OPCODE_OR: return NV_OP_OR;
+   case TGSI_OPCODE_RCP: return NV_OP_RCP;
+   case TGSI_OPCODE_RSQ: return NV_OP_RSQ;
+   case TGSI_OPCODE_SAD: return NV_OP_SAD;
+   case TGSI_OPCODE_SHL: return NV_OP_SHL;
+   case TGSI_OPCODE_SLT:
+   case TGSI_OPCODE_SGE:
+   case TGSI_OPCODE_SEQ:
+   case TGSI_OPCODE_SGT:
+   case TGSI_OPCODE_SLE:
+   case TGSI_OPCODE_SNE: return NV_OP_FSET_F32;
+   case TGSI_OPCODE_ISLT:
+   case TGSI_OPCODE_ISGE: return NV_OP_SET_S32;
+   case TGSI_OPCODE_USEQ:
+   case TGSI_OPCODE_USGE:
+   case TGSI_OPCODE_USLT:
+   case TGSI_OPCODE_USNE: return NV_OP_SET_U32;
+   case TGSI_OPCODE_TEX: return NV_OP_TEX;
+   case TGSI_OPCODE_TXP: return NV_OP_TEX;
+   case TGSI_OPCODE_TXB: return NV_OP_TXB;
+   case TGSI_OPCODE_TXL: return NV_OP_TXL;
+   case TGSI_OPCODE_XOR: return NV_OP_XOR;
+   default:
+      return NV_OP_NOP;
+   }
+}
+
+#if 0
+static ubyte
+infer_src_type(unsigned opcode)
+{
+   switch (opcode) {
+   case TGSI_OPCODE_MOV:
+   case TGSI_OPCODE_AND:
+   case TGSI_OPCODE_OR:
+   case TGSI_OPCODE_XOR:
+   case TGSI_OPCODE_SAD:
+   case TGSI_OPCODE_U2F:
+   case TGSI_OPCODE_UADD:
+   case TGSI_OPCODE_UDIV:
+   case TGSI_OPCODE_UMOD:
+   case TGSI_OPCODE_UMAD:
+   case TGSI_OPCODE_UMUL:
+   case TGSI_OPCODE_UMAX:
+   case TGSI_OPCODE_UMIN:
+   case TGSI_OPCODE_USEQ:
+   case TGSI_OPCODE_USGE:
+   case TGSI_OPCODE_USLT:
+   case TGSI_OPCODE_USNE:
+   case TGSI_OPCODE_USHR:
+      return NV_TYPE_U32;
+   case TGSI_OPCODE_I2F:
+   case TGSI_OPCODE_IDIV:
+   case TGSI_OPCODE_IMAX:
+   case TGSI_OPCODE_IMIN:
+   case TGSI_OPCODE_INEG:
+   case TGSI_OPCODE_ISGE:
+   case TGSI_OPCODE_ISHR:
+   case TGSI_OPCODE_ISLT:
+      return NV_TYPE_S32;
+   default:
+      return NV_TYPE_F32;
+   }
+}
+
+static ubyte
+infer_dst_type(unsigned opcode)
+{
+   switch (opcode) {
+   case TGSI_OPCODE_MOV:
+   case TGSI_OPCODE_F2U:
+   case TGSI_OPCODE_AND:
+   case TGSI_OPCODE_OR:
+   case TGSI_OPCODE_XOR:
+   case TGSI_OPCODE_SAD:
+   case TGSI_OPCODE_UADD:
+   case TGSI_OPCODE_UDIV:
+   case TGSI_OPCODE_UMOD:
+   case TGSI_OPCODE_UMAD:
+   case TGSI_OPCODE_UMUL:
+   case TGSI_OPCODE_UMAX:
+   case TGSI_OPCODE_UMIN:
+   case TGSI_OPCODE_USEQ:
+   case TGSI_OPCODE_USGE:
+   case TGSI_OPCODE_USLT:
+   case TGSI_OPCODE_USNE:
+   case TGSI_OPCODE_USHR:
+      return NV_TYPE_U32;
+   case TGSI_OPCODE_F2I:
+   case TGSI_OPCODE_IDIV:
+   case TGSI_OPCODE_IMAX:
+   case TGSI_OPCODE_IMIN:
+   case TGSI_OPCODE_INEG:
+   case TGSI_OPCODE_ISGE:
+   case TGSI_OPCODE_ISHR:
+   case TGSI_OPCODE_ISLT:
+      return NV_TYPE_S32;
+   default:
+      return NV_TYPE_F32;
+   }
+}
+#endif
+
+static void
+emit_store(struct bld_context *bld, const struct tgsi_full_instruction *inst,
+           unsigned chan, struct nv_value *res)
+{
+   const struct tgsi_full_dst_register *reg = &inst->Dst[0];
+   struct nv_instruction *nvi;
+   struct nv_value *mem;
+   struct nv_value *ptr = NULL;
+   int idx;
+
+   idx = reg->Register.Index;
+   assert(chan < 4);
+
+   if (reg->Register.Indirect)
+      ptr = FETCH_ADDR(reg->Indirect.Index,
+                       tgsi_util_get_src_register_swizzle(&reg->Indirect, 0));
+
+   switch (inst->Instruction.Saturate) {
+   case TGSI_SAT_NONE:
+      break;
+   case TGSI_SAT_ZERO_ONE:
+      res = bld_insn_1(bld, NV_OP_SAT, res);
+      break;
+   case TGSI_SAT_MINUS_PLUS_ONE:
+      res = bld_insn_2(bld, NV_OP_MAX_F32, res, bld_load_imm_f32(bld, -1.0f));
+      res = bld_insn_2(bld, NV_OP_MIN_F32, res, bld_load_imm_f32(bld, 1.0f));
+      break;
+   }
+
+   switch (reg->Register.File) {
+   case TGSI_FILE_OUTPUT:
+      if (!res->insn)
+         res = bld_insn_1(bld, NV_OP_MOV, res);
+
+      if (bld->pc->is_fragprog) {
+         assert(!ptr);
+         STORE_OUTP(idx, chan, res);
+      } else {
+         nvi = new_instruction(bld->pc, NV_OP_EXPORT);
+         mem = new_value(bld->pc, bld->ti->output_file, res->reg.size);
+         nv_reference(bld->pc, nvi, 0, mem);
+         nv_reference(bld->pc, nvi, 1, res);
+         if (!ptr)
+            mem->reg.address = bld->ti->output_loc[idx][chan];
+         else
+            mem->reg.address = 0x80 + idx * 16 + chan * 4;
+         nvi->fixed = 1;
+      }
+      break;
+   case TGSI_FILE_TEMPORARY:
+      assert(idx < BLD_MAX_TEMPS);
+      if (!res->insn)
+         res = bld_insn_1(bld, NV_OP_MOV, res);
+
+      assert(res->reg.file == NV_FILE_GPR);
+      assert(res->insn->bb = bld->pc->current_block);
+
+      if (bld->ti->require_stores)
+         bld_lmem_store(bld, ptr, idx * 4 + chan, res);
+      else
+         STORE_TEMP(idx, chan, res);
+      break;
+   case TGSI_FILE_ADDRESS:
+      assert(idx < BLD_MAX_ADDRS);
+      STORE_ADDR(idx, chan, res);
+      break;
+   }
+}
+
+static INLINE uint32_t
+bld_is_output_written(struct bld_context *bld, int i, int c)
+{
+   if (c < 0)
+      return bld->outputs_written[i / 8] & (0xf << ((i * 4) % 32));
+   return bld->outputs_written[i / 8] & (1 << ((i * 4 + c) % 32));
+}
+
+static void
+bld_append_vp_ucp(struct bld_context *bld)
+{
+   struct nv_value *res[6];
+   struct nv_value *ucp, *vtx, *out;
+   struct nv_instruction *insn;
+   int i, c;
+
+   assert(bld->ti->prog->vp.num_ucps <= 6);
+
+   for (c = 0; c < 4; ++c) {
+      vtx = bld_fetch_global(bld, &bld->ovs[bld->hpos_index][c]);
+
+      for (i = 0; i < bld->ti->prog->vp.num_ucps; ++i) {
+         ucp = new_value(bld->pc, NV_FILE_MEM_C(15), 4);
+         ucp->reg.address = i * 16 + c * 4;
+
+         if (c == 0)
+            res[i] = bld_insn_2(bld, NV_OP_MUL_F32, vtx, ucp);
+         else
+            res[i] = bld_insn_3(bld, NV_OP_MAD_F32, vtx, ucp, res[i]);
+      }
+   }
+
+   for (i = 0; i < bld->ti->prog->vp.num_ucps; ++i) {
+      (out = new_value(bld->pc, NV_FILE_MEM_V, 4))->reg.address = 0x2c0 + i * 4;
+      (insn = new_instruction(bld->pc, NV_OP_EXPORT))->fixed = 1;
+      nv_reference(bld->pc, insn, 0, out);
+      nv_reference(bld->pc, insn, 1, res[i]);
+   }
+}
+
+static void
+bld_export_fp_outputs(struct bld_context *bld)
+{
+   struct nv_value *vals[4];
+   struct nv_instruction *nvi;
+   int i, c, n;
+
+   for (i = 0; i < PIPE_MAX_SHADER_OUTPUTS; ++i) {
+      if (!bld_is_output_written(bld, i, -1))
+         continue;
+      for (n = 0, c = 0; c < 4; ++c) {
+         if (!bld_is_output_written(bld, i, c))
+            continue;
+         vals[n] = bld_fetch_global(bld, &bld->ovs[i][c]);
+         assert(vals[n]);
+         vals[n] = bld_insn_1(bld, NV_OP_MOV, vals[n]);
+         vals[n++]->reg.id = bld->ti->output_loc[i][c];
+      }
+      assert(n);
+
+      (nvi = new_instruction(bld->pc, NV_OP_EXPORT))->fixed = 1;
+      for (c = 0; c < n; ++c)
+         nv_reference(bld->pc, nvi, c, vals[c]);
+   }
+}
+
+static void
+bld_new_block(struct bld_context *bld, struct nv_basic_block *b)
+{
+   int i, c;
+
+   bld->pc->current_block = b;
+
+   for (i = 0; i < 4; ++i)
+      bld->saved_addr[i][0] = NULL;
+   for (i = 0; i < PIPE_MAX_SHADER_INPUTS; ++i)
+      for (c = 0; c < 4; ++c)
+         bld->saved_inputs[i][c] = NULL;
+
+   bld->out_kind = CFG_EDGE_FORWARD;
+}
+
+static struct nv_value *
+bld_get_saved_input(struct bld_context *bld, unsigned i, unsigned c)
+{
+   if (bld->saved_inputs[i][c])
+      return bld->saved_inputs[i][c];
+   return NULL;
+}
+
+static struct nv_value *
+bld_interp(struct bld_context *bld, unsigned mode, struct nv_value *val)
+{
+   unsigned cent = mode & NVC0_INTERP_CENTROID;
+
+   mode &= ~NVC0_INTERP_CENTROID;
+   
+   if (val->reg.address == 0x3fc) {
+      /* gl_FrontFacing: 0/~0 to -1.0/+1.0 */
+      val = bld_insn_1(bld, NV_OP_LINTERP, val);
+      val->insn->flat = 1;
+      val = bld_insn_2(bld, NV_OP_SHL, val, bld_imm_u32(bld, 31));
+      val = bld_insn_2(bld, NV_OP_XOR, val, bld_imm_f32(bld, -1.0f));
+      return val;
+   } else
+   if (mode == NVC0_INTERP_PERSPECTIVE) {
+      val = bld_insn_2(bld, NV_OP_PINTERP, val, bld->frag_coord[3]);
+   } else {
+      val = bld_insn_1(bld, NV_OP_LINTERP, val);
+   }
+
+   val->insn->flat = mode == NVC0_INTERP_FLAT ? 1 : 0;
+   val->insn->centroid = cent ? 1 : 0;
+   return val;
+}
+
+static struct nv_value *
+emit_fetch(struct bld_context *bld, const struct tgsi_full_instruction *insn,
+           const unsigned s, const unsigned chan)
+{
+   const struct tgsi_full_src_register *src = &insn->Src[s];
+   struct nv_value *res = NULL;
+   struct nv_value *ptr = NULL;
+   int idx, ind_idx, dim_idx;
+   unsigned swz, ind_swz, sgn;
+
+   idx = src->Register.Index;
+   swz = tgsi_util_get_full_src_register_swizzle(src, chan);
+
+   if (src->Register.Indirect) {
+      ind_idx = src->Indirect.Index;
+      ind_swz = tgsi_util_get_src_register_swizzle(&src->Indirect, 0);
+
+      ptr = FETCH_ADDR(ind_idx, ind_swz);
+   }
+
+   if (src->Register.Dimension)
+      dim_idx = src->Dimension.Index;
+   else
+      dim_idx = 0;
+
+   switch (src->Register.File) {
+   case TGSI_FILE_CONSTANT:
+      assert(dim_idx < 14);
+      res = new_value(bld->pc, NV_FILE_MEM_C(dim_idx), 4);
+      res->reg.address = idx * 16 + swz * 4;
+      res = bld_insn_1(bld, NV_OP_LD, res);
+      if (ptr)
+         bld_src_pointer(bld, res->insn, 1, ptr);
+      break;
+   case TGSI_FILE_IMMEDIATE: /* XXX: type for MOV TEMP[0], -IMM[0] */
+      assert(idx < bld->ti->immd32_nr);
+      res = bld_load_imm_u32(bld, bld->ti->immd32[idx * 4 + swz]);
+      break;
+   case TGSI_FILE_INPUT:
+      assert(!src->Register.Dimension);
+      if (!ptr) {
+         res = bld_get_saved_input(bld, idx, swz);
+         if (res)
+            return res;
+      }
+      res = new_value(bld->pc, bld->ti->input_file, 4);
+      if (ptr)
+         res->reg.address = 0x80 + idx * 16 + swz * 4;
+      else
+         res->reg.address = bld->ti->input_loc[idx][swz];
+
+      if (bld->pc->is_fragprog)
+         res = bld_interp(bld, bld->ti->interp_mode[idx], res);
+      else
+         res = bld_insn_1(bld, NV_OP_VFETCH, res);
+
+      if (ptr)
+         bld_src_pointer(bld, res->insn, res->insn->src[1] ? 2 : 1, ptr);
+      else
+         bld->saved_inputs[idx][swz] = res;
+      break;
+   case TGSI_FILE_TEMPORARY:
+      if (bld->ti->require_stores)
+         res = bld_lmem_load(bld, ptr, idx * 4 + swz);
+      else
+         res = bld_fetch_global(bld, &bld->tvs[idx][swz]);
+      break;
+   case TGSI_FILE_ADDRESS:
+      res = bld_fetch_global(bld, &bld->avs[idx][swz]);
+      break;
+   case TGSI_FILE_PREDICATE:
+      res = bld_fetch_global(bld, &bld->pvs[idx][swz]);
+      break;
+   default:
+      NOUVEAU_ERR("illegal/unhandled src reg file: %d\n", src->Register.File);
+      abort();
+      break;	   
+   }
+   if (!res)
+      return bld_undef(bld, NV_FILE_GPR);
+
+   sgn = tgsi_util_get_full_src_register_sign_mode(src, chan);
+
+   switch (sgn) {
+   case TGSI_UTIL_SIGN_KEEP:
+      break;
+   case TGSI_UTIL_SIGN_CLEAR:
+      res = bld_insn_1(bld, NV_OP_ABS_F32, res);
+      break;
+   case TGSI_UTIL_SIGN_TOGGLE:
+      res = bld_insn_1(bld, NV_OP_NEG_F32, res);
+      break;
+   case TGSI_UTIL_SIGN_SET:
+      res = bld_insn_1(bld, NV_OP_ABS_F32, res);
+      res = bld_insn_1(bld, NV_OP_NEG_F32, res);
+      break;
+   default:
+      NOUVEAU_ERR("illegal/unhandled src reg sign mode\n");
+      abort();
+      break;
+   }
+
+   return res;
+}
+
+static void
+bld_lit(struct bld_context *bld, struct nv_value *dst0[4],
+        const struct tgsi_full_instruction *insn)
+{
+   struct nv_value *val0 = NULL;
+   unsigned mask = insn->Dst[0].Register.WriteMask;
+
+   if (mask & ((1 << 0) | (1 << 3)))
+      dst0[3] = dst0[0] = bld_load_imm_f32(bld, 1.0f);
+
+   if (mask & (3 << 1)) {
+      val0 = bld_insn_2(bld, NV_OP_MAX, emit_fetch(bld, insn, 0, 0), bld->zero);
+      if (mask & (1 << 1))
+         dst0[1] = val0;
+   }
+
+   if (mask & (1 << 2)) {
+      struct nv_value *val1, *val3, *src1, *src3, *pred;
+      struct nv_value *pos128 = bld_load_imm_f32(bld, 127.999999f);
+      struct nv_value *neg128 = bld_load_imm_f32(bld, -127.999999f);
+
+      src1 = emit_fetch(bld, insn, 0, 1);
+      src3 = emit_fetch(bld, insn, 0, 3);
+
+      pred = bld_setp(bld, NV_OP_SET_F32, NV_CC_LE, val0, bld->zero);
+
+      val1 = bld_insn_2(bld, NV_OP_MAX_F32, src1, bld->zero);
+      val3 = bld_insn_2(bld, NV_OP_MAX_F32, src3, neg128);
+      val3 = bld_insn_2(bld, NV_OP_MIN_F32, val3, pos128);
+      val3 = bld_pow(bld, val1, val3);
+
+      dst0[2] = bld_insn_1(bld, NV_OP_MOV, bld->zero);
+      bld_src_predicate(bld, dst0[2]->insn, 1, pred);
+
+      dst0[2] = bld_insn_2(bld, NV_OP_SELECT, val3, dst0[2]);
+   }
+}
+
+static INLINE void
+describe_texture_target(unsigned target, int *dim,
+                        int *array, int *cube, int *shadow)
+{
+   *array = *cube = *shadow = 0;
+
+   switch (target) {
+   case TGSI_TEXTURE_1D:
+      *dim = 1;
+      break;
+   case TGSI_TEXTURE_SHADOW1D:
+      *dim = *shadow = 1;
+      break;
+   case TGSI_TEXTURE_UNKNOWN:
+   case TGSI_TEXTURE_2D:
+   case TGSI_TEXTURE_RECT:
+      *dim = 2;
+      break;
+   case TGSI_TEXTURE_SHADOW2D:
+   case TGSI_TEXTURE_SHADOWRECT:
+      *dim = 2;
+      *shadow = 1;
+      break;
+   case TGSI_TEXTURE_3D:
+      *dim = 3;
+      break;
+   case TGSI_TEXTURE_CUBE:
+      *dim = 2;
+      *cube = 1;
+      break;
+      /*
+   case TGSI_TEXTURE_CUBE_ARRAY:
+      *dim = 2;
+      *cube = *array = 1;
+      break;
+   case TGSI_TEXTURE_1D_ARRAY:
+      *dim = *array = 1;
+      break;
+   case TGSI_TEXTURE_2D_ARRAY:
+      *dim = 2;
+      *array = 1;
+      break;
+   case TGSI_TEXTURE_SHADOW1D_ARRAY:
+      *dim = *array = *shadow = 1;
+      break;
+   case TGSI_TEXTURE_SHADOW2D_ARRAY:
+      *dim = 2;
+      *array = *shadow = 1;
+      break;
+   case TGSI_TEXTURE_CUBE_ARRAY:
+      *dim = 2;
+      *array = *cube = 1;
+      break;
+      */
+   default:
+      assert(0);
+      break;
+   }
+}
+
+static struct nv_value *
+bld_clone(struct bld_context *bld, struct nv_instruction *nvi)
+{
+   struct nv_instruction *dupi = new_instruction(bld->pc, nvi->opcode);
+   struct nv_instruction *next, *prev;
+   int c;
+
+   next = dupi->next;
+   prev = dupi->prev;
+
+   *dupi = *nvi;
+
+   dupi->next = next;
+   dupi->prev = prev;
+
+   for (c = 0; c < 5 && nvi->def[c]; ++c)
+      bld_def(dupi, c, new_value_like(bld->pc, nvi->def[c]));
+
+   for (c = 0; c < 6 && nvi->src[c]; ++c) {
+      dupi->src[c] = NULL;
+      nv_reference(bld->pc, dupi, c, nvi->src[c]->value);
+   }
+
+   return dupi->def[0];
+}
+
+/* NOTE: proj(t0) = (t0 / w) / (tc3 / w) = tc0 / tc2 handled by optimizer */
+static void
+load_proj_tex_coords(struct bld_context *bld,
+                     struct nv_value *t[4], int dim, int shadow,
+                     const struct tgsi_full_instruction *insn)
+{
+   int c;
+   unsigned mask = (1 << dim) - 1;
+
+   if (shadow)
+      mask |= 4; /* depth comparison value */
+
+   t[3] = emit_fetch(bld, insn, 0, 3);
+   if (t[3]->insn->opcode == NV_OP_PINTERP) {
+      t[3] = bld_clone(bld, t[3]->insn);
+      t[3]->insn->opcode = NV_OP_LINTERP;
+      nv_reference(bld->pc, t[3]->insn, 1, NULL);
+   }
+   t[3] = bld_insn_1(bld, NV_OP_RCP, t[3]);
+
+   for (c = 0; c < 4; ++c) {
+      if (!(mask & (1 << c)))
+         continue;
+      t[c] = emit_fetch(bld, insn, 0, c);
+
+      if (t[c]->insn->opcode != NV_OP_PINTERP)
+         continue;
+      mask &= ~(1 << c);
+
+      t[c] = bld_clone(bld, t[c]->insn);
+      nv_reference(bld->pc, t[c]->insn, 1, t[3]);
+   }
+   if (mask == 0)
+      return;
+
+   t[3] = emit_fetch(bld, insn, 0, 3);
+   t[3] = bld_insn_1(bld, NV_OP_RCP, t[3]);
+
+   for (c = 0; c < 4; ++c)
+      if (mask & (1 << c))
+         t[c] = bld_insn_2(bld, NV_OP_MUL_F32, t[c], t[3]);
+}
+
+/* For a quad of threads / top left, top right, bottom left, bottom right
+ * pixels, do a different operation, and take src0 from a specific thread.
+ */
+#define QOP_ADD 0
+#define QOP_SUBR 1
+#define QOP_SUB 2
+#define QOP_MOV1 3
+
+#define QOP(a, b, c, d) \
+   ((QOP_##a << 0) | (QOP_##b << 2) | (QOP_##c << 4) | (QOP_##d << 6))
+
+static INLINE struct nv_value *
+bld_quadop(struct bld_context *bld, ubyte qop, struct nv_value *src0, int lane,
+           struct nv_value *src1, boolean wp)
+{
+   struct nv_value *val = bld_insn_2(bld, NV_OP_QUADOP, src0, src1);
+   val->insn->lanes = lane;
+   val->insn->quadop = qop;
+   if (wp) {
+      assert(!"quadop predicate write");
+   }
+   return val;
+}
+
+/* order of TGSI operands: x y z layer shadow lod/bias */
+/* order of native operands: layer x y z | lod/bias shadow */
+static struct nv_instruction *
+emit_tex(struct bld_context *bld, uint opcode, int tic, int tsc,
+         struct nv_value *dst[4], struct nv_value *arg[4],
+         int dim, int array, int cube, int shadow)
+{
+   struct nv_value *src[4];
+   struct nv_instruction *nvi, *bnd;
+   int c;
+   int s = 0;
+   boolean lodbias = opcode == NV_OP_TXB || opcode == NV_OP_TXL;
+
+   if (array)
+      arg[dim] = bld_cvt(bld, NV_TYPE_U32, NV_TYPE_F32, arg[dim]);
+
+   /* ensure that all inputs reside in a GPR */
+   for (c = 0; c < dim + array + cube + shadow; ++c)
+      (src[c] = bld_insn_1(bld, NV_OP_MOV, arg[c]))->insn->fixed = 1;
+
+   /* bind { layer x y z } and { lod/bias shadow } to adjacent regs */
+
+   bnd = new_instruction(bld->pc, NV_OP_BIND);
+   if (array) {
+      src[s] = new_value(bld->pc, NV_FILE_GPR, 4);
+      bld_def(bnd, s, src[s]);
+      nv_reference(bld->pc, bnd, s++, arg[dim + cube]);
+   }
+   for (c = 0; c < dim + cube; ++c, ++s) {
+      src[s] = bld_def(bnd, s, new_value(bld->pc, NV_FILE_GPR, 4));
+      nv_reference(bld->pc, bnd, s, arg[c]);
+   }
+
+   if (shadow || lodbias) {
+      bnd = new_instruction(bld->pc, NV_OP_BIND);
+
+      if (lodbias) {
+         src[s] = new_value(bld->pc, NV_FILE_GPR, 4);
+         bld_def(bnd, 0, src[s++]);
+         nv_reference(bld->pc, bnd, 0, arg[dim + cube + array + shadow]);
+      }
+      if (shadow) {
+         src[s] = new_value(bld->pc, NV_FILE_GPR, 4);
+         bld_def(bnd, lodbias, src[s++]);
+         nv_reference(bld->pc, bnd, lodbias, arg[dim + cube + array]);
+      }
+   }
+
+   nvi = new_instruction(bld->pc, opcode);
+   for (c = 0; c < 4; ++c)
+      dst[c] = bld_def(nvi, c, new_value(bld->pc, NV_FILE_GPR, 4));
+   for (c = 0; c < s; ++c)
+      nv_reference(bld->pc, nvi, c, src[c]);
+
+   nvi->ext.tex.t = tic;
+   nvi->ext.tex.s = tsc;
+   nvi->tex_mask = 0xf;
+   nvi->tex_cube = cube;
+   nvi->tex_dim = dim;
+   nvi->tex_cube = cube;
+   nvi->tex_shadow = shadow;
+   nvi->tex_live = 0;
+
+   return nvi;
+}
+
+/*
+static boolean
+bld_is_constant(struct nv_value *val)
+{
+   if (val->reg.file == NV_FILE_IMM)
+      return TRUE;
+   return val->insn && nvCG_find_constant(val->insn->src[0]);
+}
+*/
+
+static void
+bld_tex(struct bld_context *bld, struct nv_value *dst0[4],
+        const struct tgsi_full_instruction *insn)
+{
+   struct nv_value *t[4], *s[3];
+   uint opcode = translate_opcode(insn->Instruction.Opcode);
+   int c, dim, array, cube, shadow;
+   const int lodbias = opcode == NV_OP_TXB || opcode == NV_OP_TXL;
+   const int tic = insn->Src[1].Register.Index;
+   const int tsc = tic;
+
+   describe_texture_target(insn->Texture.Texture, &dim, &array, &cube, &shadow);
+
+   assert(dim + array + shadow + lodbias <= 5);
+
+   if (!cube && insn->Instruction.Opcode == TGSI_OPCODE_TXP)
+      load_proj_tex_coords(bld, t, dim, shadow, insn);
+   else {
+      for (c = 0; c < dim + cube + array; ++c)
+         t[c] = emit_fetch(bld, insn, 0, c);
+      if (shadow)
+         t[c] = emit_fetch(bld, insn, 0, MAX2(c, 2));
+   }
+
+   if (cube) {
+      for (c = 0; c < 3; ++c)
+         s[c] = bld_insn_1(bld, NV_OP_ABS_F32, t[c]);
+
+      s[0] = bld_insn_2(bld, NV_OP_MAX_F32, s[0], s[1]);
+      s[0] = bld_insn_2(bld, NV_OP_MAX_F32, s[0], s[2]);
+      s[0] = bld_insn_1(bld, NV_OP_RCP, s[0]);
+
+      for (c = 0; c < 3; ++c)
+         t[c] = bld_insn_2(bld, NV_OP_MUL_F32, t[c], s[0]);
+   }
+
+   if (lodbias)
+      t[dim + cube + array + shadow] = emit_fetch(bld, insn, 0, 3);
+
+   emit_tex(bld, opcode, tic, tsc, dst0, t, dim, array, cube, shadow);
+}
+
+static INLINE struct nv_value *
+bld_dot(struct bld_context *bld, const struct tgsi_full_instruction *insn,
+        int n)
+{
+   struct nv_value *dotp, *src0, *src1;
+   int c;
+
+   src0 = emit_fetch(bld, insn, 0, 0);
+   src1 = emit_fetch(bld, insn, 1, 0);
+   dotp = bld_insn_2(bld, NV_OP_MUL_F32, src0, src1);
+
+   for (c = 1; c < n; ++c) {
+      src0 = emit_fetch(bld, insn, 0, c);
+      src1 = emit_fetch(bld, insn, 1, c);
+      dotp = bld_insn_3(bld, NV_OP_MAD_F32, src0, src1, dotp);
+   }
+   return dotp;
+}
+
+#define FOR_EACH_DST0_ENABLED_CHANNEL(chan, inst) \
+   for (chan = 0; chan < 4; ++chan)               \
+      if ((inst)->Dst[0].Register.WriteMask & (1 << chan))
+
+static void
+bld_instruction(struct bld_context *bld,
+                const struct tgsi_full_instruction *insn)
+{
+   struct nv_value *src0;
+   struct nv_value *src1;
+   struct nv_value *src2;
+   struct nv_value *dst0[4] = { NULL };
+   struct nv_value *temp;
+   int c;
+   uint opcode = translate_opcode(insn->Instruction.Opcode);
+   uint8_t mask = insn->Dst[0].Register.WriteMask;
+
+#ifdef NOUVEAU_DEBUG
+   debug_printf("bld_instruction:"); tgsi_dump_instruction(insn, 1);
+#endif
+	
+   switch (insn->Instruction.Opcode) {
+   case TGSI_OPCODE_ADD:
+   case TGSI_OPCODE_MAX:
+   case TGSI_OPCODE_MIN:
+   case TGSI_OPCODE_MUL:
+      FOR_EACH_DST0_ENABLED_CHANNEL(c, insn) {
+         src0 = emit_fetch(bld, insn, 0, c);
+         src1 = emit_fetch(bld, insn, 1, c);
+         dst0[c] = bld_insn_2(bld, opcode, src0, src1);
+      }
+      break;
+   case TGSI_OPCODE_ARL:
+      src1 = bld_imm_u32(bld, 4);
+      FOR_EACH_DST0_ENABLED_CHANNEL(c, insn) {
+         src0 = emit_fetch(bld, insn, 0, c);
+         src0 = bld_insn_1(bld, NV_OP_FLOOR, src0);
+         src0->insn->ext.cvt.d = NV_TYPE_S32;
+         src0->insn->ext.cvt.s = NV_TYPE_F32;
+         dst0[c] = bld_insn_2(bld, NV_OP_SHL, src0, src1);
+      }
+      break;
+   case TGSI_OPCODE_CMP:
+      FOR_EACH_DST0_ENABLED_CHANNEL(c, insn) {
+         src0 = emit_fetch(bld, insn, 0, c);
+         src0 = bld_setp(bld, NV_OP_SET_F32, NV_CC_LT, src0, bld->zero);
+         src1 = emit_fetch(bld, insn, 1, c);
+         src2 = emit_fetch(bld, insn, 2, c);
+         dst0[c] = bld_insn_3(bld, NV_OP_SELP, src1, src2, src0);
+      }
+      break;
+   case TGSI_OPCODE_COS:
+   case TGSI_OPCODE_SIN:
+      src0 = emit_fetch(bld, insn, 0, 0);
+      temp = bld_insn_1(bld, NV_OP_PRESIN, src0);
+      if (insn->Dst[0].Register.WriteMask & 7)
+         temp = bld_insn_1(bld, opcode, temp);
+      for (c = 0; c < 3; ++c)
+         if (insn->Dst[0].Register.WriteMask & (1 << c))
+            dst0[c] = temp;
+      if (!(insn->Dst[0].Register.WriteMask & (1 << 3)))
+         break;
+      src0 = emit_fetch(bld, insn, 0, 3);
+      temp = bld_insn_1(bld, NV_OP_PRESIN, src0);
+      dst0[3] = bld_insn_1(bld, opcode, temp);
+      break;
+   case TGSI_OPCODE_DP2:
+      temp = bld_dot(bld, insn, 2);
+      FOR_EACH_DST0_ENABLED_CHANNEL(c, insn)
+         dst0[c] = temp;
+      break;
+   case TGSI_OPCODE_DP3:
+      temp = bld_dot(bld, insn, 3);
+      FOR_EACH_DST0_ENABLED_CHANNEL(c, insn)
+         dst0[c] = temp;
+      break;
+   case TGSI_OPCODE_DP4:
+      temp = bld_dot(bld, insn, 4);
+      FOR_EACH_DST0_ENABLED_CHANNEL(c, insn)
+         dst0[c] = temp;
+      break;
+   case TGSI_OPCODE_DPH:
+      src0 = bld_dot(bld, insn, 3);
+      src1 = emit_fetch(bld, insn, 1, 3);
+      temp = bld_insn_2(bld, NV_OP_ADD_F32, src0, src1);
+      FOR_EACH_DST0_ENABLED_CHANNEL(c, insn)
+         dst0[c] = temp;
+      break;
+   case TGSI_OPCODE_DST:
+      if (insn->Dst[0].Register.WriteMask & 1)
+         dst0[0] = bld_imm_f32(bld, 1.0f);
+      if (insn->Dst[0].Register.WriteMask & 2) {
+         src0 = emit_fetch(bld, insn, 0, 1);
+         src1 = emit_fetch(bld, insn, 1, 1);
+         dst0[1] = bld_insn_2(bld, NV_OP_MUL_F32, src0, src1);
+      }
+      if (insn->Dst[0].Register.WriteMask & 4)
+         dst0[2] = emit_fetch(bld, insn, 0, 2);
+      if (insn->Dst[0].Register.WriteMask & 8)
+         dst0[3] = emit_fetch(bld, insn, 1, 3);
+      break;
+   case TGSI_OPCODE_EXP:
+      src0 = emit_fetch(bld, insn, 0, 0);
+      temp = bld_insn_1(bld, NV_OP_FLOOR, src0);
+
+      if (insn->Dst[0].Register.WriteMask & 2)
+         dst0[1] = bld_insn_2(bld, NV_OP_SUB_F32, src0, temp);
+      if (insn->Dst[0].Register.WriteMask & 1) {
+         temp = bld_insn_1(bld, NV_OP_PREEX2, temp);
+         dst0[0] = bld_insn_1(bld, NV_OP_EX2, temp);
+      }
+      if (insn->Dst[0].Register.WriteMask & 4) {
+         temp = bld_insn_1(bld, NV_OP_PREEX2, src0);
+         dst0[2] = bld_insn_1(bld, NV_OP_EX2, temp);
+      }
+      if (insn->Dst[0].Register.WriteMask & 8)
+         dst0[3] = bld_imm_f32(bld, 1.0f);
+      break;
+   case TGSI_OPCODE_EX2:
+      src0 = emit_fetch(bld, insn, 0, 0);
+      temp = bld_insn_1(bld, NV_OP_PREEX2, src0);
+      temp = bld_insn_1(bld, NV_OP_EX2, temp);
+      FOR_EACH_DST0_ENABLED_CHANNEL(c, insn)
+         dst0[c] = temp;
+      break;
+   case TGSI_OPCODE_FRC:
+      FOR_EACH_DST0_ENABLED_CHANNEL(c, insn) {
+         src0 = emit_fetch(bld, insn, 0, c);
+         dst0[c] = bld_insn_1(bld, NV_OP_FLOOR, src0);
+         dst0[c] = bld_insn_2(bld, NV_OP_SUB_F32, src0, dst0[c]);
+      }
+      break;
+   case TGSI_OPCODE_KIL:
+      for (c = 0; c < 4; ++c)
+         bld_kil(bld, emit_fetch(bld, insn, 0, c));
+      break;
+   case TGSI_OPCODE_KILP:
+      (new_instruction(bld->pc, NV_OP_KIL))->fixed = 1;
+      break;
+   case TGSI_OPCODE_IF:
+   {
+      struct nv_basic_block *b = new_basic_block(bld->pc);
+
+      assert(bld->cond_lvl < BLD_MAX_COND_NESTING);
+
+      nvc0_bblock_attach(bld->pc->current_block, b, CFG_EDGE_FORWARD);
+
+      bld->join_bb[bld->cond_lvl] = bld->pc->current_block;
+      bld->cond_bb[bld->cond_lvl] = bld->pc->current_block;
+
+      src1 = bld_setp(bld, NV_OP_SET_U32, NV_CC_EQ,
+                      emit_fetch(bld, insn, 0, 0), bld->zero);
+
+      bld_flow(bld, NV_OP_BRA, src1, NULL, (bld->cond_lvl == 0));
+
+      ++bld->cond_lvl;
+      bld_new_block(bld, b);
+   }
+      break;
+   case TGSI_OPCODE_ELSE:
+   {
+      struct nv_basic_block *b = new_basic_block(bld->pc);
+
+      --bld->cond_lvl;
+      nvc0_bblock_attach(bld->join_bb[bld->cond_lvl], b, CFG_EDGE_FORWARD);
+
+      bld->cond_bb[bld->cond_lvl]->exit->target = b;
+      bld->cond_bb[bld->cond_lvl] = bld->pc->current_block;
+
+      new_instruction(bld->pc, NV_OP_BRA)->terminator = 1;
+
+      ++bld->cond_lvl;
+      bld_new_block(bld, b);
+   }
+      break;
+   case TGSI_OPCODE_ENDIF:
+   {
+      struct nv_basic_block *b = new_basic_block(bld->pc);
+
+      --bld->cond_lvl;
+      nvc0_bblock_attach(bld->pc->current_block, b, bld->out_kind);
+      nvc0_bblock_attach(bld->cond_bb[bld->cond_lvl], b, CFG_EDGE_FORWARD);
+
+      bld->cond_bb[bld->cond_lvl]->exit->target = b;
+
+      bld_new_block(bld, b);
+
+      if (!bld->cond_lvl && bld->join_bb[bld->cond_lvl]) {
+         bld->join_bb[bld->cond_lvl]->exit->prev->target = b;
+         new_instruction(bld->pc, NV_OP_JOIN)->join = 1;
+      }
+   }
+      break;
+   case TGSI_OPCODE_BGNLOOP:
+   {
+      struct nv_basic_block *bl = new_basic_block(bld->pc);
+      struct nv_basic_block *bb = new_basic_block(bld->pc);
+
+      assert(bld->loop_lvl < BLD_MAX_LOOP_NESTING);
+
+      bld->loop_bb[bld->loop_lvl] = bl;
+      bld->brkt_bb[bld->loop_lvl] = bb;
+
+      nvc0_bblock_attach(bld->pc->current_block, bl, CFG_EDGE_LOOP_ENTER);
+
+      bld_new_block(bld, bld->loop_bb[bld->loop_lvl++]);
+
+      if (bld->loop_lvl == bld->pc->loop_nesting_bound)
+         bld->pc->loop_nesting_bound++;
+
+      bld_clear_def_use(&bld->tvs[0][0], BLD_MAX_TEMPS, bld->loop_lvl);
+      bld_clear_def_use(&bld->avs[0][0], BLD_MAX_ADDRS, bld->loop_lvl);
+      bld_clear_def_use(&bld->pvs[0][0], BLD_MAX_PREDS, bld->loop_lvl);
+   }
+      break;
+   case TGSI_OPCODE_BRK:
+   {
+      struct nv_basic_block *bb = bld->brkt_bb[bld->loop_lvl - 1];
+
+      bld_flow(bld, NV_OP_BRA, NULL, bb, FALSE);
+
+      if (bld->out_kind == CFG_EDGE_FORWARD) /* else we already had BRK/CONT */
+         nvc0_bblock_attach(bld->pc->current_block, bb, CFG_EDGE_LOOP_LEAVE);
+
+      bld->out_kind = CFG_EDGE_FAKE;
+   }
+      break;
+   case TGSI_OPCODE_CONT:
+   {
+      struct nv_basic_block *bb = bld->loop_bb[bld->loop_lvl - 1];
+
+      bld_flow(bld, NV_OP_BRA, NULL, bb, FALSE);
+
+      nvc0_bblock_attach(bld->pc->current_block, bb, CFG_EDGE_BACK);
+
+      if ((bb = bld->join_bb[bld->cond_lvl - 1])) {
+         bld->join_bb[bld->cond_lvl - 1] = NULL;
+         nvc0_insn_delete(bb->exit->prev);
+      }
+      bld->out_kind = CFG_EDGE_FAKE;
+   }
+      break;
+   case TGSI_OPCODE_ENDLOOP:
+   {
+      struct nv_basic_block *bb = bld->loop_bb[bld->loop_lvl - 1];
+
+      bld_flow(bld, NV_OP_BRA, NULL, bb, FALSE);
+
+      nvc0_bblock_attach(bld->pc->current_block, bb, CFG_EDGE_BACK);
+
+      bld_loop_end(bld, bb); /* replace loop-side operand of the phis */
+
+      bld_new_block(bld, bld->brkt_bb[--bld->loop_lvl]);
+   }
+      break;
+   case TGSI_OPCODE_ABS:
+   case TGSI_OPCODE_CEIL:
+   case TGSI_OPCODE_FLR:
+   case TGSI_OPCODE_TRUNC:
+   case TGSI_OPCODE_DDX:
+   case TGSI_OPCODE_DDY:
+      FOR_EACH_DST0_ENABLED_CHANNEL(c, insn) {
+         src0 = emit_fetch(bld, insn, 0, c);
+         dst0[c] = bld_insn_1(bld, opcode, src0);
+      }	   
+      break;
+   case TGSI_OPCODE_LIT:
+      bld_lit(bld, dst0, insn);
+      break;
+   case TGSI_OPCODE_LRP:
+      FOR_EACH_DST0_ENABLED_CHANNEL(c, insn) {
+         src0 = emit_fetch(bld, insn, 0, c);
+         src1 = emit_fetch(bld, insn, 1, c);
+         src2 = emit_fetch(bld, insn, 2, c);
+         dst0[c] = bld_insn_2(bld, NV_OP_SUB_F32, src1, src2);
+         dst0[c] = bld_insn_3(bld, NV_OP_MAD_F32, dst0[c], src0, src2);
+      }
+      break;
+   case TGSI_OPCODE_MOV:
+      FOR_EACH_DST0_ENABLED_CHANNEL(c, insn)
+         dst0[c] = emit_fetch(bld, insn, 0, c);
+      break;
+   case TGSI_OPCODE_MAD:
+      FOR_EACH_DST0_ENABLED_CHANNEL(c, insn) {
+         src0 = emit_fetch(bld, insn, 0, c);
+         src1 = emit_fetch(bld, insn, 1, c);
+         src2 = emit_fetch(bld, insn, 2, c);
+         dst0[c] = bld_insn_3(bld, opcode, src0, src1, src2);
+      }
+      break;
+   case TGSI_OPCODE_POW:
+      src0 = emit_fetch(bld, insn, 0, 0);
+      src1 = emit_fetch(bld, insn, 1, 0);
+      temp = bld_pow(bld, src0, src1);
+      FOR_EACH_DST0_ENABLED_CHANNEL(c, insn)
+         dst0[c] = temp;
+      break;
+   case TGSI_OPCODE_LOG:
+      src0 = emit_fetch(bld, insn, 0, 0);
+      src0 = bld_insn_1(bld, NV_OP_ABS_F32, src0);
+      temp = bld_insn_1(bld, NV_OP_LG2, src0);
+      dst0[2] = temp;
+      if (insn->Dst[0].Register.WriteMask & 3) {
+         temp = bld_insn_1(bld, NV_OP_FLOOR, temp);
+         dst0[0] = temp;
+      }
+      if (insn->Dst[0].Register.WriteMask & 2) {
+         temp = bld_insn_1(bld, NV_OP_PREEX2, temp);
+         temp = bld_insn_1(bld, NV_OP_EX2, temp);
+         temp = bld_insn_1(bld, NV_OP_RCP, temp);
+         dst0[1] = bld_insn_2(bld, NV_OP_MUL_F32, src0, temp);
+      }
+      if (insn->Dst[0].Register.WriteMask & 8)
+         dst0[3] = bld_imm_f32(bld, 1.0f);
+      break;
+   case TGSI_OPCODE_RCP:
+   case TGSI_OPCODE_LG2:
+      src0 = emit_fetch(bld, insn, 0, 0);
+      temp = bld_insn_1(bld, opcode, src0);
+      FOR_EACH_DST0_ENABLED_CHANNEL(c, insn)
+         dst0[c] = temp;
+      break;
+   case TGSI_OPCODE_RSQ:
+      src0 = emit_fetch(bld, insn, 0, 0);
+      temp = bld_insn_1(bld, NV_OP_ABS_F32, src0);
+      temp = bld_insn_1(bld, NV_OP_RSQ, temp);
+      FOR_EACH_DST0_ENABLED_CHANNEL(c, insn)
+         dst0[c] = temp;
+      break;
+   case TGSI_OPCODE_SLT:
+   case TGSI_OPCODE_SGE:
+   case TGSI_OPCODE_SEQ:
+   case TGSI_OPCODE_SGT:
+   case TGSI_OPCODE_SLE:
+   case TGSI_OPCODE_SNE:
+   case TGSI_OPCODE_ISLT:
+   case TGSI_OPCODE_ISGE:
+   case TGSI_OPCODE_USEQ:
+   case TGSI_OPCODE_USGE:
+   case TGSI_OPCODE_USLT:
+   case TGSI_OPCODE_USNE:
+      FOR_EACH_DST0_ENABLED_CHANNEL(c, insn) {
+         src0 = emit_fetch(bld, insn, 0, c);
+         src1 = emit_fetch(bld, insn, 1, c);
+         dst0[c] = bld_insn_2(bld, opcode, src0, src1);
+         dst0[c]->insn->set_cond = translate_setcc(insn->Instruction.Opcode);
+      }
+      break;
+   case TGSI_OPCODE_SCS:
+      if (insn->Dst[0].Register.WriteMask & 0x3) {
+         src0 = emit_fetch(bld, insn, 0, 0);
+         temp = bld_insn_1(bld, NV_OP_PRESIN, src0);
+         if (insn->Dst[0].Register.WriteMask & 0x1)
+            dst0[0] = bld_insn_1(bld, NV_OP_COS, temp);
+         if (insn->Dst[0].Register.WriteMask & 0x2)
+            dst0[1] = bld_insn_1(bld, NV_OP_SIN, temp);
+      }
+      if (insn->Dst[0].Register.WriteMask & 0x4)
+         dst0[2] = bld_imm_f32(bld, 0.0f);
+      if (insn->Dst[0].Register.WriteMask & 0x8)
+         dst0[3] = bld_imm_f32(bld, 1.0f);
+      break;
+   case TGSI_OPCODE_SSG:
+      FOR_EACH_DST0_ENABLED_CHANNEL(c, insn) { /* XXX: set lt, set gt, sub */
+         src0 = emit_fetch(bld, insn, 0, c);
+         src1 = bld_setp(bld, NV_OP_SET_F32, NV_CC_EQ, src0, bld->zero);
+         temp = bld_insn_2(bld, NV_OP_AND, src0, bld_imm_u32(bld, 0x80000000));
+         temp = bld_insn_2(bld, NV_OP_OR,  temp, bld_imm_f32(bld, 1.0f));
+         dst0[c] = bld_insn_1(bld, NV_OP_MOV, temp);
+         bld_src_predicate(bld, dst0[c]->insn, 1, src1);
+      }
+      break;
+   case TGSI_OPCODE_SUB:
+      FOR_EACH_DST0_ENABLED_CHANNEL(c, insn) {
+         src0 = emit_fetch(bld, insn, 0, c);
+         src1 = emit_fetch(bld, insn, 1, c);
+         dst0[c] = bld_insn_2(bld, NV_OP_SUB_F32, src0, src1);
+      }
+      break;
+   case TGSI_OPCODE_TEX:
+   case TGSI_OPCODE_TXB:
+   case TGSI_OPCODE_TXL:
+   case TGSI_OPCODE_TXP:
+      bld_tex(bld, dst0, insn);
+      break;
+   case TGSI_OPCODE_XPD:
+      FOR_EACH_DST0_ENABLED_CHANNEL(c, insn) {
+         if (c == 3) {
+            dst0[3] = bld_imm_f32(bld, 1.0f);
+            break;
+         }
+         src0 = emit_fetch(bld, insn, 1, (c + 1) % 3);
+         src1 = emit_fetch(bld, insn, 0, (c + 2) % 3);
+         dst0[c] = bld_insn_2(bld, NV_OP_MUL_F32, src0, src1);
+
+         src0 = emit_fetch(bld, insn, 0, (c + 1) % 3);
+         src1 = emit_fetch(bld, insn, 1, (c + 2) % 3);
+         dst0[c] = bld_insn_3(bld, NV_OP_MAD_F32, src0, src1, dst0[c]);
+
+         dst0[c]->insn->src[2]->mod ^= NV_MOD_NEG;
+      }
+      break;
+   case TGSI_OPCODE_RET:
+      (new_instruction(bld->pc, NV_OP_RET))->fixed = 1;
+      break;
+   case TGSI_OPCODE_END:
+      /* VP outputs are exported in-place as scalars, optimization later */
+      if (bld->pc->is_fragprog)
+         bld_export_fp_outputs(bld);
+      if (bld->ti->append_ucp)
+         bld_append_vp_ucp(bld);
+      return;
+   default:
+      NOUVEAU_ERR("unhandled opcode %u\n", insn->Instruction.Opcode);
+      abort();
+      return;
+   }
+
+   if (insn->Dst[0].Register.File == TGSI_FILE_OUTPUT &&
+       !bld->pc->is_fragprog) {
+      struct nv_instruction *mi = NULL;
+      uint size;
+
+      if (bld->ti->append_ucp) {
+         if (bld->ti->output_loc[insn->Dst[0].Register.Index][0] == 0x70) {
+            bld->hpos_index = insn->Dst[0].Register.Index;
+            for (c = 0; c < 4; ++c)
+               if (mask & (1 << c))
+                  STORE_OUTP(insn->Dst[0].Register.Index, c, dst0[c]);
+         }
+      }
+
+      for (c = 0; c < 4; ++c)
+         if ((mask & (1 << c)) &&
+             ((dst0[c]->reg.file == NV_FILE_IMM) ||
+              (dst0[c]->reg.id == 63 && dst0[c]->reg.file == NV_FILE_GPR)))
+            dst0[c] = bld_insn_1(bld, NV_OP_MOV, dst0[c]);
+
+      c = 0;
+      if ((mask & 0x3) == 0x3) {
+         mask &= ~0x3;
+         size = 8;
+         mi = bld_insn_2(bld, NV_OP_BIND, dst0[0], dst0[1])->insn;
+      }
+      if ((mask & 0xc) == 0xc) {
+         mask &= ~0xc;
+         if (mi) {
+            size = 16;
+            nv_reference(bld->pc, mi, 2, dst0[2]);
+            nv_reference(bld->pc, mi, 3, dst0[3]);
+         } else {
+            c = 2;
+            size = 8;
+            mi = bld_insn_2(bld, NV_OP_BIND, dst0[2], dst0[3])->insn;
+         }
+      } else
+      if (mi && (mask & 0x4)) {
+         size = 12;
+         mask &= ~0x4;
+         nv_reference(bld->pc, mi, 2, dst0[2]);
+      }
+
+      if (mi) {
+         struct nv_instruction *ex = new_instruction(bld->pc, NV_OP_EXPORT);
+         int s;
+
+         nv_reference(bld->pc, ex, 0, new_value(bld->pc, NV_FILE_MEM_V, 4));
+         nv_reference(bld->pc, ex, 1, mi->def[0]);
+
+         for (s = 1; s < size / 4; ++s) {
+            bld_def(mi, s, new_value(bld->pc, NV_FILE_GPR, 4));
+            nv_reference(bld->pc, ex, s + 1, mi->def[s]);
+         }
+
+         ex->fixed = 1;
+         ex->src[0]->value->reg.size = size;
+         ex->src[0]->value->reg.address =
+            bld->ti->output_loc[insn->Dst[0].Register.Index][c];
+      }
+   }
+
+   for (c = 0; c < 4; ++c)
+      if (mask & (1 << c))
+         emit_store(bld, insn, c, dst0[c]);
+}
+
+static INLINE void
+bld_free_registers(struct bld_register *base, int n)
+{
+   int i, c;
+
+   for (i = 0; i < n; ++i)
+      for (c = 0; c < 4; ++c)
+         util_dynarray_fini(&base[i * 4 + c].vals);
+}
+
+int
+nvc0_tgsi_to_nc(struct nv_pc *pc, struct nvc0_translation_info *ti)
+{
+   struct bld_context *bld = CALLOC_STRUCT(bld_context);
+   unsigned ip;
+
+   pc->root[0] = pc->current_block = new_basic_block(pc);
+
+   bld->pc = pc;
+   bld->ti = ti;
+
+   pc->loop_nesting_bound = 1;
+
+   bld->zero = new_value(pc, NV_FILE_GPR, 4);
+   bld->zero->reg.id = 63;
+
+   if (pc->is_fragprog) {
+      struct nv_value *mem = new_value(pc, NV_FILE_MEM_V, 4);
+      mem->reg.address = 0x7c;
+
+      bld->frag_coord[3] = bld_insn_1(bld, NV_OP_LINTERP, mem);
+      bld->frag_coord[3] = bld_insn_1(bld, NV_OP_RCP, bld->frag_coord[3]);
+   }
+
+   for (ip = 0; ip < ti->num_insns; ++ip)
+      bld_instruction(bld, &ti->insns[ip]);
+
+   bld_free_registers(&bld->tvs[0][0], BLD_MAX_TEMPS);
+   bld_free_registers(&bld->avs[0][0], BLD_MAX_ADDRS);
+   bld_free_registers(&bld->pvs[0][0], BLD_MAX_PREDS);
+   bld_free_registers(&bld->ovs[0][0], PIPE_MAX_SHADER_OUTPUTS);
+
+   FREE(bld);
+   return 0;
+}
+
+/* If a variable is assigned in a loop, replace all references to the value
+ * from outside the loop with a phi value.
+ */
+static void
+bld_replace_value(struct nv_pc *pc, struct nv_basic_block *b,
+                  struct nv_value *old_val,
+                  struct nv_value *new_val)
+{
+   struct nv_instruction *nvi;
+
+   for (nvi = b->phi ? b->phi : b->entry; nvi; nvi = nvi->next) {
+      int s;
+      for (s = 0; s < 6 && nvi->src[s]; ++s)
+         if (nvi->src[s]->value == old_val)
+            nv_reference(pc, nvi, s, new_val);
+   }
+
+   b->pass_seq = pc->pass_seq;
+
+   if (b->out[0] && b->out[0]->pass_seq < pc->pass_seq)
+      bld_replace_value(pc, b->out[0], old_val, new_val);
+
+   if (b->out[1] && b->out[1]->pass_seq < pc->pass_seq)
+      bld_replace_value(pc, b->out[1], old_val, new_val);
+}
diff --git a/src/gallium/drivers/nvc0/nvc0_transfer.c b/src/gallium/drivers/nvc0/nvc0_transfer.c
new file mode 100644
index 00000000000..286b382f58e
--- /dev/null
+++ b/src/gallium/drivers/nvc0/nvc0_transfer.c
@@ -0,0 +1,381 @@
+
+#include "util/u_format.h"
+
+#include "nvc0_context.h"
+#include "nvc0_transfer.h"
+
+#include "nv50_defs.xml.h"
+
+struct nvc0_transfer {
+   struct pipe_transfer base;
+   struct nvc0_m2mf_rect rect[2];
+   uint32_t nblocksx;
+   uint32_t nblocksy;
+};
+
+static void
+nvc0_m2mf_transfer_rect(struct pipe_screen *pscreen,
+                        const struct nvc0_m2mf_rect *dst,
+                        const struct nvc0_m2mf_rect *src,
+                        uint32_t nblocksx, uint32_t nblocksy)
+{
+   struct nouveau_channel *chan = nouveau_screen(pscreen)->channel;
+   const int cpp = dst->cpp;
+   uint32_t src_ofst = src->base;
+   uint32_t dst_ofst = dst->base;
+   uint32_t height = nblocksy;
+   uint32_t sy = src->y;
+   uint32_t dy = dst->y;
+   uint32_t exec = (1 << 20);
+
+   assert(dst->cpp == src->cpp);
+
+   if (nouveau_bo_tile_layout(src->bo)) {
+      BEGIN_RING(chan, RING_MF(TILING_MODE_IN), 5);
+      OUT_RING  (chan, src->tile_mode);
+      OUT_RING  (chan, src->width * cpp);
+      OUT_RING  (chan, src->height);
+      OUT_RING  (chan, src->depth);
+      OUT_RING  (chan, src->z);
+   } else {
+      src_ofst += src->y * src->pitch + src->x * cpp;
+
+      BEGIN_RING(chan, RING_MF(PITCH_IN), 1);
+      OUT_RING  (chan, src->width * cpp);
+
+      exec |= NVC0_M2MF_EXEC_LINEAR_IN;
+   }
+
+   if (nouveau_bo_tile_layout(dst->bo)) {
+      BEGIN_RING(chan, RING_MF(TILING_MODE_OUT), 5);
+      OUT_RING  (chan, dst->tile_mode);
+      OUT_RING  (chan, dst->width * cpp);
+      OUT_RING  (chan, dst->height);
+      OUT_RING  (chan, dst->depth);
+      OUT_RING  (chan, dst->z);
+   } else {
+      dst_ofst += dst->y * dst->pitch + dst->x * cpp;
+
+      BEGIN_RING(chan, RING_MF(PITCH_OUT), 1);
+      OUT_RING  (chan, dst->width * cpp);
+
+      exec |= NVC0_M2MF_EXEC_LINEAR_OUT;
+   }
+
+   while (height) {
+      int line_count = height > 2047 ? 2047 : height;
+
+      MARK_RING (chan, 17, 4);
+
+      BEGIN_RING(chan, RING_MF(OFFSET_IN_HIGH), 2);
+      OUT_RELOCh(chan, src->bo, src_ofst, src->domain | NOUVEAU_BO_RD);
+      OUT_RELOCl(chan, src->bo, src_ofst, src->domain | NOUVEAU_BO_RD);
+
+      BEGIN_RING(chan, RING_MF(OFFSET_OUT_HIGH), 2);
+      OUT_RELOCh(chan, dst->bo, dst_ofst, dst->domain | NOUVEAU_BO_WR);
+      OUT_RELOCl(chan, dst->bo, dst_ofst, dst->domain | NOUVEAU_BO_WR);
+
+      if (!(exec & NVC0_M2MF_EXEC_LINEAR_IN)) {
+         BEGIN_RING(chan, RING_MF(TILING_POSITION_IN_X), 2);
+         OUT_RING  (chan, src->x * cpp);
+         OUT_RING  (chan, sy);
+      } else {
+         src_ofst += line_count * src->pitch;
+      }
+      if (!(exec & NVC0_M2MF_EXEC_LINEAR_OUT)) {
+         BEGIN_RING(chan, RING_MF(TILING_POSITION_OUT_X), 2);
+         OUT_RING  (chan, dst->x * cpp);
+         OUT_RING  (chan, dy);
+      } else {
+         dst_ofst += line_count * dst->pitch;
+      }
+
+      BEGIN_RING(chan, RING_MF(LINE_LENGTH_IN), 2);
+      OUT_RING  (chan, nblocksx * cpp);
+      OUT_RING  (chan, line_count);
+      BEGIN_RING(chan, RING_MF(EXEC), 1);
+      OUT_RING  (chan, exec);
+
+      height -= line_count;
+      sy += line_count;
+      dy += line_count;
+   }
+}
+
+void
+nvc0_m2mf_push_linear(struct nvc0_context *nvc0,
+                      struct nouveau_bo *dst, unsigned domain, int offset,
+                      unsigned size, void *data)
+{
+   struct nouveau_channel *chan = nvc0->screen->base.channel;
+   uint32_t *src = (uint32_t *)data;
+   unsigned count = (size + 3) / 4;
+
+   MARK_RING (chan, 8, 2);
+
+   BEGIN_RING(chan, RING_MF(OFFSET_OUT_HIGH), 2);
+   OUT_RELOCh(chan, dst, offset, domain | NOUVEAU_BO_WR);
+   OUT_RELOCl(chan, dst, offset, domain | NOUVEAU_BO_WR);
+   BEGIN_RING(chan, RING_MF(LINE_LENGTH_IN), 2);
+   OUT_RING  (chan, size);
+   OUT_RING  (chan, 1);
+   BEGIN_RING(chan, RING_MF(EXEC), 1);
+   OUT_RING  (chan, 0x100111);
+
+   while (count) {
+      unsigned nr = AVAIL_RING(chan);
+
+      if (nr < 9) {
+         FIRE_RING(chan);
+         nouveau_bo_validate(chan, dst, NOUVEAU_BO_WR);
+         continue;
+      }
+      nr = MIN2(count, nr - 1);
+      nr = MIN2(nr, NV04_PFIFO_MAX_PACKET_LEN);
+   
+      BEGIN_RING_NI(chan, RING_MF(DATA), nr);
+      OUT_RINGp (chan, src, nr);
+
+      src += nr;
+      count -= nr;
+   }
+}
+
+void
+nvc0_m2mf_copy_linear(struct nvc0_context *nvc0,
+                      struct nouveau_bo *dst, unsigned dstoff, unsigned dstdom,
+                      struct nouveau_bo *src, unsigned srcoff, unsigned srcdom,
+                      unsigned size)
+{
+   struct nouveau_channel *chan = nvc0->screen->base.channel;
+
+   while (size) {
+      unsigned bytes = MIN2(size, 1 << 17);
+
+      MARK_RING (chan, 11, 4);
+
+      BEGIN_RING(chan, RING_MF(OFFSET_OUT_HIGH), 2);
+      OUT_RELOCh(chan, dst, dstoff, dstdom | NOUVEAU_BO_WR);
+      OUT_RELOCl(chan, dst, dstoff, dstdom | NOUVEAU_BO_WR);
+      BEGIN_RING(chan, RING_MF(OFFSET_IN_HIGH), 2);
+      OUT_RELOCh(chan, src, srcoff, srcdom | NOUVEAU_BO_RD);
+      OUT_RELOCl(chan, src, srcoff, srcdom | NOUVEAU_BO_RD);
+      BEGIN_RING(chan, RING_MF(LINE_LENGTH_IN), 2);
+      OUT_RING  (chan, bytes);
+      OUT_RING  (chan, 1);
+      BEGIN_RING(chan, RING_MF(EXEC), 1);
+      OUT_RING  (chan, (1 << NVC0_M2MF_EXEC_INC__SHIFT) |
+                 NVC0_M2MF_EXEC_LINEAR_IN | NVC0_M2MF_EXEC_LINEAR_OUT);
+
+      srcoff += bytes;
+      dstoff += bytes;
+      size -= bytes;
+   }
+}
+
+static void
+nvc0_m2mf_push_rect(struct pipe_screen *pscreen,
+                    const struct nvc0_m2mf_rect *dst,
+                    const void *data,
+                    unsigned nblocksx, unsigned nblocksy)
+{
+   struct nouveau_channel *chan;
+   const uint8_t *src = (const uint8_t *)data;
+   const int cpp = dst->cpp;
+   const int line_len = nblocksx * cpp;
+   int dy = dst->y;
+
+   assert(nouveau_bo_tile_layout(dst->bo));
+
+   BEGIN_RING(chan, RING_MF(TILING_MODE_OUT), 5);
+   OUT_RING  (chan, dst->tile_mode);
+   OUT_RING  (chan, dst->width * cpp);
+   OUT_RING  (chan, dst->height);
+   OUT_RING  (chan, dst->depth);
+   OUT_RING  (chan, dst->z);
+
+   while (nblocksy) {
+      int line_count, words;
+      int size = MIN2(AVAIL_RING(chan), NV04_PFIFO_MAX_PACKET_LEN);
+
+      if (size < (12 + words)) {
+         FIRE_RING(chan);
+         continue;
+      }
+      line_count = (size * 4) / line_len;
+      words = (line_count * line_len + 3) / 4;
+
+      BEGIN_RING(chan, RING_MF(OFFSET_OUT_HIGH), 2);
+      OUT_RELOCh(chan, dst->bo, dst->base, dst->domain | NOUVEAU_BO_WR);
+      OUT_RELOCl(chan, dst->bo, dst->base, dst->domain | NOUVEAU_BO_WR);
+
+      BEGIN_RING(chan, RING_MF(TILING_POSITION_OUT_X), 2);
+      OUT_RING  (chan, dst->x * cpp);
+      OUT_RING  (chan, dy);
+      BEGIN_RING(chan, RING_MF(LINE_LENGTH_IN), 2);
+      OUT_RING  (chan, line_len);
+      OUT_RING  (chan, line_count);
+      BEGIN_RING(chan, RING_MF(EXEC), 1);
+      OUT_RING  (chan, (1 << NVC0_M2MF_EXEC_INC__SHIFT) |
+                 NVC0_M2MF_EXEC_PUSH | NVC0_M2MF_EXEC_LINEAR_IN);
+
+      BEGIN_RING_NI(chan, RING_MF(DATA), words);
+      OUT_RINGp (chan, src, words);
+
+      dy += line_count;
+      src += line_len * line_count;
+      nblocksy -= line_count;
+   }
+}
+
+struct pipe_transfer *
+nvc0_miptree_transfer_new(struct pipe_context *pctx,
+                          struct pipe_resource *res,
+                          unsigned level,
+                          unsigned usage,
+                          const struct pipe_box *box)
+{
+   struct nvc0_context *nvc0 = nvc0_context(pctx);
+   struct pipe_screen *pscreen = pctx->screen;
+   struct nouveau_device *dev = nvc0->screen->base.device;
+   struct nvc0_miptree *mt = nvc0_miptree(res);
+   struct nvc0_miptree_level *lvl = &mt->level[level];
+   struct nvc0_transfer *tx;
+   uint32_t size;
+   uint32_t w, h, d, z, layer;
+   int ret;
+
+   if (mt->layout_3d) {
+      z = box->z;
+      d = u_minify(res->depth0, level);
+      layer = 0;
+   } else {
+      z = 0;
+      d = 1;
+      layer = box->z;
+   }
+
+   tx = CALLOC_STRUCT(nvc0_transfer);
+   if (!tx)
+      return NULL;
+
+   pipe_resource_reference(&tx->base.resource, res);
+
+   tx->base.level = level;
+   tx->base.usage = usage;
+   tx->base.box = *box;
+
+   tx->nblocksx = util_format_get_nblocksx(res->format, box->width);
+   tx->nblocksy = util_format_get_nblocksy(res->format, box->height);
+
+   tx->base.stride = tx->nblocksx * util_format_get_blocksize(res->format);
+   tx->base.layer_stride = tx->nblocksy * tx->base.stride;
+
+   w = u_minify(res->width0, level);
+   h = u_minify(res->height0, level);
+
+   tx->rect[0].cpp = tx->rect[1].cpp = util_format_get_blocksize(res->format);
+
+   tx->rect[0].bo = mt->base.bo;
+   tx->rect[0].base = lvl->offset + layer * mt->layer_stride;
+   tx->rect[0].tile_mode = lvl->tile_mode;
+   tx->rect[0].x = util_format_get_nblocksx(res->format, box->x);
+   tx->rect[0].y = util_format_get_nblocksy(res->format, box->y);
+   tx->rect[0].z = z;
+   tx->rect[0].width = util_format_get_nblocksx(res->format, w);
+   tx->rect[0].height = util_format_get_nblocksy(res->format, h);
+   tx->rect[0].depth = d;
+   tx->rect[0].pitch = lvl->pitch;
+   tx->rect[0].domain = NOUVEAU_BO_VRAM;
+
+   size = tx->base.layer_stride;
+
+   ret = nouveau_bo_new(dev, NOUVEAU_BO_GART | NOUVEAU_BO_MAP, 0,
+                        size * tx->base.box.depth, &tx->rect[1].bo);
+   if (ret) {
+      FREE(tx);
+      return NULL;
+   }
+
+   tx->rect[1].width = tx->nblocksx;
+   tx->rect[1].height = tx->nblocksy;
+   tx->rect[1].depth = 1;
+   tx->rect[1].pitch = tx->base.stride;
+   tx->rect[1].domain = NOUVEAU_BO_GART;
+
+   if (usage & PIPE_TRANSFER_READ) {
+      unsigned i;
+      for (i = 0; i < box->depth; ++i) {
+         nvc0_m2mf_transfer_rect(pscreen, &tx->rect[1], &tx->rect[0],
+                                 tx->nblocksx, tx->nblocksy);
+         if (mt->layout_3d)
+            tx->rect[0].z++;
+         else
+            tx->rect[0].base += mt->layer_stride;
+         tx->rect[1].base += size;
+      }
+   }
+   tx->rect[0].z = z;
+   tx->rect[1].base = 0;
+
+   return &tx->base;
+}
+
+void
+nvc0_miptree_transfer_del(struct pipe_context *pctx,
+                          struct pipe_transfer *transfer)
+{
+   struct pipe_screen *pscreen = pctx->screen;
+   struct nvc0_transfer *tx = (struct nvc0_transfer *)transfer;
+   struct nvc0_miptree *mt = nvc0_miptree(tx->base.resource);
+   unsigned i;
+
+   if (tx->base.usage & PIPE_TRANSFER_WRITE) {
+      for (i = 0; i < tx->base.box.depth; ++i) {
+         nvc0_m2mf_transfer_rect(pscreen, &tx->rect[0], &tx->rect[1],
+                                 tx->nblocksx, tx->nblocksy);
+         if (mt->layout_3d)
+            tx->rect[0].z++;
+         else
+            tx->rect[0].base += mt->layer_stride;
+         tx->rect[1].base += tx->nblocksy * tx->base.stride;
+      }
+   }
+
+   nouveau_bo_ref(NULL, &tx->rect[1].bo);
+   pipe_resource_reference(&transfer->resource, NULL);
+
+   FREE(tx);
+}
+
+void *
+nvc0_miptree_transfer_map(struct pipe_context *pctx,
+                          struct pipe_transfer *transfer)
+{
+   struct nvc0_transfer *tx = (struct nvc0_transfer *)transfer;
+   int ret;
+   unsigned flags = 0;
+
+   if (tx->rect[1].bo->map)
+      return tx->rect[1].bo->map;
+
+   if (transfer->usage & PIPE_TRANSFER_READ)
+      flags = NOUVEAU_BO_RD;
+   if (transfer->usage & PIPE_TRANSFER_WRITE)
+      flags |= NOUVEAU_BO_WR;
+
+   ret = nouveau_bo_map(tx->rect[1].bo, flags);
+   if (ret)
+      return NULL;
+   return tx->rect[1].bo->map;
+}
+
+void
+nvc0_miptree_transfer_unmap(struct pipe_context *pctx,
+                            struct pipe_transfer *transfer)
+{
+   struct nvc0_transfer *tx = (struct nvc0_transfer *)transfer;
+
+   nouveau_bo_unmap(tx->rect[1].bo);
+}
+
diff --git a/src/gallium/drivers/nvc0/nvc0_transfer.h b/src/gallium/drivers/nvc0/nvc0_transfer.h
new file mode 100644
index 00000000000..222f72d2748
--- /dev/null
+++ b/src/gallium/drivers/nvc0/nvc0_transfer.h
@@ -0,0 +1,38 @@
+
+#ifndef __NVC0_TRANSFER_H__
+#define __NVC0_TRANSFER_H__
+
+#include "pipe/p_state.h"
+
+struct pipe_transfer *
+nvc0_miptree_transfer_new(struct pipe_context *pcontext,
+                          struct pipe_resource *pt,
+                          unsigned level,
+                          unsigned usage,
+                          const struct pipe_box *box);
+void
+nvc0_miptree_transfer_del(struct pipe_context *pcontext,
+                          struct pipe_transfer *ptx);
+void *
+nvc0_miptree_transfer_map(struct pipe_context *pcontext,
+                          struct pipe_transfer *ptx);
+void
+nvc0_miptree_transfer_unmap(struct pipe_context *pcontext,
+                            struct pipe_transfer *ptx);
+
+struct nvc0_m2mf_rect {
+   struct nouveau_bo *bo;
+   uint32_t base;
+   unsigned domain;
+   uint32_t pitch;
+   uint32_t width;
+   uint32_t x;
+   uint32_t height;
+   uint32_t y;
+   uint16_t depth;
+   uint16_t z;
+   uint16_t tile_mode;
+   uint16_t cpp;
+};
+
+#endif
diff --git a/src/gallium/drivers/nvc0/nvc0_vbo.c b/src/gallium/drivers/nvc0/nvc0_vbo.c
new file mode 100644
index 00000000000..881f0655506
--- /dev/null
+++ b/src/gallium/drivers/nvc0/nvc0_vbo.c
@@ -0,0 +1,593 @@
+/*
+ * Copyright 2010 Christoph Bumiller
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF
+ * OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "pipe/p_context.h"
+#include "pipe/p_state.h"
+#include "util/u_inlines.h"
+#include "util/u_format.h"
+#include "translate/translate.h"
+
+#include "nvc0_context.h"
+#include "nvc0_resource.h"
+
+#include "nvc0_3d.xml.h"
+
+void
+nvc0_vertex_state_delete(struct pipe_context *pipe,
+                         void *hwcso)
+{
+   struct nvc0_vertex_stateobj *so = hwcso;
+
+   if (so->translate)
+      so->translate->release(so->translate);
+   FREE(hwcso);
+}
+
+void *
+nvc0_vertex_state_create(struct pipe_context *pipe,
+                         unsigned num_elements,
+                         const struct pipe_vertex_element *elements)
+{
+    struct nvc0_vertex_stateobj *so;
+    struct translate_key transkey;
+    unsigned i;
+
+    assert(num_elements);
+
+    so = MALLOC(sizeof(*so) +
+                (num_elements - 1) * sizeof(struct nvc0_vertex_element));
+    if (!so)
+        return NULL;
+    so->num_elements = num_elements;
+    so->instance_bits = 0;
+
+    transkey.nr_elements = 0;
+    transkey.output_stride = 0;
+
+    for (i = 0; i < num_elements; ++i) {
+        const struct pipe_vertex_element *ve = &elements[i];
+        const unsigned vbi = ve->vertex_buffer_index;
+        enum pipe_format fmt = ve->src_format;
+
+        so->element[i].pipe = elements[i];
+        so->element[i].state = nvc0_format_table[fmt].vtx;
+
+        if (!so->element[i].state) {
+            switch (util_format_get_nr_components(fmt)) {
+            case 1: fmt = PIPE_FORMAT_R32_FLOAT; break;
+            case 2: fmt = PIPE_FORMAT_R32G32_FLOAT; break;
+            case 3: fmt = PIPE_FORMAT_R32G32B32_FLOAT; break;
+            case 4: fmt = PIPE_FORMAT_R32G32B32A32_FLOAT; break;
+            default:
+                assert(0);
+                return NULL;
+            }
+            so->element[i].state = nvc0_format_table[fmt].vtx;
+        }
+        so->element[i].state |= i;
+
+        if (likely(!ve->instance_divisor)) {
+            unsigned j = transkey.nr_elements++;
+
+            transkey.element[j].type = TRANSLATE_ELEMENT_NORMAL;
+            transkey.element[j].input_format = ve->src_format;
+            transkey.element[j].input_buffer = vbi;
+            transkey.element[j].input_offset = ve->src_offset;
+            transkey.element[j].instance_divisor = ve->instance_divisor;
+
+            transkey.element[j].output_format = fmt;
+            transkey.element[j].output_offset = transkey.output_stride;
+            transkey.output_stride += (util_format_get_stride(fmt, 1) + 3) & ~3;
+        } else {
+           so->instance_bits |= 1 << i;
+        }
+    }
+
+    so->translate = translate_create(&transkey);
+    so->vtx_size = transkey.output_stride / 4;
+    so->vtx_per_packet_max = NV04_PFIFO_MAX_PACKET_LEN / MAX2(so->vtx_size, 1);
+
+    return so;
+}
+
+#define NVC0_3D_VERTEX_ATTRIB_INACTIVE                                       \
+   NVC0_3D_VERTEX_ATTRIB_FORMAT_TYPE_FLOAT |                                 \
+   NVC0_3D_VERTEX_ATTRIB_FORMAT_SIZE_32 | NVC0_3D_VERTEX_ATTRIB_FORMAT_CONST
+
+#define VTX_ATTR(a, c, t, s)                            \
+   ((NVC0_3D_VTX_ATTR_DEFINE_TYPE_##t) |                \
+    (NVC0_3D_VTX_ATTR_DEFINE_SIZE_##s) |                \
+    ((a) << NVC0_3D_VTX_ATTR_DEFINE_ATTR__SHIFT) |      \
+    ((c) << NVC0_3D_VTX_ATTR_DEFINE_COMP__SHIFT))
+
+static void
+nvc0_emit_vtxattr(struct nvc0_context *nvc0, struct pipe_vertex_buffer *vb,
+                  struct pipe_vertex_element *ve, unsigned attr)
+{
+   const void *data;
+   struct nouveau_channel *chan = nvc0->screen->base.channel;
+   struct nvc0_resource *res = nvc0_resource(vb->buffer);
+   float v[4];
+   int i;
+   const unsigned nc = util_format_get_nr_components(ve->src_format);
+
+   data = nvc0_resource_map_offset(nvc0, res, vb->buffer_offset +
+                                   ve->src_offset, NOUVEAU_BO_RD);
+
+   util_format_read_4f(ve->src_format, v, 0, data, 0, 0, 0, 1, 1);
+
+   BEGIN_RING(chan, RING_3D(VTX_ATTR_DEFINE), nc + 1);
+   OUT_RING  (chan, VTX_ATTR(attr, nc, FLOAT, 32));
+   for (i = 0; i < nc; ++i)
+      OUT_RINGf(chan, v[i]);
+}
+
+void
+nvc0_vertex_arrays_validate(struct nvc0_context *nvc0)
+{
+   struct nouveau_channel *chan = nvc0->screen->base.channel;
+   struct nvc0_vertex_stateobj *vertex = nvc0->vertex;
+   struct pipe_vertex_buffer *vb;
+   struct nvc0_vertex_element *ve;
+   unsigned i;
+   boolean push = FALSE;
+
+   nvc0->vbo_fifo = 0;
+
+   for (i = 0; i < nvc0->num_vtxbufs; ++i) {
+      vb = &nvc0->vtxbuf[i];
+
+      if (!nvc0_resource_mapped_by_gpu(vb->buffer)) {
+         if (vb->stride == 0)
+            continue;
+         push = nvc0->vbo_push_hint;
+         if (!push) {
+            unsigned base, size;
+            base = vb->buffer_offset + nvc0->vbo_min_index * vb->stride;
+            size = (nvc0->vbo_max_index - nvc0->vbo_min_index + 1) * vb->stride;
+            nvc0_migrate_vertices(nvc0_resource(vb->buffer), base, size);
+            nvc0->vbo_dirty = TRUE;
+         } else
+            continue;
+      }
+      nvc0_buffer_adjust_score(nvc0, nvc0_resource(vb->buffer), 1);
+
+      nvc0_bufctx_add_resident(nvc0, NVC0_BUFCTX_VERTEX,
+                               nvc0_resource(vb->buffer), NOUVEAU_BO_RD);
+   }
+
+   BEGIN_RING(chan, RING_3D(VERTEX_ATTRIB_FORMAT(0)), vertex->num_elements);
+   for (i = 0; i < vertex->num_elements; ++i) {
+      ve = &vertex->element[i];
+      vb = &nvc0->vtxbuf[ve->pipe.vertex_buffer_index];
+
+      if (push)
+         nvc0->vbo_fifo |= 1 << i;
+
+      if (likely(vb->stride) || push) {
+         OUT_RING(chan, ve->state);
+      } else {
+         OUT_RING(chan, ve->state | NVC0_3D_VERTEX_ATTRIB_FORMAT_CONST);
+      }
+   }
+
+   for (i = 0; i < vertex->num_elements; ++i) {
+      struct nvc0_resource *res;
+      unsigned size, offset;
+      
+      ve = &vertex->element[i];
+      vb = &nvc0->vtxbuf[ve->pipe.vertex_buffer_index];
+
+      if (unlikely(ve->pipe.instance_divisor)) {
+         if (!(nvc0->state.instance_bits & (1 << i))) {
+            IMMED_RING(chan, RING_3D(VERTEX_ARRAY_PER_INSTANCE(i)), 1);
+         }
+         BEGIN_RING(chan, RING_3D(VERTEX_ARRAY_DIVISOR(i)), 1);
+         OUT_RING  (chan, ve->pipe.instance_divisor);
+      } else
+      if (unlikely(nvc0->state.instance_bits & (1 << i))) {
+         IMMED_RING(chan, RING_3D(VERTEX_ARRAY_PER_INSTANCE(i)), 0);
+      }
+
+      res = nvc0_resource(vb->buffer);
+
+      if (push || unlikely(vb->stride == 0)) {
+         if (!push)
+            nvc0_emit_vtxattr(nvc0, vb, &ve->pipe, i);
+         BEGIN_RING(chan, RING_3D(VERTEX_ARRAY_FETCH(i)), 1);
+         OUT_RING  (chan, 0);
+         continue;
+      }
+
+      size = vb->buffer->width0;
+      offset = ve->pipe.src_offset + vb->buffer_offset;
+
+      BEGIN_RING(chan, RING_3D(VERTEX_ARRAY_FETCH(i)), 1);
+      OUT_RING  (chan, (1 << 12) | vb->stride);
+      BEGIN_RING_1I(chan, RING_3D(VERTEX_ARRAY_SELECT), 5);
+      OUT_RING  (chan, i);
+      OUT_RESRCh(chan, res, size - 1, NOUVEAU_BO_RD);
+      OUT_RESRCl(chan, res, size - 1, NOUVEAU_BO_RD);
+      OUT_RESRCh(chan, res, offset, NOUVEAU_BO_RD);
+      OUT_RESRCl(chan, res, offset, NOUVEAU_BO_RD);
+   }
+   for (; i < nvc0->state.num_vtxelts; ++i) {
+      BEGIN_RING(chan, RING_3D(VERTEX_ATTRIB_FORMAT(i)), 1);
+      OUT_RING  (chan, NVC0_3D_VERTEX_ATTRIB_INACTIVE);
+      BEGIN_RING(chan, RING_3D(VERTEX_ARRAY_FETCH(i)), 1);
+      OUT_RING  (chan, 0);
+   }
+
+   nvc0->state.num_vtxelts = vertex->num_elements;
+   nvc0->state.instance_bits = vertex->instance_bits;
+}
+
+#define NVC0_PRIM_GL_CASE(n) \
+   case PIPE_PRIM_##n: return NVC0_3D_VERTEX_BEGIN_GL_PRIMITIVE_##n
+
+static INLINE unsigned
+nvc0_prim_gl(unsigned prim)
+{
+   switch (prim) {
+   NVC0_PRIM_GL_CASE(POINTS);
+   NVC0_PRIM_GL_CASE(LINES);
+   NVC0_PRIM_GL_CASE(LINE_LOOP);
+   NVC0_PRIM_GL_CASE(LINE_STRIP);
+   NVC0_PRIM_GL_CASE(TRIANGLES);
+   NVC0_PRIM_GL_CASE(TRIANGLE_STRIP);
+   NVC0_PRIM_GL_CASE(TRIANGLE_FAN);
+   NVC0_PRIM_GL_CASE(QUADS);
+   NVC0_PRIM_GL_CASE(QUAD_STRIP);
+   NVC0_PRIM_GL_CASE(POLYGON);
+   NVC0_PRIM_GL_CASE(LINES_ADJACENCY);
+   NVC0_PRIM_GL_CASE(LINE_STRIP_ADJACENCY);
+   NVC0_PRIM_GL_CASE(TRIANGLES_ADJACENCY);
+   NVC0_PRIM_GL_CASE(TRIANGLE_STRIP_ADJACENCY);
+   /*
+   NVC0_PRIM_GL_CASE(PATCHES); */
+   default:
+      return NVC0_3D_VERTEX_BEGIN_GL_PRIMITIVE_POINTS;
+      break;
+   }
+}
+
+static void
+nvc0_draw_vbo_flush_notify(struct nouveau_channel *chan)
+{
+   struct nvc0_context *nvc0 = chan->user_private;
+
+   nvc0_bufctx_emit_relocs(nvc0);
+}
+
+#if 0
+static struct nouveau_bo *
+nvc0_tfb_setup(struct nvc0_context *nvc0)
+{
+   struct nouveau_channel *chan = nvc0->screen->base.channel;
+   struct nouveau_bo *tfb = NULL;
+   int ret, i;
+
+   ret = nouveau_bo_new(nvc0->screen->base.device,
+                        NOUVEAU_BO_GART | NOUVEAU_BO_MAP, 0, 4096, &tfb);
+   if (ret)
+      return NULL;
+
+   ret = nouveau_bo_map(tfb, NOUVEAU_BO_WR);
+   if (ret)
+      return NULL;
+   memset(tfb->map, 0xee, 8 * 4 * 3);
+   nouveau_bo_unmap(tfb);
+
+   BEGIN_RING(chan, RING_3D(TFB_ENABLE), 1);
+   OUT_RING  (chan, 1);
+   BEGIN_RING(chan, RING_3D(TFB_BUFFER_ENABLE(0)), 5);
+   OUT_RING  (chan, 1);
+   OUT_RELOCh(chan, tfb, 0, NOUVEAU_BO_GART | NOUVEAU_BO_WR);
+   OUT_RELOCl(chan, tfb, 0, NOUVEAU_BO_GART | NOUVEAU_BO_WR);
+   OUT_RING  (chan, tfb->size);
+   OUT_RING  (chan, 0); /* TFB_PRIMITIVE_ID(0) */
+   BEGIN_RING(chan, RING_3D(TFB_UNK0700(0)), 3);
+   OUT_RING  (chan, 0);
+   OUT_RING  (chan, 8); /* TFB_VARYING_COUNT(0) */
+   OUT_RING  (chan, 32); /* TFB_BUFFER_STRIDE(0) */
+   BEGIN_RING(chan, RING_3D(TFB_VARYING_LOCS(0)), 2);
+   OUT_RING  (chan, 0x1f1e1d1c);
+   OUT_RING  (chan, 0xa3a2a1a0);
+   for (i = 1; i < 4; ++i) {
+      BEGIN_RING(chan, RING_3D(TFB_BUFFER_ENABLE(i)), 1);
+      OUT_RING  (chan, 0);
+   }
+   BEGIN_RING(chan, RING_3D(TFB_ENABLE), 1);
+   OUT_RING  (chan, 1);
+   BEGIN_RING(chan, RING_3D_(0x135c), 1);
+   OUT_RING  (chan, 1);
+   BEGIN_RING(chan, RING_3D_(0x135c), 1);
+   OUT_RING  (chan, 0);
+
+   return tfb;
+}
+#endif
+
+static void
+nvc0_draw_arrays(struct nvc0_context *nvc0,
+                 unsigned mode, unsigned start, unsigned count,
+                 unsigned instance_count)
+{
+   struct nouveau_channel *chan = nvc0->screen->base.channel;
+   unsigned prim;
+
+   chan->flush_notify = nvc0_draw_vbo_flush_notify;
+   chan->user_private = nvc0;
+
+   prim = nvc0_prim_gl(mode);
+
+   while (instance_count--) {
+      BEGIN_RING(chan, RING_3D(VERTEX_BEGIN_GL), 1);
+      OUT_RING  (chan, prim);
+      BEGIN_RING(chan, RING_3D(VERTEX_BUFFER_FIRST), 2);
+      OUT_RING  (chan, start);
+      OUT_RING  (chan, count);
+      IMMED_RING(chan, RING_3D(VERTEX_END_GL), 0);
+
+      prim |= NVC0_3D_VERTEX_BEGIN_GL_INSTANCE_NEXT;
+   }
+
+   chan->flush_notify = NULL;
+}
+
+static void
+nvc0_draw_elements_inline_u08(struct nouveau_channel *chan, uint8_t *map,
+                              unsigned start, unsigned count)
+{
+   map += start;
+
+   if (count & 3) {
+      unsigned i;
+      BEGIN_RING_NI(chan, RING_3D(VB_ELEMENT_U32), count & 3);
+      for (i = 0; i < (count & 3); ++i)
+         OUT_RING(chan, *map++);
+      count &= ~3;
+   }
+   while (count) {
+      unsigned i, nr = MIN2(count, NV04_PFIFO_MAX_PACKET_LEN * 4) / 4;
+
+      BEGIN_RING_NI(chan, RING_3D(VB_ELEMENT_U8), nr);
+      for (i = 0; i < nr; ++i) {
+         OUT_RING(chan,
+                  (map[3] << 24) | (map[2] << 16) | (map[1] << 8) | map[0]);
+         map += 4;
+      }
+      count -= nr * 4;
+   }
+}
+
+static void
+nvc0_draw_elements_inline_u16(struct nouveau_channel *chan, uint16_t *map,
+                              unsigned start, unsigned count)
+{
+   map += start;
+
+   if (count & 1) {
+      count &= ~1;
+      BEGIN_RING(chan, RING_3D(VB_ELEMENT_U32), 1);
+      OUT_RING  (chan, *map++);
+   }
+   while (count) {
+      unsigned i, nr = MIN2(count, NV04_PFIFO_MAX_PACKET_LEN * 2) / 2;
+
+      BEGIN_RING_NI(chan, RING_3D(VB_ELEMENT_U16), nr);
+      for (i = 0; i < nr; ++i) {
+         OUT_RING(chan, (map[1] << 16) | map[0]);
+         map += 2;
+      }
+      count -= nr * 2;
+   }
+}
+
+static void
+nvc0_draw_elements_inline_u32(struct nouveau_channel *chan, uint32_t *map,
+                              unsigned start, unsigned count)
+{
+   map += start;
+
+   while (count) {
+      const unsigned nr = MIN2(count, NV04_PFIFO_MAX_PACKET_LEN);
+
+      BEGIN_RING_NI(chan, RING_3D(VB_ELEMENT_U32), nr);
+      OUT_RINGp    (chan, map, nr);
+
+      map += nr;
+      count -= nr;
+   }
+}
+
+static void
+nvc0_draw_elements_inline_u32_short(struct nouveau_channel *chan, uint32_t *map,
+                                    unsigned start, unsigned count)
+{
+   map += start;
+
+   if (count & 1) {
+      count--;
+      BEGIN_RING(chan, RING_3D(VB_ELEMENT_U32), 1);
+      OUT_RING  (chan, *map++);
+   }
+   while (count) {
+      unsigned i, nr = MIN2(count, NV04_PFIFO_MAX_PACKET_LEN * 2) / 2;
+
+      BEGIN_RING_NI(chan, RING_3D(VB_ELEMENT_U16), nr);
+      for (i = 0; i < nr; ++i) {
+         OUT_RING(chan, (map[1] << 16) | map[0]);
+         map += 2;
+      }
+      count -= nr * 2;
+   }
+}
+
+static void
+nvc0_draw_elements(struct nvc0_context *nvc0, boolean shorten,
+                   unsigned mode, unsigned start, unsigned count,
+                   unsigned instance_count, int32_t index_bias)
+{
+   struct nouveau_channel *chan = nvc0->screen->base.channel;
+   void *data;
+   unsigned prim;
+   const unsigned index_size = nvc0->idxbuf.index_size;
+
+   chan->flush_notify = nvc0_draw_vbo_flush_notify;
+   chan->user_private = nvc0;
+
+   prim = nvc0_prim_gl(mode);
+
+   if (index_bias != nvc0->state.index_bias) {
+      BEGIN_RING(chan, RING_3D(VB_ELEMENT_BASE), 1);
+      OUT_RING  (chan, index_bias);
+      nvc0->state.index_bias = index_bias;
+   }
+
+   if (nvc0_resource_mapped_by_gpu(nvc0->idxbuf.buffer)) {
+      struct nvc0_resource *res = nvc0_resource(nvc0->idxbuf.buffer);
+      unsigned offset = nvc0->idxbuf.offset;
+      unsigned limit = nvc0->idxbuf.buffer->width0 - 1;
+
+      nvc0_buffer_adjust_score(nvc0, res, 1);
+
+      while (instance_count--) {
+         MARK_RING (chan, 11, 4);
+         BEGIN_RING(chan, RING_3D(VERTEX_BEGIN_GL), 1);
+         OUT_RING  (chan, mode);
+         BEGIN_RING(chan, RING_3D(INDEX_ARRAY_START_HIGH), 7);
+         OUT_RESRCh(chan, res, offset, NOUVEAU_BO_RD);
+         OUT_RESRCl(chan, res, offset, NOUVEAU_BO_RD);
+         OUT_RESRCh(chan, res, limit, NOUVEAU_BO_RD);
+         OUT_RESRCl(chan, res, limit, NOUVEAU_BO_RD);
+         OUT_RING  (chan, index_size >> 1);
+         OUT_RING  (chan, start);
+         OUT_RING  (chan, count);
+         IMMED_RING(chan, RING_3D(VERTEX_END_GL), 0);
+
+         nvc0_resource_fence(res, NOUVEAU_BO_RD);
+
+         mode |= NVC0_3D_VERTEX_BEGIN_GL_INSTANCE_NEXT;
+      }
+   } else {
+      data = nvc0_resource_map_offset(nvc0, nvc0_resource(nvc0->idxbuf.buffer),
+                                      nvc0->idxbuf.offset, NOUVEAU_BO_RD);
+      if (!data)
+         return;
+
+      while (instance_count--) {
+         BEGIN_RING(chan, RING_3D(VERTEX_BEGIN_GL), 1);
+         OUT_RING  (chan, prim);
+         switch (index_size) {
+         case 1:
+            nvc0_draw_elements_inline_u08(chan, data, start, count);
+            break;
+         case 2:
+            nvc0_draw_elements_inline_u16(chan, data, start, count);
+            break;
+         case 4:
+            if (shorten)
+               nvc0_draw_elements_inline_u32_short(chan, data, start, count);
+            else
+               nvc0_draw_elements_inline_u32(chan, data, start, count);
+            break;
+         default:
+            assert(0);
+            return;
+         }
+         IMMED_RING(chan, RING_3D(VERTEX_END_GL), 0);
+
+         prim |= NVC0_3D_VERTEX_BEGIN_GL_INSTANCE_NEXT;
+      }
+   }
+
+   chan->flush_notify = NULL;
+}
+
+void
+nvc0_draw_vbo(struct pipe_context *pipe, const struct pipe_draw_info *info)
+{
+   struct nvc0_context *nvc0 = nvc0_context(pipe);
+   struct nouveau_channel *chan = nvc0->screen->base.channel;
+
+   /* For picking only a few vertices from a large user buffer, push is better,
+    * if index count is larger and we expect repeated vertices, suggest upload.
+    */
+   nvc0->vbo_push_hint = /* the 64 is heuristic */
+      !(info->indexed &&
+        ((info->max_index - info->min_index + 64) < info->count));
+
+   nvc0->vbo_min_index = info->min_index;
+   nvc0->vbo_max_index = info->max_index;
+
+   nvc0_state_validate(nvc0);
+
+   if (nvc0->state.instance_base != info->start_instance) {
+      nvc0->state.instance_base = info->start_instance;
+      BEGIN_RING(chan, RING_3D(VB_INSTANCE_BASE), 1);
+      OUT_RING  (chan, info->start_instance);
+   }
+
+   if (nvc0->vbo_fifo) {
+      nvc0_push_vbo(nvc0, info);
+      return;
+   }
+
+   if (nvc0->vbo_dirty) {
+      BEGIN_RING(chan, RING_3D_(0x142c), 1);
+      OUT_RING  (chan, 0);
+      nvc0->vbo_dirty = FALSE;
+   }
+
+   if (!info->indexed) {
+      nvc0_draw_arrays(nvc0,
+                       info->mode, info->start, info->count,
+                       info->instance_count);
+   } else {
+      boolean shorten = info->max_index <= 65535;
+
+      assert(nvc0->idxbuf.buffer);
+
+      if (info->primitive_restart != nvc0->state.prim_restart) {
+         if (info->primitive_restart) {
+            BEGIN_RING(chan, RING_3D(PRIM_RESTART_ENABLE), 2);
+            OUT_RING  (chan, 1);
+            OUT_RING  (chan, info->restart_index);
+
+            if (info->restart_index > 65535)
+               shorten = FALSE;
+         } else {
+            IMMED_RING(chan, RING_3D(PRIM_RESTART_ENABLE), 0);
+         }
+         nvc0->state.prim_restart = info->primitive_restart;
+      } else
+      if (info->primitive_restart) {
+         BEGIN_RING(chan, RING_3D(PRIM_RESTART_INDEX), 1);
+         OUT_RING  (chan, info->restart_index);
+      }
+
+      nvc0_draw_elements(nvc0, shorten,
+                         info->mode, info->start, info->count,
+                         info->instance_count, info->index_bias);
+   }
+}
diff --git a/src/gallium/drivers/nvc0/nvc0_winsys.h b/src/gallium/drivers/nvc0/nvc0_winsys.h
new file mode 100644
index 00000000000..1544fb7a1de
--- /dev/null
+++ b/src/gallium/drivers/nvc0/nvc0_winsys.h
@@ -0,0 +1,120 @@
+
+#ifndef __NVC0_WINSYS_H__
+#define __NVC0_WINSYS_H__
+
+#include <stdint.h>
+#include <unistd.h>
+#include "pipe/p_defines.h"
+
+#include "nouveau/nouveau_bo.h"
+#include "nouveau/nouveau_channel.h"
+#include "nouveau/nouveau_grobj.h"
+#include "nouveau/nouveau_device.h"
+#include "nouveau/nouveau_resource.h"
+#include "nouveau/nouveau_pushbuf.h"
+#include "nouveau/nouveau_reloc.h"
+
+#include "nvc0_resource.h" /* OUT_RESRC */
+
+#ifndef NV04_PFIFO_MAX_PACKET_LEN
+#define NV04_PFIFO_MAX_PACKET_LEN 2047
+#endif
+
+#define NVC0_SUBCH_3D 1
+#define NVC0_SUBCH_2D 2
+#define NVC0_SUBCH_MF 3
+
+#define NVC0_MF_(n) NVC0_M2MF_##n
+
+#define RING_3D(n) ((NVC0_SUBCH_3D << 13) | (NVC0_3D_##n >> 2))
+#define RING_2D(n) ((NVC0_SUBCH_2D << 13) | (NVC0_2D_##n >> 2))
+#define RING_MF(n) ((NVC0_SUBCH_MF << 13) | (NVC0_MF_(n) >> 2))
+
+#define RING_3D_(m) ((NVC0_SUBCH_3D << 13) | ((m) >> 2))
+#define RING_2D_(m) ((NVC0_SUBCH_2D << 13) | ((m) >> 2))
+#define RING_MF_(m) ((NVC0_SUBCH_MF << 13) | ((m) >> 2))
+
+#define RING_GR(gr, m) (((gr)->subc << 13) | ((m) >> 2))
+
+int nouveau_pushbuf_flush(struct nouveau_channel *, unsigned min);
+
+static inline uint32_t
+nouveau_bo_tile_layout(struct nouveau_bo *bo)
+{
+   return bo->tile_flags & NOUVEAU_BO_TILE_LAYOUT_MASK;
+}
+
+static INLINE void
+nouveau_bo_validate(struct nouveau_channel *chan,
+                    struct nouveau_bo *bo, unsigned flags)
+{
+   nouveau_reloc_emit(chan, NULL, 0, NULL, bo, 0, 0, flags, 0, 0);
+}
+
+/* incremental methods */
+static INLINE void
+BEGIN_RING(struct nouveau_channel *chan, uint32_t mthd, unsigned size)
+{
+   WAIT_RING(chan, size + 1);
+   OUT_RING (chan, (0x2 << 28) | (size << 16) | mthd);
+}
+
+/* non-incremental */
+static INLINE void
+BEGIN_RING_NI(struct nouveau_channel *chan, uint32_t mthd, unsigned size)
+{
+   WAIT_RING(chan, size + 1);
+   OUT_RING (chan, (0x6 << 28) | (size << 16) | mthd);
+}
+
+/* increment-once */
+static INLINE void
+BEGIN_RING_1I(struct nouveau_channel *chan, uint32_t mthd, unsigned size)
+{
+   WAIT_RING(chan, size + 1);
+   OUT_RING (chan, (0xa << 28) | (size << 16) | mthd);
+}
+
+/* inline-data */
+static INLINE void
+IMMED_RING(struct nouveau_channel *chan, uint32_t mthd, unsigned data)
+{
+   WAIT_RING(chan, 1);
+   OUT_RING (chan, (0x8 << 28) | (data << 16) | mthd);
+}
+
+static INLINE int
+OUT_RESRCh(struct nouveau_channel *chan, struct nvc0_resource *res,
+           unsigned delta, unsigned flags)
+{
+   return OUT_RELOCh(chan, res->bo, res->offset + delta, res->domain | flags);
+}
+
+static INLINE int
+OUT_RESRCl(struct nouveau_channel *chan, struct nvc0_resource *res,
+           unsigned delta, unsigned flags)
+{
+   if (flags & NOUVEAU_BO_WR)
+      res->status |= NVC0_BUFFER_STATUS_DIRTY;
+   return OUT_RELOCl(chan, res->bo, res->offset + delta, res->domain | flags);
+}
+
+static INLINE void
+BIND_RING(struct nouveau_channel *chan, struct nouveau_grobj *gr, unsigned s)
+{
+   struct nouveau_subchannel *subc = &gr->channel->subc[s];
+
+   assert(s < 8);
+   if (subc->gr) {
+      assert(subc->gr->bound != NOUVEAU_GROBJ_BOUND_EXPLICIT);
+      subc->gr->bound = NOUVEAU_GROBJ_UNBOUND;
+   }
+   subc->gr = gr;
+   subc->gr->subc = s;
+   subc->gr->bound = NOUVEAU_GROBJ_BOUND_EXPLICIT;
+
+   BEGIN_RING(chan, RING_GR(gr, 0x0000), 1);
+   OUT_RING  (chan, gr->grclass);
+}
+
+#endif
diff --git a/src/gallium/drivers/nvfx/nv30_fragtex.c b/src/gallium/drivers/nvfx/nv30_fragtex.c
index 951fb202ed4..b609891d316 100644
--- a/src/gallium/drivers/nvfx/nv30_fragtex.c
+++ b/src/gallium/drivers/nvfx/nv30_fragtex.c
@@ -71,6 +71,7 @@ nv30_fragtex_set(struct nvfx_context *nvfx, int unit)
 	struct nvfx_sampler_view* sv = (struct nvfx_sampler_view*)nvfx->fragment_sampler_views[unit];
 	struct nouveau_bo *bo = ((struct nvfx_miptree *)sv->base.texture)->base.bo;
 	struct nouveau_channel* chan = nvfx->screen->base.channel;
+	struct nouveau_grobj *eng3d = nvfx->screen->eng3d;
 	unsigned txf;
 	unsigned tex_flags = NOUVEAU_BO_VRAM | NOUVEAU_BO_GART | NOUVEAU_BO_RD;
 	unsigned use_rect;
@@ -102,7 +103,7 @@ nv30_fragtex_set(struct nvfx_context *nvfx, int unit)
 	txf = sv->u.nv30.fmt[ps->compare + (use_rect ? 2 : 0)];
 
 	MARK_RING(chan, 9, 2);
-	OUT_RING(chan, RING_3D(NV30_3D_TEX_OFFSET(unit), 8));
+	BEGIN_RING(chan, eng3d, NV30_3D_TEX_OFFSET(unit), 8);
 	OUT_RELOC(chan, bo, sv->offset, tex_flags | NOUVEAU_BO_LOW, 0, 0);
 	OUT_RELOC(chan, bo, txf,
 		tex_flags | NOUVEAU_BO_OR,
diff --git a/src/gallium/drivers/nvfx/nv40_fragtex.c b/src/gallium/drivers/nvfx/nv40_fragtex.c
index e8ab403f722..563183d9d0c 100644
--- a/src/gallium/drivers/nvfx/nv40_fragtex.c
+++ b/src/gallium/drivers/nvfx/nv40_fragtex.c
@@ -76,6 +76,7 @@ void
 nv40_fragtex_set(struct nvfx_context *nvfx, int unit)
 {
 	struct nouveau_channel* chan = nvfx->screen->base.channel;
+	struct nouveau_grobj *eng3d = nvfx->screen->eng3d;
 	struct nvfx_sampler_state *ps = nvfx->tex_sampler[unit];
 	struct nvfx_sampler_view* sv = (struct nvfx_sampler_view*)nvfx->fragment_sampler_views[unit];
 	struct nouveau_bo *bo = ((struct nvfx_miptree *)sv->base.texture)->base.bo;
@@ -87,7 +88,7 @@ nv40_fragtex_set(struct nvfx_context *nvfx, int unit)
 	txf = sv->u.nv40.fmt[ps->compare] | ps->fmt;
 
 	MARK_RING(chan, 11, 2);
-	OUT_RING(chan, RING_3D(NV30_3D_TEX_OFFSET(unit), 8));
+	BEGIN_RING(chan, eng3d, NV30_3D_TEX_OFFSET(unit), 8);
 	OUT_RELOC(chan, bo, sv->offset, tex_flags | NOUVEAU_BO_LOW, 0, 0);
 	OUT_RELOC(chan, bo, txf, tex_flags | NOUVEAU_BO_OR,
 			NV30_3D_TEX_FORMAT_DMA0, NV30_3D_TEX_FORMAT_DMA1);
@@ -97,7 +98,7 @@ nv40_fragtex_set(struct nvfx_context *nvfx, int unit)
 	OUT_RING(chan, ps->filt | sv->filt);
 	OUT_RING(chan, sv->npot_size);
 	OUT_RING(chan, ps->bcol);
-	OUT_RING(chan, RING_3D(NV40_3D_TEX_SIZE1(unit), 1));
+	BEGIN_RING(chan, eng3d, NV40_3D_TEX_SIZE1(unit), 1);
 	OUT_RING(chan, sv->u.nv40.npot_size2);
 
 	nvfx->hw_txf[unit] = txf;
diff --git a/src/gallium/drivers/nvfx/nvfx_context.c b/src/gallium/drivers/nvfx/nvfx_context.c
index 95834d23273..6c8934d3a4a 100644
--- a/src/gallium/drivers/nvfx/nvfx_context.c
+++ b/src/gallium/drivers/nvfx/nvfx_context.c
@@ -13,13 +13,13 @@ nvfx_flush(struct pipe_context *pipe, unsigned flags,
 	struct nvfx_context *nvfx = nvfx_context(pipe);
 	struct nvfx_screen *screen = nvfx->screen;
 	struct nouveau_channel *chan = screen->base.channel;
+	struct nouveau_grobj *eng3d = screen->eng3d;
 
 	/* XXX: we need to actually be intelligent here */
 	if (flags & PIPE_FLUSH_TEXTURE_CACHE) {
-		WAIT_RING(chan, 4);
-		OUT_RING(chan, RING_3D(0x1fd8, 1));
+		BEGIN_RING(chan, eng3d, 0x1fd8, 1);
 		OUT_RING(chan, 2);
-		OUT_RING(chan, RING_3D(0x1fd8, 1));
+		BEGIN_RING(chan, eng3d, 0x1fd8, 1);
 		OUT_RING(chan, 1);
 	}
 
diff --git a/src/gallium/drivers/nvfx/nvfx_context.h b/src/gallium/drivers/nvfx/nvfx_context.h
index 6ef2a6945d7..2238aa1ad0e 100644
--- a/src/gallium/drivers/nvfx/nvfx_context.h
+++ b/src/gallium/drivers/nvfx/nvfx_context.h
@@ -339,30 +339,31 @@ extern void nvfx_init_vertprog_functions(struct nvfx_context *nvfx);
 /* nvfx_push.c */
 extern void nvfx_push_vbo(struct pipe_context *pipe, const struct pipe_draw_info *info);
 
-/* must WAIT_RING(chan, ncomp + 1) or equivalent beforehand! */
-static inline void nvfx_emit_vtx_attr(struct nouveau_channel* chan, unsigned attrib, const float* v, unsigned ncomp)
+static inline void nvfx_emit_vtx_attr(struct nouveau_channel* chan,
+		struct nouveau_grobj *eng3d, unsigned attrib, const float* v,
+		unsigned ncomp)
 {
 	switch (ncomp) {
 	case 4:
-		OUT_RING(chan, RING_3D(NV30_3D_VTX_ATTR_4F_X(attrib), 4));
+		BEGIN_RING(chan, eng3d, NV30_3D_VTX_ATTR_4F_X(attrib), 4);
 		OUT_RING(chan, fui(v[0]));
 		OUT_RING(chan, fui(v[1]));
 		OUT_RING(chan,  fui(v[2]));
 		OUT_RING(chan,  fui(v[3]));
 		break;
 	case 3:
-		OUT_RING(chan, RING_3D(NV30_3D_VTX_ATTR_3F_X(attrib), 3));
+		BEGIN_RING(chan, eng3d, NV30_3D_VTX_ATTR_3F_X(attrib), 3);
 		OUT_RING(chan,  fui(v[0]));
 		OUT_RING(chan,  fui(v[1]));
 		OUT_RING(chan,  fui(v[2]));
 		break;
 	case 2:
-		OUT_RING(chan, RING_3D(NV30_3D_VTX_ATTR_2F_X(attrib), 2));
+		BEGIN_RING(chan, eng3d, NV30_3D_VTX_ATTR_2F_X(attrib), 2);
 		OUT_RING(chan,  fui(v[0]));
 		OUT_RING(chan,  fui(v[1]));
 		break;
 	case 1:
-		OUT_RING(chan, RING_3D(NV30_3D_VTX_ATTR_1F(attrib), 1));
+		BEGIN_RING(chan, eng3d, NV30_3D_VTX_ATTR_1F(attrib), 1);
 		OUT_RING(chan,  fui(v[0]));
 		break;
 	}
diff --git a/src/gallium/drivers/nvfx/nvfx_draw.c b/src/gallium/drivers/nvfx/nvfx_draw.c
index 61f888a8ea2..81f1ec485d3 100644
--- a/src/gallium/drivers/nvfx/nvfx_draw.c
+++ b/src/gallium/drivers/nvfx/nvfx_draw.c
@@ -28,10 +28,10 @@ nvfx_render_flush(struct draw_stage *stage, unsigned flags)
 	struct nvfx_render_stage *rs = nvfx_render_stage(stage);
 	struct nvfx_context *nvfx = rs->nvfx;
 	struct nouveau_channel *chan = nvfx->screen->base.channel;
+	struct nouveau_grobj *eng3d = nvfx->screen->eng3d;
 
 	if (rs->prim != NV30_3D_VERTEX_BEGIN_END_STOP) {
-		assert(AVAIL_RING(chan) >= 2);
-		OUT_RING(chan, RING_3D(NV30_3D_VERTEX_BEGIN_END, 1));
+		BEGIN_RING(chan, eng3d, NV30_3D_VERTEX_BEGIN_END, 1);
 		OUT_RING(chan, NV30_3D_VERTEX_BEGIN_END_STOP);
 		rs->prim = NV30_3D_VERTEX_BEGIN_END_STOP;
 	}
@@ -46,6 +46,7 @@ nvfx_render_prim(struct draw_stage *stage, struct prim_header *prim,
 
 	struct nvfx_screen *screen = nvfx->screen;
 	struct nouveau_channel *chan = screen->base.channel;
+	struct nouveau_grobj *eng3d = screen->eng3d;
 	boolean no_elements = nvfx->vertprog->draw_no_elements;
 	unsigned num_attribs = nvfx->vertprog->draw_elements;
 
@@ -63,7 +64,7 @@ nvfx_render_prim(struct draw_stage *stage, struct prim_header *prim,
 	/* Switch primitive modes if necessary */
 	if (rs->prim != mode) {
 		if (rs->prim != NV30_3D_VERTEX_BEGIN_END_STOP) {
-			OUT_RING(chan, RING_3D(NV30_3D_VERTEX_BEGIN_END, 1));
+			BEGIN_RING(chan, eng3d, NV30_3D_VERTEX_BEGIN_END, 1);
 			OUT_RING(chan, NV30_3D_VERTEX_BEGIN_END_STOP);
 		}
 
@@ -74,23 +75,24 @@ nvfx_render_prim(struct draw_stage *stage, struct prim_header *prim,
 			int i;
 			for(i = 0; i < 32; ++i)
 			{
-				OUT_RING(chan, RING_3D(0x1dac, 1));
+				BEGIN_RING(chan, eng3d, 0x1dac, 1);
 				OUT_RING(chan, 0);
 			}
 		}
 
-		OUT_RING(chan, RING_3D(NV30_3D_VERTEX_BEGIN_END, 1));
+		BEGIN_RING(chan, eng3d, NV30_3D_VERTEX_BEGIN_END, 1);
 		OUT_RING  (chan, mode);
 		rs->prim = mode;
 	}
 
-	OUT_RING(chan, RING_3D_NI(NV30_3D_VERTEX_DATA, num_attribs * 4 * count));
 	if(no_elements) {
+		BEGIN_RING_NI(chan, eng3d, NV30_3D_VERTEX_DATA, 4);
 		OUT_RING(chan, 0);
 		OUT_RING(chan, 0);
 		OUT_RING(chan, 0);
 		OUT_RING(chan, 0);
 	} else {
+		BEGIN_RING_NI(chan, eng3d, NV30_3D_VERTEX_DATA, num_attribs * 4 * count);
 		for (unsigned i = 0; i < count; ++i)
 		{
 			struct vertex_header* v = prim->v[i];
diff --git a/src/gallium/drivers/nvfx/nvfx_fragprog.c b/src/gallium/drivers/nvfx/nvfx_fragprog.c
index 1740d72a8ae..dbd7c773465 100644
--- a/src/gallium/drivers/nvfx/nvfx_fragprog.c
+++ b/src/gallium/drivers/nvfx/nvfx_fragprog.c
@@ -1233,6 +1233,7 @@ void
 nvfx_fragprog_validate(struct nvfx_context *nvfx)
 {
 	struct nouveau_channel* chan = nvfx->screen->base.channel;
+	struct nouveau_grobj *eng3d = nvfx->screen->eng3d;
 	struct nvfx_pipe_fragment_program *pfp = nvfx->fragprog;
 	struct nvfx_vertex_program* vp;
 
@@ -1499,17 +1500,17 @@ update:
 		nvfx->hw_fragprog = fp;
 
 		MARK_RING(chan, 8, 1);
-		OUT_RING(chan, RING_3D(NV30_3D_FP_ACTIVE_PROGRAM, 1));
+		BEGIN_RING(chan, eng3d, NV30_3D_FP_ACTIVE_PROGRAM, 1);
 		OUT_RELOC(chan, fp->fpbo->bo, offset, NOUVEAU_BO_VRAM |
 			      NOUVEAU_BO_GART | NOUVEAU_BO_RD | NOUVEAU_BO_LOW |
 			      NOUVEAU_BO_OR, NV30_3D_FP_ACTIVE_PROGRAM_DMA0,
 			      NV30_3D_FP_ACTIVE_PROGRAM_DMA1);
-		OUT_RING(chan, RING_3D(NV30_3D_FP_CONTROL, 1));
+		BEGIN_RING(chan, eng3d, NV30_3D_FP_CONTROL, 1);
 		OUT_RING(chan, fp->fp_control);
 		if(!nvfx->is_nv4x) {
-			OUT_RING(chan, RING_3D(NV30_3D_FP_REG_CONTROL, 1));
+			BEGIN_RING(chan, eng3d, NV30_3D_FP_REG_CONTROL, 1);
 			OUT_RING(chan, (1<<16)|0x4);
-			OUT_RING(chan, RING_3D(NV30_3D_TEX_UNITS_ENABLE, 1));
+			BEGIN_RING(chan, eng3d, NV30_3D_TEX_UNITS_ENABLE, 1);
 			OUT_RING(chan, fp->samplers);
 		}
 	}
@@ -1518,8 +1519,7 @@ update:
 		unsigned pointsprite_control = fp->point_sprite_control | nvfx->rasterizer->pipe.point_quad_rasterization;
 		if(pointsprite_control != nvfx->hw_pointsprite_control)
 		{
-			WAIT_RING(chan, 2);
-			OUT_RING(chan, RING_3D(NV30_3D_POINT_SPRITE, 1));
+			BEGIN_RING(chan, eng3d, NV30_3D_POINT_SPRITE, 1);
 			OUT_RING(chan, pointsprite_control);
 			nvfx->hw_pointsprite_control = pointsprite_control;
 		}
diff --git a/src/gallium/drivers/nvfx/nvfx_fragtex.c b/src/gallium/drivers/nvfx/nvfx_fragtex.c
index fd0aff6a1a0..1c4901df0e2 100644
--- a/src/gallium/drivers/nvfx/nvfx_fragtex.c
+++ b/src/gallium/drivers/nvfx/nvfx_fragtex.c
@@ -177,6 +177,7 @@ void
 nvfx_fragtex_validate(struct nvfx_context *nvfx)
 {
 	struct nouveau_channel* chan = nvfx->screen->base.channel;
+	struct nouveau_grobj *eng3d = nvfx->screen->eng3d;
 	unsigned samplers, unit;
 
 	samplers = nvfx->dirty_samplers;
@@ -197,9 +198,8 @@ nvfx_fragtex_validate(struct nvfx_context *nvfx)
 			else
 				nv40_fragtex_set(nvfx, unit);
 		} else {
-			WAIT_RING(chan, 2);
 			/* this is OK for nv40 too */
-			OUT_RING(chan, RING_3D(NV30_3D_TEX_ENABLE(unit), 1));
+			BEGIN_RING(chan, eng3d, NV30_3D_TEX_ENABLE(unit), 1);
 			OUT_RING(chan, 0);
 			nvfx->hw_samplers &= ~(1 << unit);
 		}
diff --git a/src/gallium/drivers/nvfx/nvfx_push.c b/src/gallium/drivers/nvfx/nvfx_push.c
index ebf47e6ed30..6391741a2e5 100644
--- a/src/gallium/drivers/nvfx/nvfx_push.c
+++ b/src/gallium/drivers/nvfx/nvfx_push.c
@@ -10,6 +10,7 @@
 
 struct push_context {
 	struct nouveau_channel* chan;
+	struct nouveau_grobj *eng3d;
 
 	void *idxbuf;
 	int32_t idxbias;
@@ -27,9 +28,10 @@ static void
 emit_edgeflag(void *priv, boolean enabled)
 {
 	struct push_context* ctx = priv;
+	struct nouveau_grobj *eng3d = ctx->eng3d;
 	struct nouveau_channel *chan = ctx->chan;
 
-	OUT_RING(chan, RING_3D(NV30_3D_EDGEFLAG, 1));
+	BEGIN_RING(chan, eng3d, NV30_3D_EDGEFLAG, 1);
 	OUT_RING(chan, enabled ? 1 : 0);
 }
 
@@ -37,6 +39,7 @@ static void
 emit_vertices_lookup8(void *priv, unsigned start, unsigned count)
 {
         struct push_context *ctx = priv;
+        struct nouveau_grobj *eng3d = ctx->eng3d;
         uint8_t* elts = (uint8_t*)ctx->idxbuf + start;
 
         while(count)
@@ -44,7 +47,7 @@ emit_vertices_lookup8(void *priv, unsigned start, unsigned count)
                 unsigned push = MIN2(count, ctx->max_vertices_per_packet);
                 unsigned length = push * ctx->vertex_length;
 
-                OUT_RING(ctx->chan, RING_3D_NI(NV30_3D_VERTEX_DATA, length));
+                BEGIN_RING_NI(ctx->chan, eng3d, NV30_3D_VERTEX_DATA, length);
                 ctx->translate->run_elts8(ctx->translate, elts, push, 0, ctx->chan->cur);
                 ctx->chan->cur += length;
 
@@ -57,6 +60,7 @@ static void
 emit_vertices_lookup16(void *priv, unsigned start, unsigned count)
 {
 	struct push_context *ctx = priv;
+	struct nouveau_grobj *eng3d = ctx->eng3d;
         uint16_t* elts = (uint16_t*)ctx->idxbuf + start;
 
         while(count)
@@ -64,7 +68,7 @@ emit_vertices_lookup16(void *priv, unsigned start, unsigned count)
                 unsigned push = MIN2(count, ctx->max_vertices_per_packet);
                 unsigned length = push * ctx->vertex_length;
 
-                OUT_RING(ctx->chan, RING_3D_NI(NV30_3D_VERTEX_DATA, length));
+                BEGIN_RING_NI(ctx->chan, eng3d, NV30_3D_VERTEX_DATA, length);
                 ctx->translate->run_elts16(ctx->translate, elts, push, 0, ctx->chan->cur);
                 ctx->chan->cur += length;
 
@@ -77,6 +81,7 @@ static void
 emit_vertices_lookup32(void *priv, unsigned start, unsigned count)
 {
         struct push_context *ctx = priv;
+        struct nouveau_grobj *eng3d = ctx->eng3d;
         uint32_t* elts = (uint32_t*)ctx->idxbuf + start;
 
         while(count)
@@ -84,7 +89,7 @@ emit_vertices_lookup32(void *priv, unsigned start, unsigned count)
                 unsigned push = MIN2(count, ctx->max_vertices_per_packet);
                 unsigned length = push * ctx->vertex_length;
 
-                OUT_RING(ctx->chan, RING_3D_NI(NV30_3D_VERTEX_DATA, length));
+                BEGIN_RING_NI(ctx->chan, eng3d, NV30_3D_VERTEX_DATA, length);
                 ctx->translate->run_elts(ctx->translate, elts, push, 0, ctx->chan->cur);
                 ctx->chan->cur += length;
 
@@ -97,13 +102,14 @@ static void
 emit_vertices(void *priv, unsigned start, unsigned count)
 {
         struct push_context *ctx = priv;
+        struct nouveau_grobj *eng3d = ctx->eng3d;
 
         while(count)
         {
 		unsigned push = MIN2(count, ctx->max_vertices_per_packet);
 		unsigned length = push * ctx->vertex_length;
 
-		OUT_RING(ctx->chan, RING_3D_NI(NV30_3D_VERTEX_DATA, length));
+		BEGIN_RING_NI(ctx->chan, eng3d, NV30_3D_VERTEX_DATA, length);
 		ctx->translate->run(ctx->translate, start, push, 0, ctx->chan->cur);
 		ctx->chan->cur += length;
 
@@ -116,10 +122,11 @@ static void
 emit_ranges(void* priv, unsigned start, unsigned vc, unsigned reg)
 {
 	struct push_context* ctx = priv;
+	struct nouveau_grobj *eng3d = ctx->eng3d;
 	struct nouveau_channel *chan = ctx->chan;
 	unsigned nr = (vc & 0xff);
 	if (nr) {
-		OUT_RING(chan, RING_3D(reg, 1));
+		BEGIN_RING(chan, eng3d, reg, 1);
 		OUT_RING  (chan, ((nr - 1) << 24) | start);
 		start += nr;
 	}
@@ -130,7 +137,7 @@ emit_ranges(void* priv, unsigned start, unsigned vc, unsigned reg)
 
 		nr -= push;
 
-		OUT_RING(chan, RING_3D_NI(reg, push));
+		BEGIN_RING_NI(chan, eng3d, reg, push);
 		while (push--) {
 			OUT_RING(chan, ((0x100 - 1) << 24) | start);
 			start += 0x100;
@@ -154,12 +161,13 @@ static INLINE void
 emit_elt8(void* priv, unsigned start, unsigned vc)
 {
 	struct push_context* ctx = priv;
+	struct nouveau_grobj *eng3d = ctx->eng3d;
 	struct nouveau_channel *chan = ctx->chan;
 	uint8_t *elts = (uint8_t *)ctx->idxbuf + start;
 	int idxbias = ctx->idxbias;
 
 	if (vc & 1) {
-		OUT_RING(chan, RING_3D(NV30_3D_VB_ELEMENT_U32, 1));
+		BEGIN_RING(chan, eng3d, NV30_3D_VB_ELEMENT_U32, 1);
 		OUT_RING  (chan, elts[0]);
 		elts++; vc--;
 	}
@@ -168,7 +176,7 @@ emit_elt8(void* priv, unsigned start, unsigned vc)
 		unsigned i;
 		unsigned push = MIN2(vc, 2047 * 2);
 
-		OUT_RING(chan, RING_3D_NI(NV30_3D_VB_ELEMENT_U16, push >> 1));
+		BEGIN_RING_NI(chan, eng3d, NV30_3D_VB_ELEMENT_U16, push >> 1);
 		for (i = 0; i < push; i+=2)
 			OUT_RING(chan, ((elts[i+1] + idxbias) << 16) | (elts[i] + idxbias));
 
@@ -181,12 +189,13 @@ static INLINE void
 emit_elt16(void* priv, unsigned start, unsigned vc)
 {
 	struct push_context* ctx = priv;
+	struct nouveau_grobj *eng3d = ctx->eng3d;
 	struct nouveau_channel *chan = ctx->chan;
 	uint16_t *elts = (uint16_t *)ctx->idxbuf + start;
 	int idxbias = ctx->idxbias;
 
 	if (vc & 1) {
-		OUT_RING(chan, RING_3D(NV30_3D_VB_ELEMENT_U32, 1));
+		BEGIN_RING(chan, eng3d, NV30_3D_VB_ELEMENT_U32, 1);
 		OUT_RING  (chan, elts[0]);
 		elts++; vc--;
 	}
@@ -195,7 +204,7 @@ emit_elt16(void* priv, unsigned start, unsigned vc)
 		unsigned i;
 		unsigned push = MIN2(vc, 2047 * 2);
 
-		OUT_RING(chan, RING_3D_NI(NV30_3D_VB_ELEMENT_U16, push >> 1));
+		BEGIN_RING_NI(chan, eng3d, NV30_3D_VB_ELEMENT_U16, push >> 1);
 		for (i = 0; i < push; i+=2)
 			OUT_RING(chan, ((elts[i+1] + idxbias) << 16) | (elts[i] + idxbias));
 
@@ -208,6 +217,7 @@ static INLINE void
 emit_elt32(void* priv, unsigned start, unsigned vc)
 {
 	struct push_context* ctx = priv;
+	struct nouveau_grobj *eng3d = ctx->eng3d;
 	struct nouveau_channel *chan = ctx->chan;
 	uint32_t *elts = (uint32_t *)ctx->idxbuf + start;
 	int idxbias = ctx->idxbias;
@@ -215,8 +225,7 @@ emit_elt32(void* priv, unsigned start, unsigned vc)
 	while (vc) {
 		unsigned push = MIN2(vc, 2047);
 
-		OUT_RING(chan, RING_3D_NI(NV30_3D_VB_ELEMENT_U32, push));
-		assert(AVAIL_RING(chan) >= push);
+		BEGIN_RING_NI(chan, eng3d, NV30_3D_VB_ELEMENT_U32, push);
 		if(idxbias)
 		{
 			for(unsigned i = 0; i < push; ++i)
@@ -235,6 +244,7 @@ nvfx_push_vbo(struct pipe_context *pipe, const struct pipe_draw_info *info)
 {
 	struct nvfx_context *nvfx = nvfx_context(pipe);
 	struct nouveau_channel *chan = nvfx->screen->base.channel;
+	struct nouveau_grobj *eng3d = nvfx->screen->eng3d;
 	struct push_context ctx;
 	struct util_split_prim s;
 	unsigned instances_left = info->instance_count;
@@ -251,6 +261,7 @@ nvfx_push_vbo(struct pipe_context *pipe, const struct pipe_draw_info *info)
 			+ 4; /* potential edgeflag enable/disable */
 
 	ctx.chan = nvfx->screen->base.channel;
+	ctx.eng3d = nvfx->screen->eng3d;
 	ctx.translate = nvfx->vtxelt->translate;
 	ctx.idxbuf = NULL;
 	ctx.vertex_length = nvfx->vtxelt->vertex_length;
@@ -333,8 +344,9 @@ nvfx_push_vbo(struct pipe_context *pipe, const struct pipe_draw_info *info)
 
 		nvfx->vtxelt->per_instance[i].base.fetch_rgba_float(v, per_instance[i].map, 0, 0);
 
-		WAIT_RING(chan, 5);
-		nvfx_emit_vtx_attr(chan, nvfx->vtxelt->per_instance[i].base.idx, v, nvfx->vtxelt->per_instance[i].base.ncomp);
+		nvfx_emit_vtx_attr(chan, eng3d,
+				   nvfx->vtxelt->per_instance[i].base.idx, v,
+				   nvfx->vtxelt->per_instance[i].base.ncomp);
 	}
 
 	/* per-instance loop */
@@ -374,15 +386,18 @@ nvfx_push_vbo(struct pipe_context *pipe, const struct pipe_draw_info *info)
 					int i;
 					for(i = 0; i < 32; ++i)
 					{
-						OUT_RING(chan, RING_3D(0x1dac, 1));
+						BEGIN_RING(chan, eng3d,
+							   0x1dac, 1);
 						OUT_RING(chan, 0);
 					}
 				}
 
-				OUT_RING(chan, RING_3D(NV30_3D_VERTEX_BEGIN_END, 1));
+				BEGIN_RING(chan, eng3d,
+					   NV30_3D_VERTEX_BEGIN_END, 1);
 				OUT_RING(chan, hw_mode);
 				done = util_split_prim_next(&s, max_verts);
-				OUT_RING(chan, RING_3D(NV30_3D_VERTEX_BEGIN_END, 1));
+				BEGIN_RING(chan, eng3d,
+					   NV30_3D_VERTEX_BEGIN_END, 1);
 				OUT_RING(chan, 0);
 
 				if(done)
@@ -406,8 +421,10 @@ nvfx_push_vbo(struct pipe_context *pipe, const struct pipe_draw_info *info)
 				per_instance[i].step = 0;
 
 				nvfx->vtxelt->per_instance[i].base.fetch_rgba_float(v, per_instance[i].map, 0, 0);
-				WAIT_RING(chan, 5);
-				nvfx_emit_vtx_attr(chan, nvfx->vtxelt->per_instance[i].base.idx, v, nvfx->vtxelt->per_instance[i].base.ncomp);
+				nvfx_emit_vtx_attr(chan, eng3d,
+						   nvfx->vtxelt->per_instance[i].base.idx,
+						   v,
+						   nvfx->vtxelt->per_instance[i].base.ncomp);
 			}
 		}
 	}
diff --git a/src/gallium/drivers/nvfx/nvfx_query.c b/src/gallium/drivers/nvfx/nvfx_query.c
index 3935ffd7f92..3cd6bf1e477 100644
--- a/src/gallium/drivers/nvfx/nvfx_query.c
+++ b/src/gallium/drivers/nvfx/nvfx_query.c
@@ -49,6 +49,7 @@ nvfx_query_begin(struct pipe_context *pipe, struct pipe_query *pq)
 	struct nvfx_query *q = nvfx_query(pq);
 	struct nvfx_screen *screen = nvfx->screen;
 	struct nouveau_channel *chan = screen->base.channel;
+	struct nouveau_grobj *eng3d = screen->eng3d;
 	uint64_t tmp;
 
 	assert(!nvfx->query);
@@ -72,10 +73,9 @@ nvfx_query_begin(struct pipe_context *pipe, struct pipe_query *pq)
 
 	nouveau_notifier_reset(nvfx->screen->query, q->object->start);
 
-	WAIT_RING(chan, 4);
-	OUT_RING(chan, RING_3D(NV30_3D_QUERY_RESET, 1));
+	BEGIN_RING(chan, eng3d, NV30_3D_QUERY_RESET, 1);
 	OUT_RING(chan, 1);
-	OUT_RING(chan, RING_3D(NV30_3D_QUERY_ENABLE, 1));
+	BEGIN_RING(chan, eng3d, NV30_3D_QUERY_ENABLE, 1);
 	OUT_RING(chan, 1);
 
 	q->ready = FALSE;
@@ -88,15 +88,15 @@ nvfx_query_end(struct pipe_context *pipe, struct pipe_query *pq)
 {
 	struct nvfx_context *nvfx = nvfx_context(pipe);
 	struct nouveau_channel *chan = nvfx->screen->base.channel;
+	struct nouveau_grobj *eng3d = nvfx->screen->eng3d;
 	struct nvfx_query *q = nvfx_query(pq);
 
 	assert(nvfx->query == pq);
 
-	WAIT_RING(chan, 4);
-	OUT_RING(chan, RING_3D(NV30_3D_QUERY_GET, 1));
+	BEGIN_RING(chan, eng3d, NV30_3D_QUERY_GET, 1);
 	OUT_RING  (chan, (0x01 << NV30_3D_QUERY_GET_UNK24__SHIFT) |
 		   ((q->object->start * 32) << NV30_3D_QUERY_GET_OFFSET__SHIFT));
-	OUT_RING(chan, RING_3D(NV30_3D_QUERY_ENABLE, 1));
+	BEGIN_RING(chan, eng3d, NV30_3D_QUERY_ENABLE, 1);
 	OUT_RING(chan, 0);
 	FIRE_RING(chan);
 
diff --git a/src/gallium/drivers/nvfx/nvfx_screen.c b/src/gallium/drivers/nvfx/nvfx_screen.c
index 273fac76a9f..15db574a1d0 100644
--- a/src/gallium/drivers/nvfx/nvfx_screen.c
+++ b/src/gallium/drivers/nvfx/nvfx_screen.c
@@ -302,98 +302,100 @@ nvfx_screen_destroy(struct pipe_screen *pscreen)
 static void nv30_screen_init(struct nvfx_screen *screen)
 {
 	struct nouveau_channel *chan = screen->base.channel;
+	struct nouveau_grobj *eng3d = screen->eng3d;
 	int i;
 
 	/* TODO: perhaps we should do some of this on nv40 too? */
 	for (i=1; i<8; i++) {
-		OUT_RING(chan, RING_3D(NV30_3D_VIEWPORT_CLIP_HORIZ(i), 1));
+		BEGIN_RING(chan, eng3d, NV30_3D_VIEWPORT_CLIP_HORIZ(i), 1);
 		OUT_RING(chan, 0);
-		OUT_RING(chan, RING_3D(NV30_3D_VIEWPORT_CLIP_VERT(i), 1));
+		BEGIN_RING(chan, eng3d, NV30_3D_VIEWPORT_CLIP_VERT(i), 1);
 		OUT_RING(chan, 0);
 	}
 
-	OUT_RING(chan, RING_3D(0x220, 1));
+	BEGIN_RING(chan, eng3d, 0x220, 1);
 	OUT_RING(chan, 1);
 
-	OUT_RING(chan, RING_3D(0x03b0, 1));
+	BEGIN_RING(chan, eng3d, 0x03b0, 1);
 	OUT_RING(chan, 0x00100000);
-	OUT_RING(chan, RING_3D(0x1454, 1));
+	BEGIN_RING(chan, eng3d, 0x1454, 1);
 	OUT_RING(chan, 0);
-	OUT_RING(chan, RING_3D(0x1d80, 1));
+	BEGIN_RING(chan, eng3d, 0x1d80, 1);
 	OUT_RING(chan, 3);
-	OUT_RING(chan, RING_3D(0x1450, 1));
+	BEGIN_RING(chan, eng3d, 0x1450, 1);
 	OUT_RING(chan, 0x00030004);
 
 	/* NEW */
-	OUT_RING(chan, RING_3D(0x1e98, 1));
+	BEGIN_RING(chan, eng3d, 0x1e98, 1);
 	OUT_RING(chan, 0);
-	OUT_RING(chan, RING_3D(0x17e0, 3));
+	BEGIN_RING(chan, eng3d, 0x17e0, 3);
 	OUT_RING(chan, fui(0.0));
 	OUT_RING(chan, fui(0.0));
 	OUT_RING(chan, fui(1.0));
-	OUT_RING(chan, RING_3D(0x1f80, 16));
+	BEGIN_RING(chan, eng3d, 0x1f80, 16);
 	for (i=0; i<16; i++) {
 		OUT_RING(chan, (i==8) ? 0x0000ffff : 0);
 	}
 
-	OUT_RING(chan, RING_3D(0x120, 3));
+	BEGIN_RING(chan, eng3d, 0x120, 3);
 	OUT_RING(chan, 0);
 	OUT_RING(chan, 1);
 	OUT_RING(chan, 2);
 
-	OUT_RING(chan, RING_3D(0x1d88, 1));
+	BEGIN_RING(chan, eng3d, 0x1d88, 1);
 	OUT_RING(chan, 0x00001200);
 
-	OUT_RING(chan, RING_3D(NV30_3D_RC_ENABLE, 1));
+	BEGIN_RING(chan, eng3d, NV30_3D_RC_ENABLE, 1);
 	OUT_RING(chan, 0);
 
-	OUT_RING(chan, RING_3D(NV30_3D_DEPTH_RANGE_NEAR, 2));
+	BEGIN_RING(chan, eng3d, NV30_3D_DEPTH_RANGE_NEAR, 2);
 	OUT_RING(chan, fui(0.0));
 	OUT_RING(chan, fui(1.0));
 
-	OUT_RING(chan, RING_3D(NV30_3D_MULTISAMPLE_CONTROL, 1));
+	BEGIN_RING(chan, eng3d, NV30_3D_MULTISAMPLE_CONTROL, 1);
 	OUT_RING(chan, 0xffff0000);
 
 	/* enables use of vp rather than fixed-function somehow */
-	OUT_RING(chan, RING_3D(0x1e94, 1));
+	BEGIN_RING(chan, eng3d, 0x1e94, 1);
 	OUT_RING(chan, 0x13);
 }
 
 static void nv40_screen_init(struct nvfx_screen *screen)
 {
 	struct nouveau_channel *chan = screen->base.channel;
+	struct nouveau_grobj *eng3d = screen->eng3d;
 
-	OUT_RING(chan, RING_3D(NV40_3D_DMA_COLOR2, 2));
+	BEGIN_RING(chan, eng3d, NV40_3D_DMA_COLOR2, 2);
 	OUT_RING(chan, screen->base.channel->vram->handle);
 	OUT_RING(chan, screen->base.channel->vram->handle);
 
-	OUT_RING(chan, RING_3D(0x1450, 1));
+	BEGIN_RING(chan, eng3d, 0x1450, 1);
 	OUT_RING(chan, 0x00000004);
 
-	OUT_RING(chan, RING_3D(0x1ea4, 3));
+	BEGIN_RING(chan, eng3d, 0x1ea4, 3);
 	OUT_RING(chan, 0x00000010);
 	OUT_RING(chan, 0x01000100);
 	OUT_RING(chan, 0xff800006);
 
 	/* vtxprog output routing */
-	OUT_RING(chan, RING_3D(0x1fc4, 1));
+	BEGIN_RING(chan, eng3d, 0x1fc4, 1);
 	OUT_RING(chan, 0x06144321);
-	OUT_RING(chan, RING_3D(0x1fc8, 2));
+	BEGIN_RING(chan, eng3d, 0x1fc8, 2);
 	OUT_RING(chan, 0xedcba987);
 	OUT_RING(chan, 0x0000006f);
-	OUT_RING(chan, RING_3D(0x1fd0, 1));
+	BEGIN_RING(chan, eng3d, 0x1fd0, 1);
 	OUT_RING(chan, 0x00171615);
-	OUT_RING(chan, RING_3D(0x1fd4, 1));
+	BEGIN_RING(chan, eng3d, 0x1fd4, 1);
 	OUT_RING(chan, 0x001b1a19);
 
-	OUT_RING(chan, RING_3D(0x1ef8, 1));
+	BEGIN_RING(chan, eng3d, 0x1ef8, 1);
 	OUT_RING(chan, 0x0020ffff);
-	OUT_RING(chan, RING_3D(0x1d64, 1));
+	BEGIN_RING(chan, eng3d, 0x1d64, 1);
 	OUT_RING(chan, 0x01d300d4);
-	OUT_RING(chan, RING_3D(0x1e94, 1));
+	BEGIN_RING(chan, eng3d, 0x1e94, 1);
 	OUT_RING(chan, 0x00000001);
 
-	OUT_RING(chan, RING_3D(NV40_3D_MIPMAP_ROUNDING, 1));
+	BEGIN_RING(chan, eng3d, NV40_3D_MIPMAP_ROUNDING, 1);
 	OUT_RING(chan, NV40_3D_MIPMAP_ROUNDING_MODE_DOWN);
 }
 
@@ -573,25 +575,25 @@ nvfx_screen_create(struct pipe_winsys *ws, struct nouveau_device *dev)
 
 	/* Static eng3d initialisation */
 	/* note that we just started using the channel, so we must have space in the pushbuffer */
-	OUT_RING(chan, RING_3D(NV30_3D_DMA_NOTIFY, 1));
+	BEGIN_RING(chan, screen->eng3d, NV30_3D_DMA_NOTIFY, 1);
 	OUT_RING(chan, screen->sync->handle);
-	OUT_RING(chan, RING_3D(NV30_3D_DMA_TEXTURE0, 2));
+	BEGIN_RING(chan, screen->eng3d, NV30_3D_DMA_TEXTURE0, 2);
 	OUT_RING(chan, chan->vram->handle);
 	OUT_RING(chan, chan->gart->handle);
-	OUT_RING(chan, RING_3D(NV30_3D_DMA_COLOR1, 1));
+	BEGIN_RING(chan, screen->eng3d, NV30_3D_DMA_COLOR1, 1);
 	OUT_RING(chan, chan->vram->handle);
-	OUT_RING(chan, RING_3D(NV30_3D_DMA_COLOR0, 2));
+	BEGIN_RING(chan, screen->eng3d, NV30_3D_DMA_COLOR0, 2);
 	OUT_RING(chan, chan->vram->handle);
 	OUT_RING(chan, chan->vram->handle);
-	OUT_RING(chan, RING_3D(NV30_3D_DMA_VTXBUF0, 2));
+	BEGIN_RING(chan, screen->eng3d, NV30_3D_DMA_VTXBUF0, 2);
 	OUT_RING(chan, chan->vram->handle);
 	OUT_RING(chan, chan->gart->handle);
 
-	OUT_RING(chan, RING_3D(NV30_3D_DMA_FENCE, 2));
+	BEGIN_RING(chan, screen->eng3d, NV30_3D_DMA_FENCE, 2);
 	OUT_RING(chan, 0);
 	OUT_RING(chan, screen->query->handle);
 
-	OUT_RING(chan, RING_3D(NV30_3D_DMA_UNK1AC, 2));
+	BEGIN_RING(chan, screen->eng3d, NV30_3D_DMA_UNK1AC, 2);
 	OUT_RING(chan, chan->vram->handle);
 	OUT_RING(chan, chan->vram->handle);
 
diff --git a/src/gallium/drivers/nvfx/nvfx_state_emit.c b/src/gallium/drivers/nvfx/nvfx_state_emit.c
index 501fdd4430c..40ae4f5bd21 100644
--- a/src/gallium/drivers/nvfx/nvfx_state_emit.c
+++ b/src/gallium/drivers/nvfx/nvfx_state_emit.c
@@ -7,11 +7,11 @@ void
 nvfx_state_viewport_validate(struct nvfx_context *nvfx)
 {
 	struct nouveau_channel *chan = nvfx->screen->base.channel;
+	struct nouveau_grobj *eng3d = nvfx->screen->eng3d;
 	struct pipe_viewport_state *vpt = &nvfx->viewport;
 
-	WAIT_RING(chan, 11);
 	if(nvfx->render_mode == HW) {
-		OUT_RING(chan, RING_3D(NV30_3D_VIEWPORT_TRANSLATE_X, 8));
+		BEGIN_RING(chan, eng3d, NV30_3D_VIEWPORT_TRANSLATE_X, 8);
 		OUT_RINGf(chan, vpt->translate[0]);
 		OUT_RINGf(chan, vpt->translate[1]);
 		OUT_RINGf(chan, vpt->translate[2]);
@@ -20,10 +20,10 @@ nvfx_state_viewport_validate(struct nvfx_context *nvfx)
 		OUT_RINGf(chan, vpt->scale[1]);
 		OUT_RINGf(chan, vpt->scale[2]);
 		OUT_RINGf(chan, vpt->scale[3]);
-		OUT_RING(chan, RING_3D(0x1d78, 1));
+		BEGIN_RING(chan, eng3d, 0x1d78, 1);
 		OUT_RING(chan, 1);
 	} else {
-		OUT_RING(chan, RING_3D(NV30_3D_VIEWPORT_TRANSLATE_X, 8));
+		BEGIN_RING(chan, eng3d, NV30_3D_VIEWPORT_TRANSLATE_X, 8);
 		OUT_RINGf(chan, 0.0f);
 		OUT_RINGf(chan, 0.0f);
 		OUT_RINGf(chan, 0.0f);
@@ -32,7 +32,7 @@ nvfx_state_viewport_validate(struct nvfx_context *nvfx)
 		OUT_RINGf(chan, 1.0f);
 		OUT_RINGf(chan, 1.0f);
 		OUT_RINGf(chan, 1.0f);
-		OUT_RING(chan, RING_3D(0x1d78, 1));
+		BEGIN_RING(chan, eng3d, 0x1d78, 1);
 		OUT_RING(chan, nvfx->is_nv4x ? 0x110 : 1);
 	}
 }
@@ -41,6 +41,7 @@ void
 nvfx_state_scissor_validate(struct nvfx_context *nvfx)
 {
 	struct nouveau_channel *chan = nvfx->screen->base.channel;
+	struct nouveau_grobj *eng3d = nvfx->screen->eng3d;
 	struct pipe_rasterizer_state *rast = &nvfx->rasterizer->pipe;
 	struct pipe_scissor_state *s = &nvfx->scissor;
 
@@ -48,8 +49,7 @@ nvfx_state_scissor_validate(struct nvfx_context *nvfx)
 		return;
 	nvfx->state.scissor_enabled = rast->scissor;
 
-	WAIT_RING(chan, 3);
-	OUT_RING(chan, RING_3D(NV30_3D_SCISSOR_HORIZ, 2));
+	BEGIN_RING(chan, eng3d, NV30_3D_SCISSOR_HORIZ, 2);
 	if (nvfx->state.scissor_enabled) {
 		OUT_RING(chan, ((s->maxx - s->minx) << 16) | s->minx);
 		OUT_RING(chan, ((s->maxy - s->miny) << 16) | s->miny);
@@ -63,12 +63,12 @@ void
 nvfx_state_sr_validate(struct nvfx_context *nvfx)
 {
 	struct nouveau_channel* chan = nvfx->screen->base.channel;
+	struct nouveau_grobj *eng3d = nvfx->screen->eng3d;
 	struct pipe_stencil_ref *sr = &nvfx->stencil_ref;
 
-	WAIT_RING(chan, 4);
-	OUT_RING(chan, RING_3D(NV30_3D_STENCIL_FUNC_REF(0), 1));
+	BEGIN_RING(chan, eng3d, NV30_3D_STENCIL_FUNC_REF(0), 1);
 	OUT_RING(chan, sr->ref_value[0]);
-	OUT_RING(chan, RING_3D(NV30_3D_STENCIL_FUNC_REF(1), 1));
+	BEGIN_RING(chan, eng3d, NV30_3D_STENCIL_FUNC_REF(1), 1);
 	OUT_RING(chan, sr->ref_value[1]);
 }
 
@@ -76,10 +76,10 @@ void
 nvfx_state_blend_colour_validate(struct nvfx_context *nvfx)
 {
 	struct nouveau_channel* chan = nvfx->screen->base.channel;
+	struct nouveau_grobj *eng3d = nvfx->screen->eng3d;
 	struct pipe_blend_color *bcol = &nvfx->blend_colour;
 
-	WAIT_RING(chan, 2);
-	OUT_RING(chan, RING_3D(NV30_3D_BLEND_COLOR, 1));
+	BEGIN_RING(chan, eng3d, NV30_3D_BLEND_COLOR, 1);
 	OUT_RING(chan, ((float_to_ubyte(bcol->color[3]) << 24) |
 		       (float_to_ubyte(bcol->color[0]) << 16) |
 		       (float_to_ubyte(bcol->color[1]) <<  8) |
@@ -90,9 +90,9 @@ void
 nvfx_state_stipple_validate(struct nvfx_context *nvfx)
 {
 	struct nouveau_channel *chan = nvfx->screen->base.channel;
+	struct nouveau_grobj *eng3d = nvfx->screen->eng3d;
 
-	WAIT_RING(chan, 33);
-	OUT_RING(chan, RING_3D(NV30_3D_POLYGON_STIPPLE_PATTERN(0), 32));
+	BEGIN_RING(chan, eng3d, NV30_3D_POLYGON_STIPPLE_PATTERN(0), 32);
 	OUT_RINGp(chan, nvfx->stipple, 32);
 }
 
@@ -100,12 +100,12 @@ static void
 nvfx_coord_conventions_validate(struct nvfx_context* nvfx)
 {
 	struct nouveau_channel* chan = nvfx->screen->base.channel;
+	struct nouveau_grobj *eng3d = nvfx->screen->eng3d;
 	unsigned value = nvfx->hw_fragprog->coord_conventions;
 	if(value & NV30_3D_COORD_CONVENTIONS_ORIGIN_INVERTED)
 		value |= nvfx->framebuffer.height << NV30_3D_COORD_CONVENTIONS_HEIGHT__SHIFT;
 
-	WAIT_RING(chan, 2);
-	OUT_RING(chan, RING_3D(NV30_3D_COORD_CONVENTIONS, 1));
+	BEGIN_RING(chan, eng3d, NV30_3D_COORD_CONVENTIONS, 1);
 	OUT_RING(chan, value);
 }
 
@@ -113,6 +113,7 @@ static void
 nvfx_ucp_validate(struct nvfx_context* nvfx)
 {
 	struct nouveau_channel* chan = nvfx->screen->base.channel;
+	struct nouveau_grobj *eng3d = nvfx->screen->eng3d;
 	unsigned enables[7] =
 	{
 			0,
@@ -126,17 +127,15 @@ nvfx_ucp_validate(struct nvfx_context* nvfx)
 
 	if(!nvfx->use_vp_clipping)
 	{
-		WAIT_RING(chan, 2);
-		OUT_RING(chan, RING_3D(NV30_3D_VP_CLIP_PLANES_ENABLE, 1));
+		BEGIN_RING(chan, eng3d, NV30_3D_VP_CLIP_PLANES_ENABLE, 1);
 		OUT_RING(chan, 0);
 
-		WAIT_RING(chan, 6 * 4 + 1);
-		OUT_RING(chan, RING_3D(NV30_3D_VP_CLIP_PLANE(0, 0), nvfx->clip.nr * 4));
+		BEGIN_RING(chan, eng3d, NV30_3D_VP_CLIP_PLANE(0, 0),
+			   nvfx->clip.nr * 4);
 		OUT_RINGp(chan, &nvfx->clip.ucp[0][0], nvfx->clip.nr * 4);
 	}
 
-	WAIT_RING(chan, 2);
-	OUT_RING(chan, RING_3D(NV30_3D_VP_CLIP_PLANES_ENABLE, 1));
+	BEGIN_RING(chan, eng3d, NV30_3D_VP_CLIP_PLANES_ENABLE, 1);
 	OUT_RING(chan, enables[nvfx->clip.nr]);
 }
 
@@ -144,38 +143,37 @@ static void
 nvfx_vertprog_ucp_validate(struct nvfx_context* nvfx)
 {
 	struct nouveau_channel* chan = nvfx->screen->base.channel;
+	struct nouveau_grobj *eng3d = nvfx->screen->eng3d;
 	unsigned i;
 	struct nvfx_vertex_program* vp = nvfx->hw_vertprog;
 	if(nvfx->clip.nr != vp->clip_nr)
 	{
 		unsigned idx;
-		WAIT_RING(chan, 14);
 
 		/* remove last instruction bit */
 		if(vp->clip_nr >= 0)
 		{
 			idx = vp->nr_insns - 7 + vp->clip_nr;
-			OUT_RING(chan, RING_3D(NV30_3D_VP_UPLOAD_FROM_ID, 1));
+			BEGIN_RING(chan, eng3d, NV30_3D_VP_UPLOAD_FROM_ID, 1);
 			OUT_RING(chan,  vp->exec->start + idx);
-			OUT_RING(chan, RING_3D(NV30_3D_VP_UPLOAD_INST(0), 4));
+			BEGIN_RING(chan, eng3d, NV30_3D_VP_UPLOAD_INST(0), 4);
 			OUT_RINGp (chan, vp->insns[idx].data, 4);
 		}
 
 		 /* set last instruction bit */
 		idx = vp->nr_insns - 7 + nvfx->clip.nr;
-		OUT_RING(chan, RING_3D(NV30_3D_VP_UPLOAD_FROM_ID, 1));
+		BEGIN_RING(chan, eng3d, NV30_3D_VP_UPLOAD_FROM_ID, 1);
 		OUT_RING(chan,  vp->exec->start + idx);
-		OUT_RING(chan, RING_3D(NV30_3D_VP_UPLOAD_INST(0), 4));
+		BEGIN_RING(chan, eng3d, NV30_3D_VP_UPLOAD_INST(0), 4);
 		OUT_RINGp(chan, vp->insns[idx].data, 3);
 		OUT_RING(chan, vp->insns[idx].data[3] | 1);
 		vp->clip_nr = nvfx->clip.nr;
 	}
 
 	// TODO: only do this for the ones changed
-	WAIT_RING(chan, 6 * 6);
 	for(i = 0; i < nvfx->clip.nr; ++i)
 	{
-		OUT_RING(chan, RING_3D(NV30_3D_VP_UPLOAD_CONST_ID, 5));
+		BEGIN_RING(chan, eng3d, NV30_3D_VP_UPLOAD_CONST_ID, 5);
 		OUT_RING(chan, vp->data->start + i);
 		OUT_RINGp (chan, nvfx->clip.ucp[i], 4);
 	}
@@ -185,6 +183,7 @@ static boolean
 nvfx_state_validate_common(struct nvfx_context *nvfx)
 {
 	struct nouveau_channel* chan = nvfx->screen->base.channel;
+	struct nouveau_grobj *eng3d = nvfx->screen->eng3d;
 	unsigned dirty;
 	unsigned still_dirty = 0;
 	int new_fb_mode = -1; /* 1 = all swizzled, 0 = make all linear */
@@ -287,8 +286,7 @@ nvfx_state_validate_common(struct nvfx_context *nvfx)
 
 		if(vp_output != nvfx->hw_vp_output)
 		{
-			WAIT_RING(chan, 2);
-			OUT_RING(chan, RING_3D(NV40_3D_VP_RESULT_EN, 1));
+			BEGIN_RING(chan, eng3d, NV40_3D_VP_RESULT_EN, 1);
 			OUT_RING(chan, vp_output);
 			nvfx->hw_vp_output = vp_output;
 		}
@@ -320,8 +318,7 @@ nvfx_state_validate_common(struct nvfx_context *nvfx)
 
 	if(dirty & NVFX_NEW_ZSA || (new_fb_mode >= 0))
 	{
-		WAIT_RING(chan, 3);
-		OUT_RING(chan, RING_3D(NV30_3D_DEPTH_WRITE_ENABLE, 2));
+		BEGIN_RING(chan, eng3d, NV30_3D_DEPTH_WRITE_ENABLE, 2);
 		OUT_RING(chan, nvfx->framebuffer.zsbuf && nvfx->zsa->pipe.depth.writemask);
 	        OUT_RING(chan, nvfx->framebuffer.zsbuf && nvfx->zsa->pipe.depth.enabled);
 	}
@@ -334,10 +331,9 @@ nvfx_state_validate_common(struct nvfx_context *nvfx)
 		// TODO: what about nv30?
 		if(nvfx->is_nv4x)
 		{
-			WAIT_RING(chan, 4);
-			OUT_RING(chan, RING_3D(NV40_3D_TEX_CACHE_CTL, 1));
+			BEGIN_RING(chan, eng3d, NV40_3D_TEX_CACHE_CTL, 1);
 			OUT_RING(chan, 2);
-			OUT_RING(chan, RING_3D(NV40_3D_TEX_CACHE_CTL, 1));
+			BEGIN_RING(chan, eng3d, NV40_3D_TEX_CACHE_CTL, 1);
 			OUT_RING(chan, 1);
 		}
 	}
diff --git a/src/gallium/drivers/nvfx/nvfx_state_fb.c b/src/gallium/drivers/nvfx/nvfx_state_fb.c
index 816bb89f2c6..f9fed94044a 100644
--- a/src/gallium/drivers/nvfx/nvfx_state_fb.c
+++ b/src/gallium/drivers/nvfx/nvfx_state_fb.c
@@ -96,6 +96,7 @@ nvfx_framebuffer_validate(struct nvfx_context *nvfx, unsigned prepare_result)
 {
 	struct pipe_framebuffer_state *fb = &nvfx->framebuffer;
 	struct nouveau_channel *chan = nvfx->screen->base.channel;
+	struct nouveau_grobj *eng3d = nvfx->screen->eng3d;
 	uint32_t rt_enable, rt_format;
 	int i;
 	unsigned rt_flags = NOUVEAU_BO_RDWR | NOUVEAU_BO_VRAM;
@@ -204,11 +205,11 @@ nvfx_framebuffer_validate(struct nvfx_context *nvfx, unsigned prepare_result)
 
 		//printf("rendering to bo %p [%i] at offset %i with pitch %i\n", rt0->bo, rt0->bo->handle, rt0->offset, pitch);
 
-		OUT_RING(chan, RING_3D(NV30_3D_DMA_COLOR0, 1));
+		BEGIN_RING(chan, eng3d, NV30_3D_DMA_COLOR0, 1);
 		OUT_RELOC(chan, rt0->bo, 0,
 			      rt_flags | NOUVEAU_BO_OR,
 			      chan->vram->handle, chan->gart->handle);
-		OUT_RING(chan, RING_3D(NV30_3D_COLOR0_PITCH, 2));
+		BEGIN_RING(chan, eng3d, NV30_3D_COLOR0_PITCH, 2);
 		OUT_RING(chan, pitch);
 		OUT_RELOC(chan, rt0->bo,
 			      rt0->offset, rt_flags | NOUVEAU_BO_LOW,
@@ -216,11 +217,11 @@ nvfx_framebuffer_validate(struct nvfx_context *nvfx, unsigned prepare_result)
 	}
 
 	if (rt_enable & NV30_3D_RT_ENABLE_COLOR1) {
-		OUT_RING(chan, RING_3D(NV30_3D_DMA_COLOR1, 1));
+		BEGIN_RING(chan, eng3d, NV30_3D_DMA_COLOR1, 1);
 		OUT_RELOC(chan, nvfx->hw_rt[1].bo, 0,
 			      rt_flags | NOUVEAU_BO_OR,
 			      chan->vram->handle, chan->gart->handle);
-		OUT_RING(chan, RING_3D(NV30_3D_COLOR1_OFFSET, 2));
+		BEGIN_RING(chan, eng3d, NV30_3D_COLOR1_OFFSET, 2);
 		OUT_RELOC(chan, nvfx->hw_rt[1].bo,
 				nvfx->hw_rt[1].offset, rt_flags | NOUVEAU_BO_LOW,
 			      0, 0);
@@ -230,68 +231,68 @@ nvfx_framebuffer_validate(struct nvfx_context *nvfx, unsigned prepare_result)
 	if(nvfx->is_nv4x)
 	{
 		if (rt_enable & NV40_3D_RT_ENABLE_COLOR2) {
-			OUT_RING(chan, RING_3D(NV40_3D_DMA_COLOR2, 1));
+			BEGIN_RING(chan, eng3d, NV40_3D_DMA_COLOR2, 1);
 			OUT_RELOC(chan, nvfx->hw_rt[2].bo, 0,
 				      rt_flags | NOUVEAU_BO_OR,
 				      chan->vram->handle, chan->gart->handle);
-			OUT_RING(chan, RING_3D(NV40_3D_COLOR2_OFFSET, 1));
+			BEGIN_RING(chan, eng3d, NV40_3D_COLOR2_OFFSET, 1);
 			OUT_RELOC(chan, nvfx->hw_rt[2].bo,
 				      nvfx->hw_rt[2].offset, rt_flags | NOUVEAU_BO_LOW,
 				      0, 0);
-			OUT_RING(chan, RING_3D(NV40_3D_COLOR2_PITCH, 1));
+			BEGIN_RING(chan, eng3d, NV40_3D_COLOR2_PITCH, 1);
 			OUT_RING(chan, nvfx->hw_rt[2].pitch);
 		}
 
 		if (rt_enable & NV40_3D_RT_ENABLE_COLOR3) {
-			OUT_RING(chan, RING_3D(NV40_3D_DMA_COLOR3, 1));
+			BEGIN_RING(chan, eng3d, NV40_3D_DMA_COLOR3, 1);
 			OUT_RELOC(chan, nvfx->hw_rt[3].bo, 0,
 				      rt_flags | NOUVEAU_BO_OR,
 				      chan->vram->handle, chan->gart->handle);
-			OUT_RING(chan, RING_3D(NV40_3D_COLOR3_OFFSET, 1));
+			BEGIN_RING(chan, eng3d, NV40_3D_COLOR3_OFFSET, 1);
 			OUT_RELOC(chan, nvfx->hw_rt[3].bo,
 					nvfx->hw_rt[3].offset, rt_flags | NOUVEAU_BO_LOW,
 				      0, 0);
-			OUT_RING(chan, RING_3D(NV40_3D_COLOR3_PITCH, 1));
+			BEGIN_RING(chan, eng3d, NV40_3D_COLOR3_PITCH, 1);
 			OUT_RING(chan, nvfx->hw_rt[3].pitch);
 		}
 	}
 
 	if (fb->zsbuf) {
-		OUT_RING(chan, RING_3D(NV30_3D_DMA_ZETA, 1));
+		BEGIN_RING(chan, eng3d, NV30_3D_DMA_ZETA, 1);
 		OUT_RELOC(chan, nvfx->hw_zeta.bo, 0,
 			      rt_flags | NOUVEAU_BO_OR,
 			      chan->vram->handle, chan->gart->handle);
-		OUT_RING(chan, RING_3D(NV30_3D_ZETA_OFFSET, 1));
+		BEGIN_RING(chan, eng3d, NV30_3D_ZETA_OFFSET, 1);
 		/* TODO: reverse engineer LMA */
 		OUT_RELOC(chan, nvfx->hw_zeta.bo,
 			     nvfx->hw_zeta.offset, rt_flags | NOUVEAU_BO_LOW, 0, 0);
 	        if(nvfx->is_nv4x) {
-			OUT_RING(chan, RING_3D(NV40_3D_ZETA_PITCH, 1));
+			BEGIN_RING(chan, eng3d, NV40_3D_ZETA_PITCH, 1);
 			OUT_RING(chan, nvfx->hw_zeta.pitch);
 		}
 	}
 	else if(nvfx->is_nv4x) {
-		OUT_RING(chan, RING_3D(NV40_3D_ZETA_PITCH, 1));
+		BEGIN_RING(chan, eng3d, NV40_3D_ZETA_PITCH, 1);
 		OUT_RING(chan, 64);
 	}
 
-	OUT_RING(chan, RING_3D(NV30_3D_RT_ENABLE, 1));
+	BEGIN_RING(chan, eng3d, NV30_3D_RT_ENABLE, 1);
 	OUT_RING(chan, rt_enable);
-	OUT_RING(chan, RING_3D(NV30_3D_RT_HORIZ, 3));
+	BEGIN_RING(chan, eng3d, NV30_3D_RT_HORIZ, 3);
 	OUT_RING(chan, (w << 16) | 0);
 	OUT_RING(chan, (h << 16) | 0);
 	OUT_RING(chan, rt_format);
-	OUT_RING(chan, RING_3D(NV30_3D_VIEWPORT_HORIZ, 2));
+	BEGIN_RING(chan, eng3d, NV30_3D_VIEWPORT_HORIZ, 2);
 	OUT_RING(chan, (w << 16) | 0);
 	OUT_RING(chan, (h << 16) | 0);
-	OUT_RING(chan, RING_3D(NV30_3D_VIEWPORT_CLIP_HORIZ(0), 2));
+	BEGIN_RING(chan, eng3d, NV30_3D_VIEWPORT_CLIP_HORIZ(0), 2);
 	OUT_RING(chan, ((w - 1) << 16) | 0);
 	OUT_RING(chan, ((h - 1) << 16) | 0);
 
 	if(!nvfx->is_nv4x) {
 		/* Wonder why this is needed, context should all be set to zero on init */
 		/* TODO: we can most likely remove this, after putting it in context init */
-		OUT_RING(chan, RING_3D(NV30_3D_VIEWPORT_TX_ORIGIN, 1));
+		BEGIN_RING(chan, eng3d, NV30_3D_VIEWPORT_TX_ORIGIN, 1);
 		OUT_RING(chan, 0);
 	}
 	nvfx->relocs_needed &=~ NVFX_RELOCATE_FRAMEBUFFER;
diff --git a/src/gallium/drivers/nvfx/nvfx_surface.c b/src/gallium/drivers/nvfx/nvfx_surface.c
index 6fd6c47081b..be31853d717 100644
--- a/src/gallium/drivers/nvfx/nvfx_surface.c
+++ b/src/gallium/drivers/nvfx/nvfx_surface.c
@@ -168,8 +168,8 @@ nvfx_get_blitter(struct pipe_context* pipe, int copy)
 	if(nvfx->query && !nvfx->blitters_in_use)
 	{
 		struct nouveau_channel* chan = nvfx->screen->base.channel;
-		WAIT_RING(chan, 2);
-		OUT_RING(chan, RING_3D(NV30_3D_QUERY_ENABLE, 1));
+		struct nouveau_grobj *eng3d = nvfx->screen->eng3d;
+		BEGIN_RING(chan, eng3d, NV30_3D_QUERY_ENABLE, 1);
 		OUT_RING(chan, 0);
 	}
 
@@ -209,8 +209,8 @@ nvfx_put_blitter(struct pipe_context* pipe, struct blitter_context* blitter)
 	if(nvfx->query && !nvfx->blitters_in_use)
 	{
 		struct nouveau_channel* chan = nvfx->screen->base.channel;
-		WAIT_RING(chan, 2);
-		OUT_RING(chan, RING_3D(NV30_3D_QUERY_ENABLE, 1));
+		struct nouveau_grobj *eng3d = nvfx->screen->eng3d;
+		BEGIN_RING(chan, eng3d, NV30_3D_QUERY_ENABLE, 1);
 		OUT_RING(chan, 1);
 	}
 }
diff --git a/src/gallium/drivers/nvfx/nvfx_vbo.c b/src/gallium/drivers/nvfx/nvfx_vbo.c
index 339b31786d6..01dacb43dad 100644
--- a/src/gallium/drivers/nvfx/nvfx_vbo.c
+++ b/src/gallium/drivers/nvfx/nvfx_vbo.c
@@ -246,6 +246,7 @@ boolean
 nvfx_vbo_validate(struct nvfx_context *nvfx)
 {
 	struct nouveau_channel* chan = nvfx->screen->base.channel;
+	struct nouveau_grobj *eng3d = nvfx->screen->eng3d;
 	int i;
 	int elements = MAX2(nvfx->vtxelt->num_elements, nvfx->hw_vtxelt_nr);
 	unsigned vb_flags = nvfx->screen->vertex_buffer_reloc_flags | NOUVEAU_BO_RD;
@@ -261,11 +262,11 @@ nvfx_vbo_validate(struct nvfx_context *nvfx)
 		struct nvfx_buffer* buffer = nvfx_buffer(vb->buffer);
 		float v[4];
 		ve->fetch_rgba_float(v, buffer->data + vb->buffer_offset + ve->src_offset, 0, 0);
-		nvfx_emit_vtx_attr(chan, ve->idx, v, ve->ncomp);
+		nvfx_emit_vtx_attr(chan, eng3d, ve->idx, v, ve->ncomp);
 	}
 
 
-	OUT_RING(chan, RING_3D(NV30_3D_VTXFMT(0), elements));
+	BEGIN_RING(chan, eng3d, NV30_3D_VTXFMT(0), elements);
 	if(nvfx->use_vertex_buffers)
 	{
 		unsigned idx = 0;
@@ -296,12 +297,12 @@ nvfx_vbo_validate(struct nvfx_context *nvfx)
 		unsigned i;
 		/* seems to be some kind of cache flushing */
 		for(i = 0; i < 3; ++i) {
-			OUT_RING(chan, RING_3D(0x1718, 1));
+			BEGIN_RING(chan, eng3d, 0x1718, 1);
 			OUT_RING(chan, 0);
 		}
 	}
 
-	OUT_RING(chan, RING_3D(NV30_3D_VTXBUF(0), elements));
+	BEGIN_RING(chan, eng3d, NV30_3D_VTXBUF(0), elements);
 	if(nvfx->use_vertex_buffers)
 	{
 		unsigned idx = 0;
@@ -329,7 +330,7 @@ nvfx_vbo_validate(struct nvfx_context *nvfx)
 			OUT_RING(chan, 0);
 	}
 
-	OUT_RING(chan, RING_3D(0x1710, 1));
+	BEGIN_RING(chan, eng3d, 0x1710, 1);
 	OUT_RING(chan, 0);
 
 	nvfx->hw_vtxelt_nr = nvfx->vtxelt->num_elements;
@@ -341,15 +342,14 @@ void
 nvfx_vbo_swtnl_validate(struct nvfx_context *nvfx)
 {
 	struct nouveau_channel* chan = nvfx->screen->base.channel;
+	struct nouveau_grobj *eng3d = nvfx->screen->eng3d;
 	unsigned num_outputs = nvfx->vertprog->draw_elements;
 	int elements = MAX2(num_outputs, nvfx->hw_vtxelt_nr);
 
 	if (!elements)
 		return;
 
-	WAIT_RING(chan, (1 + 6 + 1 + 2) + elements * 2);
-
-	OUT_RING(chan, RING_3D(NV30_3D_VTXFMT(0), elements));
+	BEGIN_RING(chan, eng3d, NV30_3D_VTXFMT(0), elements);
 	for(unsigned i = 0; i < num_outputs; ++i)
 		OUT_RING(chan, (4 << NV30_3D_VTXFMT_SIZE__SHIFT) | NV30_3D_VTXFMT_TYPE_V32_FLOAT);
 	for(unsigned i = num_outputs; i < elements; ++i)
@@ -359,16 +359,16 @@ nvfx_vbo_swtnl_validate(struct nvfx_context *nvfx)
 		unsigned i;
 		/* seems to be some kind of cache flushing */
 		for(i = 0; i < 3; ++i) {
-			OUT_RING(chan, RING_3D(0x1718, 1));
+			BEGIN_RING(chan, eng3d, 0x1718, 1);
 			OUT_RING(chan, 0);
 		}
 	}
 
-	OUT_RING(chan, RING_3D(NV30_3D_VTXBUF(0), elements));
+	BEGIN_RING(chan, eng3d, NV30_3D_VTXBUF(0), elements);
 	for (unsigned i = 0; i < elements; i++)
 		OUT_RING(chan, 0);
 
-	OUT_RING(chan, RING_3D(0x1710, 1));
+	BEGIN_RING(chan, eng3d, 0x1710, 1);
 	OUT_RING(chan, 0);
 
 	nvfx->hw_vtxelt_nr = num_outputs;
@@ -591,18 +591,10 @@ nvfx_set_vertex_buffers(struct pipe_context *pipe, unsigned count,
 {
 	struct nvfx_context *nvfx = nvfx_context(pipe);
 
-	for(unsigned i = 0; i < count; ++i)
-	{
-		pipe_resource_reference(&nvfx->vtxbuf[i].buffer, vb[i].buffer);
-		nvfx->vtxbuf[i].buffer_offset = vb[i].buffer_offset;
-		nvfx->vtxbuf[i].max_index = vb[i].max_index;
-		nvfx->vtxbuf[i].stride = vb[i].stride;
-	}
-
-	for(unsigned i = count; i < nvfx->vtxbuf_nr; ++i)
-		pipe_resource_reference(&nvfx->vtxbuf[i].buffer, 0);
+        util_copy_vertex_buffers(nvfx->vtxbuf,
+                                 &nvfx->vtxbuf_nr,
+                                 vb, count);
 
-	nvfx->vtxbuf_nr = count;
 	nvfx->use_vertex_buffers = -1;
 	nvfx->draw_dirty |= NVFX_NEW_ARRAYS;
 }
diff --git a/src/gallium/drivers/nvfx/nvfx_vertprog.c b/src/gallium/drivers/nvfx/nvfx_vertprog.c
index e543fda50ef..a11941f3d51 100644
--- a/src/gallium/drivers/nvfx/nvfx_vertprog.c
+++ b/src/gallium/drivers/nvfx/nvfx_vertprog.c
@@ -1182,6 +1182,7 @@ nvfx_vertprog_validate(struct nvfx_context *nvfx)
 {
 	struct nvfx_screen *screen = nvfx->screen;
 	struct nouveau_channel *chan = screen->base.channel;
+	struct nouveau_grobj *eng3d = screen->eng3d;
 	struct nvfx_pipe_vertex_program *pvp = nvfx->vertprog;
 	struct nvfx_vertex_program* vp;
 	struct pipe_resource *constbuf;
@@ -1341,7 +1342,6 @@ nvfx_vertprog_validate(struct nvfx_context *nvfx)
 		}
 		*/
 
-		WAIT_RING(chan, 6 * vp->nr_consts);
 		for (i = nvfx->use_vp_clipping ? 6 : 0; i < vp->nr_consts; i++) {
 			struct nvfx_vertex_program_data *vpd = &vp->consts[i];
 
@@ -1356,7 +1356,7 @@ nvfx_vertprog_validate(struct nvfx_context *nvfx)
 
 			//printf("upload into %i + %i: %f %f %f %f\n", vp->data->start, i, vpd->value[0], vpd->value[1], vpd->value[2], vpd->value[3]);
 
-			OUT_RING(chan, RING_3D(NV30_3D_VP_UPLOAD_CONST_ID, 5));
+			BEGIN_RING(chan, eng3d, NV30_3D_VP_UPLOAD_CONST_ID, 5);
 			OUT_RING(chan, i + vp->data->start);
 			OUT_RINGp(chan, (uint32_t *)vpd->value, 4);
 		}
@@ -1364,11 +1364,10 @@ nvfx_vertprog_validate(struct nvfx_context *nvfx)
 
 	/* Upload vtxprog */
 	if (upload_code) {
-		WAIT_RING(chan, 2 + 5 * vp->nr_insns);
-		OUT_RING(chan, RING_3D(NV30_3D_VP_UPLOAD_FROM_ID, 1));
+		BEGIN_RING(chan, eng3d, NV30_3D_VP_UPLOAD_FROM_ID, 1);
 		OUT_RING(chan, vp->exec->start);
 		for (i = 0; i < vp->nr_insns; i++) {
-			OUT_RING(chan, RING_3D(NV30_3D_VP_UPLOAD_INST(0), 4));
+			BEGIN_RING(chan, eng3d, NV30_3D_VP_UPLOAD_INST(0), 4);
 			//printf("%08x %08x %08x %08x\n", vp->insns[i].data[0], vp->insns[i].data[1], vp->insns[i].data[2], vp->insns[i].data[3]);
 			OUT_RINGp(chan, vp->insns[i].data, 4);
 		}
@@ -1377,11 +1376,10 @@ nvfx_vertprog_validate(struct nvfx_context *nvfx)
 
 	if(nvfx->dirty & (NVFX_NEW_VERTPROG))
 	{
-		WAIT_RING(chan, 6);
-		OUT_RING(chan, RING_3D(NV30_3D_VP_START_FROM_ID, 1));
+		BEGIN_RING(chan, eng3d, NV30_3D_VP_START_FROM_ID, 1);
 		OUT_RING(chan, vp->exec->start);
 		if(nvfx->is_nv4x) {
-			OUT_RING(chan, RING_3D(NV40_3D_VP_ATTRIB_EN, 1));
+			BEGIN_RING(chan, eng3d, NV40_3D_VP_ATTRIB_EN, 1);
 			OUT_RING(chan, vp->ir);
 		}
 	}
diff --git a/src/gallium/drivers/r300/r300_blit.c b/src/gallium/drivers/r300/r300_blit.c
index 6e886433bc5..017db48485b 100644
--- a/src/gallium/drivers/r300/r300_blit.c
+++ b/src/gallium/drivers/r300/r300_blit.c
@@ -326,6 +326,7 @@ void r300_flush_depth_stencil(struct pipe_context *pipe,
     r300->z_decomp_rd = FALSE;
 
     tex->zmask_in_use[level] = FALSE;
+    pipe_surface_reference(&dstsurf, NULL);
 }
 
 /* Copy a block of pixels from one surface to another using HW. */
diff --git a/src/gallium/drivers/r300/r300_chipset.c b/src/gallium/drivers/r300/r300_chipset.c
index 583e981a4d2..2b183f62c56 100644
--- a/src/gallium/drivers/r300/r300_chipset.c
+++ b/src/gallium/drivers/r300/r300_chipset.c
@@ -366,7 +366,7 @@ void r300_parse_chipset(struct r300_capabilities* caps)
             caps->family = CHIP_FAMILY_RV530;
             caps->num_vert_fpus = 5;
             caps->is_r500 = TRUE;
-            /*caps->hiz_ram = RV530_HIZ_LIMIT;*/
+            caps->hiz_ram = RV530_HIZ_LIMIT;
             caps->zmask_ram = PIPE_ZMASK_SIZE;
             break;
 
diff --git a/src/gallium/drivers/r300/r300_context.c b/src/gallium/drivers/r300/r300_context.c
index bf1b8c33c00..91263ad7bcd 100644
--- a/src/gallium/drivers/r300/r300_context.c
+++ b/src/gallium/drivers/r300/r300_context.c
@@ -35,6 +35,10 @@
 #include "r300_screen_buffer.h"
 #include "r300_winsys.h"
 
+#ifdef HAVE_LLVM
+#include "gallivm/lp_bld_init.h"
+#endif
+
 static void r300_update_num_contexts(struct r300_screen *r300screen,
                                      int diff)
 {
@@ -86,6 +90,7 @@ static void r300_release_referenced_objects(struct r300_context *r300)
     /* Vertex buffers. */
     for (i = 0; i < r300->vertex_buffer_count; i++) {
         pipe_resource_reference(&r300->vertex_buffer[i].buffer, NULL);
+        pipe_resource_reference(&r300->valid_vertex_buffer[i], NULL);
     }
 
     /* If there are any queries pending or not destroyed, remove them now. */
@@ -101,9 +106,14 @@ static void r300_destroy_context(struct pipe_context* context)
 
     if (r300->blitter)
         util_blitter_destroy(r300->blitter);
-    if (r300->draw)
+    if (r300->draw) {
         draw_destroy(r300->draw);
 
+#ifdef HAVE_LLVM
+        gallivm_destroy(r300->gallivm);
+#endif
+    }
+
     if (r300->upload_vb)
         u_upload_destroy(r300->upload_vb);
     if (r300->upload_ib)
@@ -422,7 +432,12 @@ struct pipe_context* r300_create_context(struct pipe_screen* screen,
 
     if (!r300screen->caps.has_tcl) {
         /* Create a Draw. This is used for SW TCL. */
+#ifdef HAVE_LLVM
+        r300->gallivm = gallivm_create();
+        r300->draw = draw_create_gallivm(&r300->context, r300->gallivm);
+#else
         r300->draw = draw_create(&r300->context);
+#endif
         if (r300->draw == NULL)
             goto fail;
         /* Enable our renderer. */
@@ -456,14 +471,14 @@ struct pipe_context* r300_create_context(struct pipe_screen* screen,
             goto fail;
 
     r300->upload_ib = u_upload_create(&r300->context,
-				      32 * 1024, 16,
+				      64 * 1024, 16,
 				      PIPE_BIND_INDEX_BUFFER);
 
     if (r300->upload_ib == NULL)
         goto fail;
 
     r300->upload_vb = u_upload_create(&r300->context,
-				      128 * 1024, 16,
+				      1024 * 1024, 16,
 				      PIPE_BIND_VERTEX_BUFFER);
     if (r300->upload_vb == NULL)
         goto fail;
diff --git a/src/gallium/drivers/r300/r300_context.h b/src/gallium/drivers/r300/r300_context.h
index 39dcde06106..1a14d2b79e5 100644
--- a/src/gallium/drivers/r300/r300_context.h
+++ b/src/gallium/drivers/r300/r300_context.h
@@ -440,9 +440,6 @@ struct r300_translate_context {
     /* Translate cache for incompatible vertex offset/stride/format fallback. */
     struct translate_cache *translate_cache;
 
-    /* The vertex buffer slot containing the translated buffer. */
-    unsigned vb_slot;
-
     /* Saved and new vertex element state. */
     void *saved_velems, *new_velems;
 };
@@ -459,6 +456,7 @@ struct r300_context {
     struct r300_screen *screen;
 
     /* Draw module. Used mostly for SW TCL. */
+    struct gallivm_state *gallivm;
     struct draw_context* draw;
     /* Vertex buffer for SW TCL. */
     struct pipe_resource* vbo;
@@ -557,12 +555,15 @@ struct r300_context {
     struct r300_atom *first_dirty, *last_dirty;
 
     /* Vertex buffers for Gallium. */
+    /* May contain user buffers. */
     struct pipe_vertex_buffer vertex_buffer[PIPE_MAX_ATTRIBS];
+    /* Contains only non-user buffers. */
+    struct pipe_resource *valid_vertex_buffer[PIPE_MAX_ATTRIBS];
     int vertex_buffer_count;
     int vertex_buffer_max_index;
+    boolean any_user_vbs;
     /* Vertex elements for Gallium. */
     struct r300_vertex_element_state *velems;
-    bool any_user_vbs;
 
     struct pipe_index_buffer index_buffer;
 
@@ -612,12 +613,15 @@ struct r300_context {
     int vs_const_base;
 
     /* AOS (PACKET3_3D_LOAD_VBPNTR) command buffer for the case offset=0. */
-    uint32_t aos_cb[(16 * 3 + 1) / 2];
-    boolean aos_dirty;
+    uint32_t vertex_arrays_cb[(16 * 3 + 1) / 2];
+    boolean vertex_arrays_dirty;
 
     /* Whether any buffer (FB, textures, VBOs) has been set, but buffers
      * haven't been validated yet. */
     boolean validate_buffers;
+    /* Whether user buffers have been validated. */
+    boolean upload_vb_validated;
+    boolean upload_ib_validated;
 };
 
 #define foreach_atom(r300, atom) \
@@ -679,7 +683,8 @@ void r300_resume_query(struct r300_context *r300,
 void r300_stop_query(struct r300_context *r300);
 
 /* r300_render_translate.c */
-void r300_begin_vertex_translate(struct r300_context *r300);
+void r300_begin_vertex_translate(struct r300_context *r300,
+                                 int min_index, int max_index);
 void r300_end_vertex_translate(struct r300_context *r300);
 void r300_translate_index_buffer(struct r300_context *r300,
                                  struct pipe_resource **index_buffer,
@@ -697,7 +702,8 @@ void r500_emit_index_bias(struct r300_context *r300, int index_bias);
 enum r300_fb_state_change {
     R300_CHANGED_FB_STATE = 0,
     R300_CHANGED_CBZB_FLAG,
-    R300_CHANGED_ZCLEAR_FLAG
+    R300_CHANGED_ZCLEAR_FLAG,
+    R300_CHANGED_MULTIWRITE
 };
 
 void r300_mark_fb_state_dirty(struct r300_context *r300,
diff --git a/src/gallium/drivers/r300/r300_cs.h b/src/gallium/drivers/r300/r300_cs.h
index 67fb0096a8c..6726f100e1b 100644
--- a/src/gallium/drivers/r300/r300_cs.h
+++ b/src/gallium/drivers/r300/r300_cs.h
@@ -106,26 +106,26 @@
  * Writing relocations.
  */
 
-#define OUT_CS_RELOC(bo, offset, rd, wd) do { \
+#define OUT_CS_RELOC(bo, offset) do { \
     assert(bo); \
     OUT_CS(offset); \
-    cs_winsys->cs_write_reloc(cs_copy, bo, rd, wd); \
+    cs_winsys->cs_write_reloc(cs_copy, bo); \
     CS_DEBUG(cs_count -= 2;) \
 } while (0)
 
-#define OUT_CS_BUF_RELOC(bo, offset, rd, wd) do { \
+#define OUT_CS_BUF_RELOC(bo, offset) do { \
     assert(bo); \
-    OUT_CS_RELOC(r300_buffer(bo)->cs_buf, offset, rd, wd); \
+    OUT_CS_RELOC(r300_buffer(bo)->cs_buf, offset); \
 } while (0)
 
-#define OUT_CS_TEX_RELOC(tex, offset, rd, wd) do { \
+#define OUT_CS_TEX_RELOC(tex, offset) do { \
     assert(tex); \
-    OUT_CS_RELOC(tex->cs_buffer, offset, rd, wd); \
+    OUT_CS_RELOC(tex->cs_buffer, offset); \
 } while (0)
 
-#define OUT_CS_BUF_RELOC_NO_OFFSET(bo, rd, wd) do { \
+#define OUT_CS_BUF_RELOC_NO_OFFSET(bo) do { \
     assert(bo); \
-    cs_winsys->cs_write_reloc(cs_copy, r300_buffer(bo)->cs_buf, rd, wd); \
+    cs_winsys->cs_write_reloc(cs_copy, r300_buffer(bo)->cs_buf); \
     CS_DEBUG(cs_count -= 2;) \
 } while (0)
 
diff --git a/src/gallium/drivers/r300/r300_emit.c b/src/gallium/drivers/r300/r300_emit.c
index 9e0df30e527..d14cdcbbaf0 100644
--- a/src/gallium/drivers/r300/r300_emit.c
+++ b/src/gallium/drivers/r300/r300_emit.c
@@ -353,10 +353,10 @@ void r300_emit_aa_state(struct r300_context *r300, unsigned size, void *state)
 
     if (aa->dest) {
         OUT_CS_REG_SEQ(R300_RB3D_AARESOLVE_OFFSET, 1);
-        OUT_CS_RELOC(aa->dest->cs_buffer, aa->dest->offset, 0, aa->dest->domain);
+        OUT_CS_RELOC(aa->dest->cs_buffer, aa->dest->offset);
 
         OUT_CS_REG_SEQ(R300_RB3D_AARESOLVE_PITCH, 1);
-        OUT_CS_RELOC(aa->dest->cs_buffer, aa->dest->pitch, 0, aa->dest->domain);
+        OUT_CS_RELOC(aa->dest->cs_buffer, aa->dest->pitch);
     }
 
     OUT_CS_REG(R300_RB3D_AARESOLVE_CTL, aa->aaresolve_ctl);
@@ -369,6 +369,8 @@ void r300_emit_fb_state(struct r300_context* r300, unsigned size, void* state)
     struct r300_surface* surf;
     unsigned i;
     boolean can_hyperz = r300->rws->get_value(r300->rws, R300_CAN_HYPERZ);
+    uint32_t rb3d_cctl = 0;
+
     CS_LOCALS(r300);
 
     BEGIN_CS(size);
@@ -376,21 +378,24 @@ void r300_emit_fb_state(struct r300_context* r300, unsigned size, void* state)
     /* NUM_MULTIWRITES replicates COLOR[0] to all colorbuffers, which is not
      * what we usually want. */
     if (r300->screen->caps.is_r500) {
-        OUT_CS_REG(R300_RB3D_CCTL,
-            R300_RB3D_CCTL_INDEPENDENT_COLORFORMAT_ENABLE_ENABLE);
-    } else {
-        OUT_CS_REG(R300_RB3D_CCTL, 0);
+        rb3d_cctl = R300_RB3D_CCTL_INDEPENDENT_COLORFORMAT_ENABLE_ENABLE;
     }
+    if (fb->nr_cbufs &&
+        r300_fragment_shader_writes_all(r300_fs(r300))) {
+        rb3d_cctl |= R300_RB3D_CCTL_NUM_MULTIWRITES(fb->nr_cbufs);
+    }
+
+    OUT_CS_REG(R300_RB3D_CCTL, rb3d_cctl);
 
     /* Set up colorbuffers. */
     for (i = 0; i < fb->nr_cbufs; i++) {
         surf = r300_surface(fb->cbufs[i]);
 
         OUT_CS_REG_SEQ(R300_RB3D_COLOROFFSET0 + (4 * i), 1);
-        OUT_CS_RELOC(surf->cs_buffer, surf->offset, 0, surf->domain);
+        OUT_CS_RELOC(surf->cs_buffer, surf->offset);
 
         OUT_CS_REG_SEQ(R300_RB3D_COLORPITCH0 + (4 * i), 1);
-        OUT_CS_RELOC(surf->cs_buffer, surf->pitch, 0, surf->domain);
+        OUT_CS_RELOC(surf->cs_buffer, surf->pitch);
     }
 
     /* Set up the ZB part of the CBZB clear. */
@@ -400,10 +405,10 @@ void r300_emit_fb_state(struct r300_context* r300, unsigned size, void* state)
         OUT_CS_REG(R300_ZB_FORMAT, surf->cbzb_format);
 
         OUT_CS_REG_SEQ(R300_ZB_DEPTHOFFSET, 1);
-        OUT_CS_RELOC(surf->cs_buffer, surf->cbzb_midpoint_offset, 0, surf->domain);
+        OUT_CS_RELOC(surf->cs_buffer, surf->cbzb_midpoint_offset);
 
         OUT_CS_REG_SEQ(R300_ZB_DEPTHPITCH, 1);
-        OUT_CS_RELOC(surf->cs_buffer, surf->cbzb_pitch, 0, surf->domain);
+        OUT_CS_RELOC(surf->cs_buffer, surf->cbzb_pitch);
 
         DBG(r300, DBG_CBZB,
             "CBZB clearing cbuf %08x %08x\n", surf->cbzb_format,
@@ -416,10 +421,10 @@ void r300_emit_fb_state(struct r300_context* r300, unsigned size, void* state)
         OUT_CS_REG(R300_ZB_FORMAT, surf->format);
 
         OUT_CS_REG_SEQ(R300_ZB_DEPTHOFFSET, 1);
-        OUT_CS_RELOC(surf->cs_buffer, surf->offset, 0, surf->domain);
+        OUT_CS_RELOC(surf->cs_buffer, surf->offset);
 
         OUT_CS_REG_SEQ(R300_ZB_DEPTHPITCH, 1);
-        OUT_CS_RELOC(surf->cs_buffer, surf->pitch, 0, surf->domain);
+        OUT_CS_RELOC(surf->cs_buffer, surf->pitch);
 
         if (can_hyperz) {
             uint32_t surf_pitch;
@@ -482,15 +487,21 @@ void r300_emit_fb_state_pipelined(struct r300_context *r300,
 {
     struct pipe_framebuffer_state* fb =
             (struct pipe_framebuffer_state*)r300->fb_state.state;
-    unsigned i;
+    unsigned i, num_cbufs = fb->nr_cbufs;
     CS_LOCALS(r300);
 
+    /* If we use the multiwrite feature, the colorbuffers 2,3,4 must be
+     * marked as UNUSED in the US block. */
+    if (r300_fragment_shader_writes_all(r300_fs(r300))) {
+        num_cbufs = MIN2(num_cbufs, 1);
+    }
+
     BEGIN_CS(size);
 
     /* Colorbuffer format in the US block.
      * (must be written after unpipelined regs) */
     OUT_CS_REG_SEQ(R300_US_OUT_FMT_0, 4);
-    for (i = 0; i < fb->nr_cbufs; i++) {
+    for (i = 0; i < num_cbufs; i++) {
         OUT_CS(r300_surface(fb->cbufs[i])->format);
     }
     for (; i < 4; i++) {
@@ -578,28 +589,24 @@ static void r300_emit_query_end_frag_pipes(struct r300_context *r300,
             /* pipe 3 only */
             OUT_CS_REG(R300_SU_REG_DEST, 1 << 3);
             OUT_CS_REG_SEQ(R300_ZB_ZPASS_ADDR, 1);
-            OUT_CS_RELOC(buf, (query->num_results + 3) * 4,
-                    0, query->domain);
+            OUT_CS_RELOC(buf, (query->num_results + 3) * 4);
         case 3:
             /* pipe 2 only */
             OUT_CS_REG(R300_SU_REG_DEST, 1 << 2);
             OUT_CS_REG_SEQ(R300_ZB_ZPASS_ADDR, 1);
-            OUT_CS_RELOC(buf, (query->num_results + 2) * 4,
-                    0, query->domain);
+            OUT_CS_RELOC(buf, (query->num_results + 2) * 4);
         case 2:
             /* pipe 1 only */
             /* As mentioned above, accomodate RV380 and older. */
             OUT_CS_REG(R300_SU_REG_DEST,
                     1 << (caps->high_second_pipe ? 3 : 1));
             OUT_CS_REG_SEQ(R300_ZB_ZPASS_ADDR, 1);
-            OUT_CS_RELOC(buf, (query->num_results + 1) * 4,
-                    0, query->domain);
+            OUT_CS_RELOC(buf, (query->num_results + 1) * 4);
         case 1:
             /* pipe 0 only */
             OUT_CS_REG(R300_SU_REG_DEST, 1 << 0);
             OUT_CS_REG_SEQ(R300_ZB_ZPASS_ADDR, 1);
-            OUT_CS_RELOC(buf, (query->num_results + 0) * 4,
-                    0, query->domain);
+            OUT_CS_RELOC(buf, (query->num_results + 0) * 4);
             break;
         default:
             fprintf(stderr, "r300: Implementation error: Chipset reports %d"
@@ -621,7 +628,7 @@ static void rv530_emit_query_end_single_z(struct r300_context *r300,
     BEGIN_CS(8);
     OUT_CS_REG(RV530_FG_ZBREG_DEST, RV530_FG_ZBREG_DEST_PIPE_SELECT_0);
     OUT_CS_REG_SEQ(R300_ZB_ZPASS_ADDR, 1);
-    OUT_CS_RELOC(buf, query->num_results * 4, 0, query->domain);
+    OUT_CS_RELOC(buf, query->num_results * 4);
     OUT_CS_REG(RV530_FG_ZBREG_DEST, RV530_FG_ZBREG_DEST_PIPE_SELECT_ALL);
     END_CS;
 }
@@ -635,10 +642,10 @@ static void rv530_emit_query_end_double_z(struct r300_context *r300,
     BEGIN_CS(14);
     OUT_CS_REG(RV530_FG_ZBREG_DEST, RV530_FG_ZBREG_DEST_PIPE_SELECT_0);
     OUT_CS_REG_SEQ(R300_ZB_ZPASS_ADDR, 1);
-    OUT_CS_RELOC(buf, (query->num_results + 0) * 4, 0, query->domain);
+    OUT_CS_RELOC(buf, (query->num_results + 0) * 4);
     OUT_CS_REG(RV530_FG_ZBREG_DEST, RV530_FG_ZBREG_DEST_PIPE_SELECT_1);
     OUT_CS_REG_SEQ(R300_ZB_ZPASS_ADDR, 1);
-    OUT_CS_RELOC(buf, (query->num_results + 1) * 4, 0, query->domain);
+    OUT_CS_RELOC(buf, (query->num_results + 1) * 4);
     OUT_CS_REG(RV530_FG_ZBREG_DEST, RV530_FG_ZBREG_DEST_PIPE_SELECT_ALL);
     END_CS;
 }
@@ -799,24 +806,23 @@ void r300_emit_textures_state(struct r300_context *r300,
             OUT_CS_REG(R300_TX_FORMAT2_0 + (i * 4), texstate->format.format2);
 
             OUT_CS_REG_SEQ(R300_TX_OFFSET_0 + (i * 4), 1);
-            OUT_CS_TEX_RELOC(tex, texstate->format.tile_config, tex->domain,
-                             0);
+            OUT_CS_TEX_RELOC(tex, texstate->format.tile_config);
         }
     }
     END_CS;
 }
 
-static void r300_update_aos_cb(struct r300_context *r300, unsigned packet_size)
+static void r300_update_vertex_arrays_cb(struct r300_context *r300, unsigned packet_size)
 {
     struct pipe_vertex_buffer *vb1, *vb2, *vbuf = r300->vertex_buffer;
     struct pipe_vertex_element *velem = r300->velems->velem;
     unsigned *hw_format_size = r300->velems->hw_format_size;
-    unsigned size1, size2, aos_count = r300->velems->count;
+    unsigned size1, size2, vertex_array_count = r300->velems->count;
     int i;
     CB_LOCALS;
 
-    BEGIN_CB(r300->aos_cb, packet_size);
-    for (i = 0; i < aos_count - 1; i += 2) {
+    BEGIN_CB(r300->vertex_arrays_cb, packet_size);
+    for (i = 0; i < vertex_array_count - 1; i += 2) {
         vb1 = &vbuf[velem[i].vertex_buffer_index];
         vb2 = &vbuf[velem[i+1].vertex_buffer_index];
         size1 = hw_format_size[i];
@@ -828,7 +834,7 @@ static void r300_update_aos_cb(struct r300_context *r300, unsigned packet_size)
         OUT_CB(vb2->buffer_offset + velem[i+1].src_offset);
     }
 
-    if (aos_count & 1) {
+    if (vertex_array_count & 1) {
         vb1 = &vbuf[velem[i].vertex_buffer_index];
         size1 = hw_format_size[i];
 
@@ -837,34 +843,35 @@ static void r300_update_aos_cb(struct r300_context *r300, unsigned packet_size)
     }
     END_CB;
 
-    r300->aos_dirty = FALSE;
+    r300->vertex_arrays_dirty = FALSE;
 }
 
-void r300_emit_aos(struct r300_context* r300, int offset, boolean indexed)
+void r300_emit_vertex_arrays(struct r300_context* r300, int offset, boolean indexed)
 {
     struct pipe_vertex_buffer *vbuf = r300->vertex_buffer;
+    struct pipe_resource **valid_vbuf = r300->valid_vertex_buffer;
     struct pipe_vertex_element *velem = r300->velems->velem;
     struct r300_buffer *buf;
     int i;
-    unsigned aos_count = r300->velems->count;
-    unsigned packet_size = (aos_count * 3 + 1) / 2;
+    unsigned vertex_array_count = r300->velems->count;
+    unsigned packet_size = (vertex_array_count * 3 + 1) / 2;
     CS_LOCALS(r300);
 
-    BEGIN_CS(2 + packet_size + aos_count * 2);
+    BEGIN_CS(2 + packet_size + vertex_array_count * 2);
     OUT_CS_PKT3(R300_PACKET3_3D_LOAD_VBPNTR, packet_size);
-    OUT_CS(aos_count | (!indexed ? R300_VC_FORCE_PREFETCH : 0));
+    OUT_CS(vertex_array_count | (!indexed ? R300_VC_FORCE_PREFETCH : 0));
 
     if (!offset) {
-        if (r300->aos_dirty) {
-            r300_update_aos_cb(r300, packet_size);
+        if (r300->vertex_arrays_dirty) {
+            r300_update_vertex_arrays_cb(r300, packet_size);
         }
-        OUT_CS_TABLE(r300->aos_cb, packet_size);
+        OUT_CS_TABLE(r300->vertex_arrays_cb, packet_size);
     } else {
         struct pipe_vertex_buffer *vb1, *vb2;
         unsigned *hw_format_size = r300->velems->hw_format_size;
         unsigned size1, size2;
 
-        for (i = 0; i < aos_count - 1; i += 2) {
+        for (i = 0; i < vertex_array_count - 1; i += 2) {
             vb1 = &vbuf[velem[i].vertex_buffer_index];
             vb2 = &vbuf[velem[i+1].vertex_buffer_index];
             size1 = hw_format_size[i];
@@ -876,7 +883,7 @@ void r300_emit_aos(struct r300_context* r300, int offset, boolean indexed)
             OUT_CS(vb2->buffer_offset + velem[i+1].src_offset + offset * vb2->stride);
         }
 
-        if (aos_count & 1) {
+        if (vertex_array_count & 1) {
             vb1 = &vbuf[velem[i].vertex_buffer_index];
             size1 = hw_format_size[i];
 
@@ -885,14 +892,14 @@ void r300_emit_aos(struct r300_context* r300, int offset, boolean indexed)
         }
     }
 
-    for (i = 0; i < aos_count; i++) {
-        buf = r300_buffer(vbuf[velem[i].vertex_buffer_index].buffer);
-        OUT_CS_BUF_RELOC_NO_OFFSET(&buf->b.b, buf->domain, 0);
+    for (i = 0; i < vertex_array_count; i++) {
+        buf = r300_buffer(valid_vbuf[velem[i].vertex_buffer_index]);
+        OUT_CS_BUF_RELOC_NO_OFFSET(&buf->b.b);
     }
     END_CS;
 }
 
-void r300_emit_aos_swtcl(struct r300_context *r300, boolean indexed)
+void r300_emit_vertex_arrays_swtcl(struct r300_context *r300, boolean indexed)
 {
     CS_LOCALS(r300);
 
@@ -912,7 +919,7 @@ void r300_emit_aos_swtcl(struct r300_context *r300, boolean indexed)
     OUT_CS(r300->vertex_info.size |
             (r300->vertex_info.size << 8));
     OUT_CS(r300->draw_vbo_offset);
-    OUT_CS_BUF_RELOC(r300->vbo, 0, r300_buffer(r300->vbo)->domain, 0);
+    OUT_CS_BUF_RELOC(r300->vbo, 0);
     END_CS;
 }
 
@@ -1208,68 +1215,77 @@ boolean r300_emit_buffer_validate(struct r300_context *r300,
                                   boolean do_validate_vertex_buffers,
                                   struct pipe_resource *index_buffer)
 {
-    struct pipe_framebuffer_state* fb =
+    struct pipe_framebuffer_state *fb =
         (struct pipe_framebuffer_state*)r300->fb_state.state;
     struct r300_textures_state *texstate =
         (struct r300_textures_state*)r300->textures_state.state;
-    struct r300_texture* tex;
-    struct pipe_vertex_buffer *vbuf = r300->vertex_buffer;
-    struct pipe_vertex_element *velem = r300->velems->velem;
-    struct pipe_resource *pbuf;
+    struct r300_texture *tex;
     unsigned i;
-
-    /* Clean out BOs. */
-    r300->rws->cs_reset_buffers(r300->cs);
-
-    /* Color buffers... */
-    for (i = 0; i < fb->nr_cbufs; i++) {
-        tex = r300_texture(fb->cbufs[i]->texture);
-        assert(tex && tex->buffer && "cbuf is marked, but NULL!");
-        r300->rws->cs_add_buffer(r300->cs, tex->cs_buffer, 0,
-                                 r300_surface(fb->cbufs[i])->domain);
-    }
-    /* ...depth buffer... */
-    if (fb->zsbuf) {
-        tex = r300_texture(fb->zsbuf->texture);
-        assert(tex && tex->buffer && "zsbuf is marked, but NULL!");
-        r300->rws->cs_add_buffer(r300->cs, tex->cs_buffer, 0,
-                                 r300_surface(fb->zsbuf)->domain);
-    }
-    /* ...textures... */
-    for (i = 0; i < texstate->count; i++) {
-        if (!(texstate->tx_enable & (1 << i))) {
-            continue;
+    boolean flushed = FALSE;
+
+validate:
+    if (r300->fb_state.dirty) {
+        /* Color buffers... */
+        for (i = 0; i < fb->nr_cbufs; i++) {
+            tex = r300_texture(fb->cbufs[i]->texture);
+            assert(tex && tex->buffer && "cbuf is marked, but NULL!");
+            r300->rws->cs_add_reloc(r300->cs, tex->cs_buffer, 0,
+                                    r300_surface(fb->cbufs[i])->domain);
         }
+        /* ...depth buffer... */
+        if (fb->zsbuf) {
+            tex = r300_texture(fb->zsbuf->texture);
+            assert(tex && tex->buffer && "zsbuf is marked, but NULL!");
+            r300->rws->cs_add_reloc(r300->cs, tex->cs_buffer, 0,
+                                    r300_surface(fb->zsbuf)->domain);
+        }
+    }
+    if (r300->textures_state.dirty) {
+        /* ...textures... */
+        for (i = 0; i < texstate->count; i++) {
+            if (!(texstate->tx_enable & (1 << i))) {
+                continue;
+            }
 
-        tex = r300_texture(texstate->sampler_views[i]->base.texture);
-        r300->rws->cs_add_buffer(r300->cs, tex->cs_buffer, tex->domain, 0);
+            tex = r300_texture(texstate->sampler_views[i]->base.texture);
+            r300->rws->cs_add_reloc(r300->cs, tex->cs_buffer, tex->domain, 0);
+        }
     }
     /* ...occlusion query buffer... */
     if (r300->query_current)
-        r300->rws->cs_add_buffer(r300->cs, r300->query_current->cs_buffer,
-                                 0, r300->query_current->domain);
+        r300->rws->cs_add_reloc(r300->cs, r300->query_current->cs_buffer,
+                                0, r300->query_current->domain);
     /* ...vertex buffer for SWTCL path... */
     if (r300->vbo)
-        r300->rws->cs_add_buffer(r300->cs, r300_buffer(r300->vbo)->cs_buf,
-                                 r300_buffer(r300->vbo)->domain, 0);
+        r300->rws->cs_add_reloc(r300->cs, r300_buffer(r300->vbo)->cs_buf,
+                                r300_buffer(r300->vbo)->domain, 0);
     /* ...vertex buffers for HWTCL path... */
     if (do_validate_vertex_buffers) {
-        for (i = 0; i < r300->velems->count; i++) {
-            pbuf = vbuf[velem[i].vertex_buffer_index].buffer;
-            if (!pbuf)
+        struct pipe_resource **buf = r300->valid_vertex_buffer;
+        struct pipe_resource **last = r300->valid_vertex_buffer +
+                                      r300->vertex_buffer_count;
+        for (; buf != last; buf++) {
+            if (!*buf)
                 continue;
 
-            r300->rws->cs_add_buffer(r300->cs, r300_buffer(pbuf)->cs_buf,
-                                     r300_buffer(pbuf)->domain, 0);
+            r300->rws->cs_add_reloc(r300->cs, r300_buffer(*buf)->cs_buf,
+                                    r300_buffer(*buf)->domain, 0);
         }
     }
     /* ...and index buffer for HWTCL path. */
     if (index_buffer)
-        r300->rws->cs_add_buffer(r300->cs, r300_buffer(index_buffer)->cs_buf,
-                                 r300_buffer(index_buffer)->domain, 0);
+        r300->rws->cs_add_reloc(r300->cs, r300_buffer(index_buffer)->cs_buf,
+                                r300_buffer(index_buffer)->domain, 0);
 
+    /* Now do the validation. */
     if (!r300->rws->cs_validate(r300->cs)) {
-        return FALSE;
+        /* Ooops, an infinite loop, give up. */
+        if (flushed)
+            return FALSE;
+
+        r300->context.flush(&r300->context, 0, NULL);
+        flushed = TRUE;
+        goto validate;
     }
 
     return TRUE;
diff --git a/src/gallium/drivers/r300/r300_emit.h b/src/gallium/drivers/r300/r300_emit.h
index 278dbcb4c7c..acea51d942f 100644
--- a/src/gallium/drivers/r300/r300_emit.h
+++ b/src/gallium/drivers/r300/r300_emit.h
@@ -31,7 +31,7 @@ struct r300_vertex_program_code;
 
 uint32_t pack_float24(float f);
 
-void r300_emit_aos(struct r300_context* r300, int offset, boolean indexed);
+void r300_emit_vertex_arrays(struct r300_context* r300, int offset, boolean indexed);
 
 void r300_emit_blend_state(struct r300_context* r300,
                            unsigned size, void* state);
@@ -86,7 +86,7 @@ void r300_emit_scissor_state(struct r300_context* r300,
 void r300_emit_textures_state(struct r300_context *r300,
                               unsigned size, void *state);
 
-void r300_emit_aos_swtcl(struct r300_context *r300, boolean indexed);
+void r300_emit_vertex_arrays_swtcl(struct r300_context *r300, boolean indexed);
 
 void r300_emit_vap_invariant_state(struct r300_context *r300,
                                    unsigned size, void *state);
diff --git a/src/gallium/drivers/r300/r300_flush.c b/src/gallium/drivers/r300/r300_flush.c
index 451fe525b40..b250532ba92 100644
--- a/src/gallium/drivers/r300/r300_flush.c
+++ b/src/gallium/drivers/r300/r300_flush.c
@@ -70,6 +70,12 @@ static void r300_flush(struct pipe_context* pipe,
         }
 
         r300->validate_buffers = TRUE;
+        r300->upload_vb_validated = FALSE;
+        r300->upload_ib_validated = FALSE;
+    } else {
+        /* Even if hw is not dirty, we should at least reset the CS in case
+         * the space checking failed for the first draw operation. */
+        r300->rws->cs_flush(r300->cs);
     }
 
     /* reset flushed query */
diff --git a/src/gallium/drivers/r300/r300_fs.c b/src/gallium/drivers/r300/r300_fs.c
index 2936c3486e2..6d4091dc87d 100644
--- a/src/gallium/drivers/r300/r300_fs.c
+++ b/src/gallium/drivers/r300/r300_fs.c
@@ -395,6 +395,13 @@ static void r300_translate_fragment_shader(
 
     find_output_registers(&compiler, shader);
 
+    shader->write_all = FALSE;
+    for (i = 0; i < shader->info.num_properties; i++) {
+        if (shader->info.properties[i].name == TGSI_PROPERTY_FS_COLOR0_WRITES_ALL_CBUFS) {
+            shader->write_all = TRUE;
+        }
+    }
+
     if (compiler.Base.Debug & RC_DBG_LOG) {
         DBG(r300, DBG_FP, "r300: Initial fragment program\n");
         tgsi_dump(tokens, 0);
diff --git a/src/gallium/drivers/r300/r300_fs.h b/src/gallium/drivers/r300/r300_fs.h
index 51bfa88c5ef..c86a90b85ae 100644
--- a/src/gallium/drivers/r300/r300_fs.h
+++ b/src/gallium/drivers/r300/r300_fs.h
@@ -54,6 +54,9 @@ struct r300_fragment_shader_code {
     uint32_t *cb_code;
 
     struct r300_fragment_shader_code* next;
+
+    boolean write_all;
+
 };
 
 struct r300_fragment_shader {
@@ -81,4 +84,10 @@ static INLINE boolean r300_fragment_shader_writes_depth(struct r300_fragment_sha
     return (fs->shader->code.writes_depth) ? TRUE : FALSE;
 }
 
+static INLINE boolean r300_fragment_shader_writes_all(struct r300_fragment_shader *fs)
+{
+    if (!fs)
+        return FALSE;
+    return (fs->shader->write_all) ? TRUE : FALSE;
+}
 #endif /* R300_FS_H */
diff --git a/src/gallium/drivers/r300/r300_reg.h b/src/gallium/drivers/r300/r300_reg.h
index 613186e8156..d1154dee40a 100644
--- a/src/gallium/drivers/r300/r300_reg.h
+++ b/src/gallium/drivers/r300/r300_reg.h
@@ -2631,8 +2631,8 @@ USE OR OTHER DEALINGS IN THE SOFTWARE.
 #define R300_ZB_BW_CNTL                     0x4f1c
 #	define R300_HIZ_DISABLE                              (0 << 0)
 #	define R300_HIZ_ENABLE                               (1 << 0)
-#	define R300_HIZ_MIN                                  (0 << 1)
-#	define R300_HIZ_MAX                                  (1 << 1)
+#	define R300_HIZ_MAX                                  (0 << 1)
+#	define R300_HIZ_MIN                                  (1 << 1)
 #	define R300_FAST_FILL_DISABLE                        (0 << 2)
 #	define R300_FAST_FILL_ENABLE                         (1 << 2)
 #	define R300_RD_COMP_DISABLE                          (0 << 3)
diff --git a/src/gallium/drivers/r300/r300_render.c b/src/gallium/drivers/r300/r300_render.c
index f58d511e11b..e660ca68f1b 100644
--- a/src/gallium/drivers/r300/r300_render.c
+++ b/src/gallium/drivers/r300/r300_render.c
@@ -130,7 +130,7 @@ void r500_emit_index_bias(struct r300_context *r300, int index_bias)
 
 /* This function splits the index bias value into two parts:
  * - buffer_offset: the value that can be safely added to buffer offsets
- *   in r300_emit_aos (it must yield a positive offset when added to
+ *   in r300_emit_vertex_arrays (it must yield a positive offset when added to
  *   a vertex buffer offset)
  * - index_offset: the value that must be manually subtracted from indices
  *   in an index buffer to achieve negative offsets. */
@@ -166,8 +166,8 @@ static void r300_split_index_bias(struct r300_context *r300, int index_bias,
 enum r300_prepare_flags {
     PREP_FIRST_DRAW     = (1 << 0), /* call emit_dirty_state and friends? */
     PREP_VALIDATE_VBOS  = (1 << 1), /* validate VBOs? */
-    PREP_EMIT_AOS       = (1 << 2), /* call emit_aos? */
-    PREP_EMIT_AOS_SWTCL = (1 << 3), /* call emit_aos_swtcl? */
+    PREP_EMIT_AOS       = (1 << 2), /* call emit_vertex_arrays? */
+    PREP_EMIT_AOS_SWTCL = (1 << 3), /* call emit_vertex_arrays_swtcl? */
     PREP_INDEXED        = (1 << 4)  /* is this draw_elements? */
 };
 
@@ -185,8 +185,8 @@ static boolean r300_reserve_cs_dwords(struct r300_context *r300,
 {
     boolean flushed        = FALSE;
     boolean first_draw     = flags & PREP_FIRST_DRAW;
-    boolean emit_aos       = flags & PREP_EMIT_AOS;
-    boolean emit_aos_swtcl = flags & PREP_EMIT_AOS_SWTCL;
+    boolean emit_vertex_arrays       = flags & PREP_EMIT_AOS;
+    boolean emit_vertex_arrays_swtcl = flags & PREP_EMIT_AOS_SWTCL;
 
     /* Add dirty state, index offset, and AOS. */
     if (first_draw) {
@@ -195,11 +195,11 @@ static boolean r300_reserve_cs_dwords(struct r300_context *r300,
         if (r300->screen->caps.index_bias_supported)
             cs_dwords += 2; /* emit_index_offset */
 
-        if (emit_aos)
-            cs_dwords += 55; /* emit_aos */
+        if (emit_vertex_arrays)
+            cs_dwords += 55; /* emit_vertex_arrays */
 
-        if (emit_aos_swtcl)
-            cs_dwords += 7; /* emit_aos_swtcl */
+        if (emit_vertex_arrays_swtcl)
+            cs_dwords += 7; /* emit_vertex_arrays_swtcl */
     }
 
     cs_dwords += r300_get_num_cs_end_dwords(r300);
@@ -218,30 +218,24 @@ static boolean r300_reserve_cs_dwords(struct r300_context *r300,
  * \param r300          The context.
  * \param flags         See r300_prepare_flags.
  * \param index_buffer  The index buffer to validate. The parameter may be NULL.
- * \param aos_offset    The offset passed to emit_aos.
+ * \param buffer_offset The offset passed to emit_vertex_arrays.
  * \param index_bias    The index bias to emit.
  * \return TRUE if rendering should be skipped
  */
 static boolean r300_emit_states(struct r300_context *r300,
                                 enum r300_prepare_flags flags,
                                 struct pipe_resource *index_buffer,
-                                int aos_offset,
+                                int buffer_offset,
                                 int index_bias)
 {
     boolean first_draw     = flags & PREP_FIRST_DRAW;
-    boolean emit_aos       = flags & PREP_EMIT_AOS;
-    boolean emit_aos_swtcl = flags & PREP_EMIT_AOS_SWTCL;
+    boolean emit_vertex_arrays       = flags & PREP_EMIT_AOS;
+    boolean emit_vertex_arrays_swtcl = flags & PREP_EMIT_AOS_SWTCL;
     boolean indexed        = flags & PREP_INDEXED;
     boolean validate_vbos  = flags & PREP_VALIDATE_VBOS;
 
     /* Validate buffers and emit dirty state if needed. */
     if (first_draw) {
-        /* upload buffers first */
-        if (r300->screen->caps.has_tcl && r300->any_user_vbs) {
-            r300_upload_user_buffers(r300);
-            r300->any_user_vbs = false;
-        }
-
         if (r300->validate_buffers) {
             if (!r300_emit_buffer_validate(r300, validate_vbos,
                                            index_buffer)) {
@@ -253,6 +247,12 @@ static boolean r300_emit_states(struct r300_context *r300,
             /* Consider the validation done only if everything was validated. */
             if (validate_vbos) {
                 r300->validate_buffers = FALSE;
+                if (r300->any_user_vbs)
+                    r300->upload_vb_validated = TRUE;
+                if (r300->index_buffer.buffer &&
+                    r300_is_user_buffer(r300->index_buffer.buffer)) {
+                    r300->upload_ib_validated = TRUE;
+                }
             }
         }
 
@@ -264,11 +264,11 @@ static boolean r300_emit_states(struct r300_context *r300,
                 r500_emit_index_bias(r300, 0);
         }
 
-        if (emit_aos)
-            r300_emit_aos(r300, aos_offset, indexed);
+        if (emit_vertex_arrays)
+            r300_emit_vertex_arrays(r300, buffer_offset, indexed);
 
-        if (emit_aos_swtcl)
-            r300_emit_aos_swtcl(r300, indexed);
+        if (emit_vertex_arrays_swtcl)
+            r300_emit_vertex_arrays_swtcl(r300, indexed);
     }
 
     return TRUE;
@@ -281,7 +281,7 @@ static boolean r300_emit_states(struct r300_context *r300,
  * \param flags         See r300_prepare_flags.
  * \param index_buffer  The index buffer to validate. The parameter may be NULL.
  * \param cs_dwords     The number of dwords to reserve in CS.
- * \param aos_offset    The offset passed to emit_aos.
+ * \param buffer_offset The offset passed to emit_vertex_arrays.
  * \param index_bias    The index bias to emit.
  * \return TRUE if rendering should be skipped
  */
@@ -289,20 +289,20 @@ static boolean r300_prepare_for_rendering(struct r300_context *r300,
                                           enum r300_prepare_flags flags,
                                           struct pipe_resource *index_buffer,
                                           unsigned cs_dwords,
-                                          int aos_offset,
+                                          int buffer_offset,
                                           int index_bias)
 {
     if (r300_reserve_cs_dwords(r300, flags, cs_dwords))
         flags |= PREP_FIRST_DRAW;
 
-    return r300_emit_states(r300, flags, index_buffer, aos_offset, index_bias);
+    return r300_emit_states(r300, flags, index_buffer, buffer_offset, index_bias);
 }
 
 static boolean immd_is_good_idea(struct r300_context *r300,
                                  unsigned count)
 {
     struct pipe_vertex_element* velem;
-    struct pipe_vertex_buffer* vbuf;
+    struct pipe_resource *buf;
     boolean checked[PIPE_MAX_ATTRIBS] = {0};
     unsigned vertex_element_count = r300->velems->count;
     unsigned i, vbi;
@@ -326,14 +326,13 @@ static boolean immd_is_good_idea(struct r300_context *r300,
         vbi = velem->vertex_buffer_index;
 
         if (!checked[vbi]) {
-            vbuf = &r300->vertex_buffer[vbi];
+            buf = r300->valid_vertex_buffer[vbi];
 
-            if (!(r300_buffer(vbuf->buffer)->domain & R300_DOMAIN_GTT)) {
+            if (!(r300_buffer(buf)->domain & R300_DOMAIN_GTT)) {
                 return FALSE;
             }
 
-            if (r300_buffer_is_referenced(&r300->context,
-                                          vbuf->buffer,
+            if (r300_buffer_is_referenced(&r300->context, buf,
                                           R300_REF_CS | R300_REF_HW)) {
                 /* It's a very bad idea to map it... */
                 return FALSE;
@@ -392,7 +391,7 @@ static void r300_emit_draw_arrays_immediate(struct r300_context *r300,
         /* Map the buffer. */
         if (!transfer[vbi]) {
             map[vbi] = (uint32_t*)pipe_buffer_map(&r300->context,
-                                                  vbuf->buffer,
+                                                  r300->valid_vertex_buffer[vbi],
                                                   PIPE_TRANSFER_READ,
 						  &transfer[vbi]);
             map[vbi] += (vbuf->buffer_offset / 4) + stride[i] * start;
@@ -424,7 +423,6 @@ static void r300_emit_draw_arrays_immediate(struct r300_context *r300,
         vbi = r300->velems->velem[i].vertex_buffer_index;
 
         if (transfer[vbi]) {
-            vbuf = &r300->vertex_buffer[vbi];
             pipe_buffer_unmap(&r300->context, transfer[vbi]);
             transfer[vbi] = NULL;
         }
@@ -467,10 +465,10 @@ static void r300_emit_draw_elements(struct r300_context *r300,
                                     unsigned maxIndex,
                                     unsigned mode,
                                     unsigned start,
-                                    unsigned count)
+                                    unsigned count,
+                                    uint16_t *imm_indices3)
 {
-    uint32_t count_dwords;
-    uint32_t offset_dwords = indexSize * start / sizeof(uint32_t);
+    uint32_t count_dwords, offset_dwords;
     boolean alt_num_verts = count > 65535;
     CS_LOCALS(r300);
 
@@ -480,20 +478,42 @@ static void r300_emit_draw_elements(struct r300_context *r300,
         return;
     }
 
-    maxIndex = MIN2(maxIndex, r300->vertex_buffer_max_index);
-
     DBG(r300, DBG_DRAW, "r300: Indexbuf of %u indices, min %u max %u\n",
         count, minIndex, maxIndex);
 
-    BEGIN_CS(13 + (alt_num_verts ? 2 : 0));
-    if (alt_num_verts) {
-        OUT_CS_REG(R500_VAP_ALT_NUM_VERTICES, count);
-    }
+    BEGIN_CS(5);
     OUT_CS_REG(R300_GA_COLOR_CONTROL,
             r300_provoking_vertex_fixes(r300, mode));
     OUT_CS_REG_SEQ(R300_VAP_VF_MAX_VTX_INDX, 2);
     OUT_CS(maxIndex);
     OUT_CS(minIndex);
+    END_CS;
+
+    /* If start is odd, render the first triangle with indices embedded
+     * in the command stream. This will increase start by 3 and make it
+     * even. We can then proceed without a fallback. */
+    if (indexSize == 2 && (start & 1) &&
+        mode == PIPE_PRIM_TRIANGLES) {
+        BEGIN_CS(4);
+        OUT_CS_PKT3(R300_PACKET3_3D_DRAW_INDX_2, 2);
+        OUT_CS(R300_VAP_VF_CNTL__PRIM_WALK_INDICES | (3 << 16) |
+               R300_VAP_VF_CNTL__PRIM_TRIANGLES);
+        OUT_CS(imm_indices3[1] << 16 | imm_indices3[0]);
+        OUT_CS(imm_indices3[2]);
+        END_CS;
+
+        start += 3;
+        count -= 3;
+        if (!count)
+           return;
+    }
+
+    offset_dwords = indexSize * start / sizeof(uint32_t);
+
+    BEGIN_CS(8 + (alt_num_verts ? 2 : 0));
+    if (alt_num_verts) {
+        OUT_CS_REG(R500_VAP_ALT_NUM_VERTICES, count);
+    }
     OUT_CS_PKT3(R300_PACKET3_3D_DRAW_INDX_2, 0);
     if (indexSize == 4) {
         count_dwords = count;
@@ -517,16 +537,13 @@ static void r300_emit_draw_elements(struct r300_context *r300,
     OUT_CS(R300_INDX_BUFFER_ONE_REG_WR | (R300_VAP_PORT_IDX0 >> 2) |
            (0 << R300_INDX_BUFFER_SKIP_SHIFT));
     OUT_CS(offset_dwords << 2);
-    OUT_CS_BUF_RELOC(indexBuffer, count_dwords,
-                     r300_buffer(indexBuffer)->domain, 0);
+    OUT_CS_BUF_RELOC(indexBuffer, count_dwords);
 
     END_CS;
 }
 
 /* This is the fast-path drawing & emission for HW TCL. */
 static void r300_draw_range_elements(struct pipe_context* pipe,
-                                     struct pipe_resource* indexBuffer,
-                                     unsigned indexSize,
                                      int indexBias,
                                      unsigned minIndex,
                                      unsigned maxIndex,
@@ -535,13 +552,15 @@ static void r300_draw_range_elements(struct pipe_context* pipe,
                                      unsigned count)
 {
     struct r300_context* r300 = r300_context(pipe);
+    struct pipe_resource *indexBuffer = r300->index_buffer.buffer;
+    unsigned indexSize = r300->index_buffer.index_size;
     struct pipe_resource* orgIndexBuffer = indexBuffer;
     boolean alt_num_verts = r300->screen->caps.is_r500 &&
                             count > 65536 &&
                             r300->rws->get_value(r300->rws, R300_VID_DRM_2_3_0);
     unsigned short_count;
     int buffer_offset = 0, index_offset = 0; /* for index bias emulation */
-    unsigned new_offset;
+    uint16_t indices3[3];
 
     if (indexBias && !r300->screen->caps.index_bias_supported) {
         r300_split_index_bias(r300, indexBias, &buffer_offset, &index_offset);
@@ -553,43 +572,48 @@ static void r300_draw_range_elements(struct pipe_context* pipe,
     r300_update_derived_state(r300);
 
     /* Fallback for misaligned ushort indices. */
-    if (indexSize == 2 && start % 2 == 1) {
+    if (indexSize == 2 && (start & 1) &&
+        !r300_is_user_buffer(indexBuffer)) {
         struct pipe_transfer *transfer;
         struct pipe_resource *userbuf;
+
         uint16_t *ptr = pipe_buffer_map(pipe, indexBuffer,
                                         PIPE_TRANSFER_READ, &transfer);
 
-        /* Copy the mapped index buffer directly to the upload buffer.
-         * The start index will be aligned simply from the fact that
-         * every sub-buffer in u_upload_mgr is aligned. */
-        userbuf = pipe->screen->user_buffer_create(pipe->screen,
-                                                   ptr + start, count * 2,
-                                                   PIPE_BIND_INDEX_BUFFER);
-        indexBuffer = userbuf;
-        r300_upload_index_buffer(r300, &indexBuffer, indexSize, 0, count, &new_offset);
-        pipe_resource_reference(&userbuf, NULL);
+        if (mode == PIPE_PRIM_TRIANGLES) {
+           memcpy(indices3, ptr + start, 6);
+        } else {
+            /* Copy the mapped index buffer directly to the upload buffer.
+             * The start index will be aligned simply from the fact that
+             * every sub-buffer in u_upload_mgr is aligned. */
+            userbuf = pipe->screen->user_buffer_create(pipe->screen,
+                                                       ptr, 0,
+                                                       PIPE_BIND_INDEX_BUFFER);
+            indexBuffer = userbuf;
+            r300_upload_index_buffer(r300, &indexBuffer, indexSize, &start, count);
+            pipe_resource_reference(&userbuf, NULL);
+        }
         pipe_buffer_unmap(pipe, transfer);
     } else {
-        r300_upload_index_buffer(r300, &indexBuffer, indexSize, start, count, &new_offset);
+        if (r300_is_user_buffer(indexBuffer))
+            r300_upload_index_buffer(r300, &indexBuffer, indexSize, &start, count);
     }
 
-    start = new_offset;
-
-    /* 15 dwords for emit_draw_elements. Give up if the function fails. */
+    /* 19 dwords for emit_draw_elements. Give up if the function fails. */
     if (!r300_prepare_for_rendering(r300,
             PREP_FIRST_DRAW | PREP_VALIDATE_VBOS | PREP_EMIT_AOS |
-            PREP_INDEXED, indexBuffer, 15, buffer_offset, indexBias))
+            PREP_INDEXED, indexBuffer, 19, buffer_offset, indexBias))
         goto done;
 
     if (alt_num_verts || count <= 65535) {
         r300_emit_draw_elements(r300, indexBuffer, indexSize,
-				minIndex, maxIndex, mode, start, count);
+				minIndex, maxIndex, mode, start, count, indices3);
     } else {
         do {
             short_count = MIN2(count, 65534);
             r300_emit_draw_elements(r300, indexBuffer, indexSize,
                                      minIndex, maxIndex,
-                                     mode, start, short_count);
+                                     mode, start, short_count, indices3);
 
             start += short_count;
             count -= short_count;
@@ -598,7 +622,7 @@ static void r300_draw_range_elements(struct pipe_context* pipe,
             if (count) {
                 if (!r300_prepare_for_rendering(r300,
                         PREP_VALIDATE_VBOS | PREP_EMIT_AOS | PREP_INDEXED,
-                        indexBuffer, 15, buffer_offset, indexBias))
+                        indexBuffer, 19, buffer_offset, indexBias))
                     goto done;
             }
         } while (count);
@@ -659,7 +683,8 @@ static void r300_draw_vbo(struct pipe_context* pipe,
     unsigned count = info->count;
     boolean translate = FALSE;
     boolean indexed = info->indexed && r300->index_buffer.buffer;
-    unsigned start_indexed = 0;
+    unsigned min_index = 0;
+    unsigned max_index = r300->vertex_buffer_max_index;
 
     if (r300->skip_rendering) {
         return;
@@ -669,43 +694,61 @@ static void r300_draw_vbo(struct pipe_context* pipe,
         return;
     }
 
-    /* Index buffer range checking. */
     if (indexed) {
-        assert(r300->index_buffer.offset % r300->index_buffer.index_size == 0);
-
-        /* Compute start for draw_elements, taking the offset into account. */
-        start_indexed =
+        int real_min_index, real_max_index;
+        /* Compute the start for draw_elements, taking the offset into account. */
+        unsigned start_indexed =
             info->start +
             (r300->index_buffer.offset / r300->index_buffer.index_size);
 
+        assert(r300->index_buffer.offset % r300->index_buffer.index_size == 0);
+
+        /* Index buffer range checking. */
         if ((start_indexed + count) * r300->index_buffer.index_size >
             r300->index_buffer.buffer->width0) {
             fprintf(stderr, "r300: Invalid index buffer range. Skipping rendering.\n");
             return;
         }
-    }
 
-    /* Set up fallback for incompatible vertex layout if needed. */
-    if (r300->incompatible_vb_layout || r300->velems->incompatible_layout) {
-        r300_begin_vertex_translate(r300);
-        translate = TRUE;
-    }
+        min_index = MAX2(min_index, info->min_index);
+        max_index = MIN2(max_index, info->max_index);
+        real_min_index = (int)min_index - info->index_bias;
+        real_max_index = (int)max_index - info->index_bias;
 
-    if (indexed) {
-        r300_draw_range_elements(pipe,
-                                 r300->index_buffer.buffer,
-                                 r300->index_buffer.index_size,
-                                 info->index_bias,
-                                 info->min_index,
-                                 info->max_index,
-                                 info->mode,
-                                 start_indexed,
-                                 count);
+        if (max_index >= (1 << 24) - 1) {
+            fprintf(stderr, "r300: Invalid max_index: %i. Skipping rendering...\n", max_index);
+            return;
+        }
+
+        /* Set up the fallback for an incompatible vertex layout if needed. */
+        if (r300->incompatible_vb_layout || r300->velems->incompatible_layout) {
+            r300_begin_vertex_translate(r300, real_min_index, real_max_index);
+            translate = TRUE;
+        }
+
+        /* Upload vertex buffers. */
+        if (r300->any_user_vbs) {
+            r300_upload_user_buffers(r300, real_min_index, real_max_index);
+        }
+
+        r300_draw_range_elements(pipe, info->index_bias, min_index, max_index,
+                                 info->mode, start_indexed, count);
     } else {
-        r300_draw_arrays(pipe,
-                         info->mode,
-                         info->start,
-                         count);
+        min_index = MAX2(min_index, info->start);
+        max_index = MIN2(max_index, info->start + count - 1);
+
+        /* Set up the fallback for an incompatible vertex layout if needed. */
+        if (r300->incompatible_vb_layout || r300->velems->incompatible_layout) {
+            r300_begin_vertex_translate(r300, min_index, max_index);
+            translate = TRUE;
+        }
+
+        /* Upload vertex buffers. */
+        if (r300->any_user_vbs) {
+            r300_upload_user_buffers(r300, min_index, max_index);
+        }
+
+        r300_draw_arrays(pipe, info->mode, info->start, count);
     }
 
     if (translate) {
@@ -1041,8 +1084,7 @@ static struct vbuf_render* r300_render_create(struct r300_context* r300)
 
     r300render->r300 = r300;
 
-    /* XXX find real numbers plz */
-    r300render->base.max_vertex_buffer_bytes = 128 * 1024;
+    r300render->base.max_vertex_buffer_bytes = 1024 * 1024;
     r300render->base.max_indices = 16 * 1024;
 
     r300render->base.get_vertex_info = r300_render_get_vertex_info;
diff --git a/src/gallium/drivers/r300/r300_render_translate.c b/src/gallium/drivers/r300/r300_render_translate.c
index 26e00a2cad9..c48062c8084 100644
--- a/src/gallium/drivers/r300/r300_render_translate.c
+++ b/src/gallium/drivers/r300/r300_render_translate.c
@@ -31,7 +31,10 @@
 #include "translate/translate.h"
 #include "util/u_index_modify.h"
 
-void r300_begin_vertex_translate(struct r300_context *r300)
+/* XXX Optimization: use min_index and translate only that range. */
+/* XXX Use the uploader. */
+void r300_begin_vertex_translate(struct r300_context *r300,
+                                 int min_index, int max_index)
 {
     struct pipe_context *pipe = &r300->context;
     struct translate_key key = {0};
@@ -44,6 +47,7 @@ void r300_begin_vertex_translate(struct r300_context *r300)
     struct pipe_transfer *vb_transfer[PIPE_MAX_ATTRIBS] = {0}, *out_transfer;
     struct pipe_resource *out_buffer;
     unsigned i, num_verts;
+    unsigned slot;
 
     /* Initialize the translate key, i.e. the recipe how vertices should be
      * translated. */
@@ -108,12 +112,12 @@ void r300_begin_vertex_translate(struct r300_context *r300)
             vb_map[i] = pipe_buffer_map(pipe, vb->buffer,
                                         PIPE_TRANSFER_READ, &vb_transfer[i]);
 
-            tr->set_buffer(tr, i, vb_map[i], vb->stride, vb->max_index);
+            tr->set_buffer(tr, i, vb_map[i], vb->stride, max_index);
         }
     }
 
     /* Create and map the output buffer. */
-    num_verts = r300->vertex_buffer_max_index + 1;
+    num_verts = max_index + 1;
 
     out_buffer = pipe_buffer_create(&r300->screen->screen,
                                     PIPE_BIND_VERTEX_BUFFER,
@@ -135,19 +139,23 @@ void r300_begin_vertex_translate(struct r300_context *r300)
     pipe_buffer_unmap(pipe, out_transfer);
 
     /* Setup the new vertex buffer in the first free slot. */
+    slot = ~0;
     for (i = 0; i < PIPE_MAX_ATTRIBS; i++) {
         struct pipe_vertex_buffer *vb = &r300->vertex_buffer[i];
 
         if (!vb->buffer) {
-            pipe_resource_reference(&vb->buffer, out_buffer);
+            pipe_resource_reference(&r300->valid_vertex_buffer[i], out_buffer);
             vb->buffer_offset = 0;
-            vb->max_index = num_verts - 1;
             vb->stride = key.output_stride;
-            r300->tran.vb_slot = i;
+            slot = i;
+            /* XXX probably need to preserve the real count for u_blitter_save_*. */
+            r300->vertex_buffer_count = MAX2(r300->vertex_buffer_count, i+1);
             r300->validate_buffers = TRUE;
             break;
         }
     }
+    /* XXX This may fail. */
+    assert(slot != ~0);
 
     /* Save and replace vertex elements. */
     {
@@ -161,7 +169,7 @@ void r300_begin_vertex_translate(struct r300_context *r300)
                 new_velems[i].instance_divisor = ve->velem[i].instance_divisor;
                 new_velems[i].src_format = te->output_format;
                 new_velems[i].src_offset = te->output_offset;
-                new_velems[i].vertex_buffer_index = r300->tran.vb_slot;
+                new_velems[i].vertex_buffer_index = slot;
             } else {
                 memcpy(&new_velems[i], &ve->velem[i],
                        sizeof(struct pipe_vertex_element));
@@ -183,12 +191,9 @@ void r300_end_vertex_translate(struct r300_context *r300)
     /* Restore vertex elements. */
     pipe->bind_vertex_elements_state(pipe, r300->tran.saved_velems);
     pipe->delete_vertex_elements_state(pipe, r300->tran.new_velems);
-
-    /* Delete the now-unused VBO. */
-    pipe_resource_reference(&r300->vertex_buffer[r300->tran.vb_slot].buffer,
-                            NULL);
 }
 
+/* XXX Use the uploader. */
 void r300_translate_index_buffer(struct r300_context *r300,
                                  struct pipe_resource **index_buffer,
                                  unsigned *index_size, unsigned index_offset,
diff --git a/src/gallium/drivers/r300/r300_screen.c b/src/gallium/drivers/r300/r300_screen.c
index 36060ab4d08..c75aeaa10a7 100644
--- a/src/gallium/drivers/r300/r300_screen.c
+++ b/src/gallium/drivers/r300/r300_screen.c
@@ -34,6 +34,10 @@
 
 #include "draw/draw_context.h"
 
+#ifdef HAVE_LLVM
+#include "gallivm/lp_bld_init.h"
+#endif
+
 /* Return the identifier behind whom the brave coders responsible for this
  * amalgamation of code, sweat, and duct tape, routinely obscure their names.
  *
@@ -486,5 +490,9 @@ struct pipe_screen* r300_screen_create(struct r300_winsys_screen *rws)
 
     util_format_s3tc_init();
 
+#ifdef HAVE_LLVM
+    lp_build_init();
+#endif
+
     return &r300screen->screen;
 }
diff --git a/src/gallium/drivers/r300/r300_screen_buffer.c b/src/gallium/drivers/r300/r300_screen_buffer.c
index 44364435221..e3cf45479fd 100644
--- a/src/gallium/drivers/r300/r300_screen_buffer.c
+++ b/src/gallium/drivers/r300/r300_screen_buffer.c
@@ -40,7 +40,7 @@ unsigned r300_buffer_is_referenced(struct pipe_context *context,
     struct r300_context *r300 = r300_context(context);
     struct r300_buffer *rbuf = r300_buffer(buf);
 
-    if (r300_buffer_is_user_buffer(buf))
+    if (r300_is_user_buffer(buf))
  	return PIPE_UNREFERENCED;
 
     if (r300->rws->cs_is_buffer_referenced(r300->cs, rbuf->cs_buf, domain))
@@ -56,72 +56,72 @@ static unsigned r300_buffer_is_referenced_by_cs(struct pipe_context *context,
     return r300_buffer_is_referenced(context, buf, R300_REF_CS);
 }
 
-/* External helper, not required to implent u_resource_vtbl:
- */
-int r300_upload_index_buffer(struct r300_context *r300,
-			     struct pipe_resource **index_buffer,
-			     unsigned index_size,
-			     unsigned start,
-			     unsigned count,
-			     unsigned *out_offset)
+void r300_upload_index_buffer(struct r300_context *r300,
+			      struct pipe_resource **index_buffer,
+			      unsigned index_size, unsigned *start,
+			      unsigned count)
 {
-   struct pipe_resource *upload_buffer = NULL;
-   unsigned index_offset = start * index_size;
-   int ret = 0;
-
-    if (r300_buffer_is_user_buffer(*index_buffer)) {
-	ret = u_upload_buffer(r300->upload_ib,
-			      index_offset,
-			      count * index_size,
-			      *index_buffer,
-			      &index_offset,
-			      &upload_buffer);
-	if (ret) {
-	    goto done;
-	}
-	*index_buffer = upload_buffer;
-	*out_offset = index_offset / index_size;
-    } else
-        *out_offset = start;
-
- done:
-    //    if (upload_buffer)
-    //	pipe_resource_reference(&upload_buffer, NULL);
-    return ret;
+    unsigned index_offset;
+    uint8_t *ptr = r300_buffer(*index_buffer)->user_buffer;
+    boolean flushed;
+
+    *index_buffer = NULL;
+
+    u_upload_data(r300->upload_ib,
+                  0, count * index_size,
+                  ptr + (*start * index_size),
+                  &index_offset,
+                  index_buffer, &flushed);
+
+    *start = index_offset / index_size;
+
+    if (flushed || !r300->upload_ib_validated) {
+        r300->upload_ib_validated = FALSE;
+        r300->validate_buffers = TRUE;
+    }
 }
 
-/* External helper, not required to implement u_resource_vtbl:
- */
-int r300_upload_user_buffers(struct r300_context *r300)
+void r300_upload_user_buffers(struct r300_context *r300,
+                              int min_index, int max_index)
 {
-    enum pipe_error ret = PIPE_OK;
-    int i, nr;
-
-    nr = r300->velems->count;
+    int i, nr = r300->velems->count;
+    unsigned count = max_index + 1 - min_index;
+    boolean flushed;
 
     for (i = 0; i < nr; i++) {
-        struct pipe_vertex_buffer *vb =
-            &r300->vertex_buffer[r300->velems->velem[i].vertex_buffer_index];
-
-        if (r300_buffer_is_user_buffer(vb->buffer)) {
-            struct pipe_resource *upload_buffer = NULL;
-            unsigned offset = 0; /*vb->buffer_offset * 4;*/
-            unsigned size = vb->buffer->width0;
-            unsigned upload_offset;
-            ret = u_upload_buffer(r300->upload_vb,
-                                  offset, size,
-                                  vb->buffer,
-                                  &upload_offset, &upload_buffer);
-            if (ret)
-                return ret;
-
-            pipe_resource_reference(&vb->buffer, NULL);
-            vb->buffer = upload_buffer;
-            vb->buffer_offset = upload_offset;
-            r300->validate_buffers = TRUE;
+        unsigned index = r300->velems->velem[i].vertex_buffer_index;
+        struct pipe_vertex_buffer *vb = &r300->vertex_buffer[index];
+        struct r300_buffer *userbuf = r300_buffer(vb->buffer);
+
+        if (userbuf && userbuf->user_buffer) {
+            unsigned first, size;
+
+            if (vb->stride) {
+                first = vb->stride * min_index;
+                size = vb->stride * count;
+            } else {
+                first = 0;
+                size = r300->velems->hw_format_size[i];
+            }
+
+            u_upload_data(r300->upload_vb, first, size,
+                          userbuf->user_buffer + first,
+                          &vb->buffer_offset,
+                          &r300->valid_vertex_buffer[index],
+                          &flushed);
+
+            vb->buffer_offset -= first;
+
+            r300->vertex_arrays_dirty = TRUE;
+
+            if (flushed || !r300->upload_vb_validated) {
+                r300->upload_vb_validated = FALSE;
+                r300->validate_buffers = TRUE;
+            }
+        } else {
+            assert(r300->valid_vertex_buffer[index]);
         }
     }
-    return ret;
 }
 
 static void r300_buffer_destroy(struct pipe_screen *screen,
@@ -181,49 +181,17 @@ r300_buffer_transfer_map( struct pipe_context *pipe,
     struct r300_winsys_screen *rws = r300screen->rws;
     struct r300_buffer *rbuf = r300_buffer(transfer->resource);
     uint8_t *map;
-    boolean flush = FALSE;
-    unsigned i;
 
     if (rbuf->user_buffer)
         return (uint8_t *) rbuf->user_buffer + transfer->box.x;
     if (rbuf->constant_buffer)
         return (uint8_t *) rbuf->constant_buffer + transfer->box.x;
 
-    /* check if the mapping is to a range we already flushed */
-    if (transfer->usage & PIPE_TRANSFER_DISCARD) {
-	for (i = 0; i < rbuf->num_ranges; i++) {
-	    if ((transfer->box.x >= rbuf->ranges[i].start) &&
-		(transfer->box.x < rbuf->ranges[i].end))
-		flush = TRUE;
-
-	    if (flush) {
-		/* unreference this hw buffer and allocate a new one */
-		rws->buffer_reference(rws, &rbuf->buf, NULL);
-
-		rbuf->num_ranges = 0;
-                rbuf->buf =
-                    r300screen->rws->buffer_create(r300screen->rws,
-                                                   rbuf->b.b.width0, 16,
-                                                   rbuf->b.b.bind,
-                                                   rbuf->b.b.usage,
-                                                   rbuf->domain);
-                rbuf->cs_buf =
-                    r300screen->rws->buffer_get_cs_handle(r300screen->rws,
-                                                          rbuf->buf);
-		break;
-	    }
-	}
-    }
-
     map = rws->buffer_map(rws, rbuf->buf, r300->cs, transfer->usage);
 
     if (map == NULL)
         return NULL;
 
-    /* map_buffer() returned a pointer to the beginning of the buffer,
-     * but transfers are expected to return a pointer to just the
-     * region specified in the box.
-     */
     return map + transfer->box.x;
 }
 
@@ -231,30 +199,7 @@ static void r300_buffer_transfer_flush_region( struct pipe_context *pipe,
 					       struct pipe_transfer *transfer,
 					       const struct pipe_box *box)
 {
-    struct r300_buffer *rbuf = r300_buffer(transfer->resource);
-    unsigned i;
-    unsigned offset = transfer->box.x + box->x;
-    unsigned length = box->width;
-
-    assert(box->x + box->width <= transfer->box.width);
-
-    if (rbuf->user_buffer)
-	return;
-    if (rbuf->constant_buffer)
-        return;
-
-    /* mark the range as used */
-    for(i = 0; i < rbuf->num_ranges; ++i) {
-	if(offset <= rbuf->ranges[i].end && rbuf->ranges[i].start <= (offset+box->width)) {
-	    rbuf->ranges[i].start = MIN2(rbuf->ranges[i].start, offset);
-	    rbuf->ranges[i].end   = MAX2(rbuf->ranges[i].end, (offset+length));
-	    return;
-	}
-    }
-
-    rbuf->ranges[rbuf->num_ranges].start = offset;
-    rbuf->ranges[rbuf->num_ranges].end = offset+length;
-    rbuf->num_ranges++;
+    /* no-op */
 }
 
 static void r300_buffer_transfer_unmap( struct pipe_context *pipe,
@@ -278,26 +223,26 @@ static void r300_buffer_transfer_inline_write(struct pipe_context *pipe,
                                               unsigned stride,
                                               unsigned layer_stride)
 {
+    struct r300_context *r300 = r300_context(pipe);
+    struct r300_winsys_screen *rws = r300->screen->rws;
     struct r300_buffer *rbuf = r300_buffer(resource);
-    struct pipe_transfer *transfer = NULL;
     uint8_t *map = NULL;
 
     if (rbuf->constant_buffer) {
         memcpy(rbuf->constant_buffer + box->x, data, box->width);
         return;
     }
+    assert(rbuf->user_buffer == NULL);
 
-    transfer = r300_buffer_get_transfer(pipe, resource, 0,
-                        PIPE_TRANSFER_WRITE | PIPE_TRANSFER_DISCARD, box);
-    map = r300_buffer_transfer_map(pipe, transfer);
+    map = rws->buffer_map(rws, rbuf->buf, r300->cs,
+                          PIPE_TRANSFER_WRITE | PIPE_TRANSFER_DISCARD | usage);
 
-    memcpy(map, data, box->width);
+    memcpy(map + box->x, data, box->width);
 
-    r300_buffer_transfer_unmap(pipe, transfer);
-    r300_buffer_transfer_destroy(pipe, transfer);
+    rws->buffer_unmap(rws, rbuf->buf);
 }
 
-struct u_resource_vtbl r300_buffer_vtbl = 
+struct u_resource_vtbl r300_buffer_vtbl =
 {
    u_default_resource_get_handle,      /* get_handle */
    r300_buffer_destroy,                /* resource_destroy */
@@ -326,7 +271,6 @@ struct pipe_resource *r300_buffer_create(struct pipe_screen *screen,
     pipe_reference_init(&rbuf->b.b.reference, 1);
     rbuf->b.b.screen = screen;
     rbuf->domain = R300_DOMAIN_GTT;
-    rbuf->num_ranges = 0;
     rbuf->buf = NULL;
     rbuf->constant_buffer = NULL;
     rbuf->user_buffer = NULL;
@@ -354,8 +298,7 @@ struct pipe_resource *r300_buffer_create(struct pipe_screen *screen,
 }
 
 struct pipe_resource *r300_user_buffer_create(struct pipe_screen *screen,
-					      void *ptr,
-					      unsigned bytes,
+					      void *ptr, unsigned size,
 					      unsigned bind)
 {
     struct r300_screen *r300screen = r300_screen(screen);
@@ -372,13 +315,12 @@ struct pipe_resource *r300_user_buffer_create(struct pipe_screen *screen,
     rbuf->b.b.format = PIPE_FORMAT_R8_UNORM;
     rbuf->b.b.usage = PIPE_USAGE_IMMUTABLE;
     rbuf->b.b.bind = bind;
-    rbuf->b.b.width0 = bytes;
+    rbuf->b.b.width0 = ~0;
     rbuf->b.b.height0 = 1;
     rbuf->b.b.depth0 = 1;
     rbuf->b.b.array_size = 1;
     rbuf->b.b.flags = 0;
     rbuf->domain = R300_DOMAIN_GTT;
-    rbuf->num_ranges = 0;
     rbuf->buf = NULL;
     rbuf->constant_buffer = NULL;
     rbuf->user_buffer = ptr;
diff --git a/src/gallium/drivers/r300/r300_screen_buffer.h b/src/gallium/drivers/r300/r300_screen_buffer.h
index 0b3555dd813..58dec8539b6 100644
--- a/src/gallium/drivers/r300/r300_screen_buffer.h
+++ b/src/gallium/drivers/r300/r300_screen_buffer.h
@@ -57,27 +57,24 @@ struct r300_buffer
 
     uint8_t *user_buffer;
     uint8_t *constant_buffer;
-    struct r300_buffer_range ranges[R300_BUFFER_MAX_RANGES];
-    unsigned num_ranges;
 };
 
 /* Functions. */
 
-int r300_upload_user_buffers(struct r300_context *r300);
+void r300_upload_user_buffers(struct r300_context *r300,
+			      int min_index, int max_index);
 
-int r300_upload_index_buffer(struct r300_context *r300,
-			     struct pipe_resource **index_buffer,
-			     unsigned index_size,
-			     unsigned start,
-			     unsigned count, unsigned *out_offset);
+void r300_upload_index_buffer(struct r300_context *r300,
+			      struct pipe_resource **index_buffer,
+			      unsigned index_size, unsigned *start,
+			      unsigned count);
 
 struct pipe_resource *r300_buffer_create(struct pipe_screen *screen,
 					 const struct pipe_resource *templ);
 
 struct pipe_resource *r300_user_buffer_create(struct pipe_screen *screen,
-					      void *ptr,
-					      unsigned bytes,
-					      unsigned usage);
+					      void *ptr, unsigned size,
+					      unsigned bind);
 
 unsigned r300_buffer_is_referenced(struct pipe_context *context,
 				   struct pipe_resource *buf,
@@ -90,7 +87,7 @@ static INLINE struct r300_buffer *r300_buffer(struct pipe_resource *buffer)
     return (struct r300_buffer *)buffer;
 }
 
-static INLINE boolean r300_buffer_is_user_buffer(struct pipe_resource *buffer)
+static INLINE boolean r300_is_user_buffer(struct pipe_resource *buffer)
 {
     return r300_buffer(buffer)->user_buffer ? true : false;
 }
diff --git a/src/gallium/drivers/r300/r300_state.c b/src/gallium/drivers/r300/r300_state.c
index f902db54cc1..3a97b76a4c8 100644
--- a/src/gallium/drivers/r300/r300_state.c
+++ b/src/gallium/drivers/r300/r300_state.c
@@ -686,13 +686,22 @@ void r300_mark_fb_state_dirty(struct r300_context *r300,
     struct pipe_framebuffer_state *state = r300->fb_state.state;
     boolean can_hyperz = r300->rws->get_value(r300->rws, R300_CAN_HYPERZ);
 
-    /* What is marked as dirty depends on the enum r300_fb_state_change. */
     r300_mark_atom_dirty(r300, &r300->gpu_flush);
     r300_mark_atom_dirty(r300, &r300->fb_state);
-    r300_mark_atom_dirty(r300, &r300->hyperz_state);
 
+    /* What is marked as dirty depends on the enum r300_fb_state_change. */
     if (change == R300_CHANGED_FB_STATE) {
         r300_mark_atom_dirty(r300, &r300->aa_state);
+    }
+
+    if (change == R300_CHANGED_FB_STATE ||
+        change == R300_CHANGED_CBZB_FLAG ||
+        change == R300_CHANGED_ZCLEAR_FLAG) {
+        r300_mark_atom_dirty(r300, &r300->hyperz_state);
+    }
+
+    if (change == R300_CHANGED_FB_STATE ||
+        change == R300_CHANGED_MULTIWRITE) {
         r300_mark_atom_dirty(r300, &r300->fb_state_pipelined);
     }
 
@@ -876,16 +885,25 @@ static void r300_bind_fs_state(struct pipe_context* pipe, void* shader)
 {
     struct r300_context* r300 = r300_context(pipe);
     struct r300_fragment_shader* fs = (struct r300_fragment_shader*)shader;
+    struct pipe_framebuffer_state *fb = r300->fb_state.state;
+    boolean last_multi_write;
 
     if (fs == NULL) {
         r300->fs.state = NULL;
         return;
     }
 
+    last_multi_write = r300_fragment_shader_writes_all(r300_fs(r300));
+
     r300->fs.state = fs;
     r300_pick_fragment_shader(r300);
     r300_mark_fs_code_dirty(r300);
 
+    if (fb->nr_cbufs > 1 &&
+        last_multi_write != r300_fragment_shader_writes_all(fs)) {
+        r300_mark_fb_state_dirty(r300, R300_CHANGED_MULTIWRITE);
+    }
+
     r300_mark_atom_dirty(r300, &r300->rs_block_state); /* Will be updated before the emission. */
 }
 
@@ -1447,15 +1465,15 @@ static void r300_set_vertex_buffers(struct pipe_context* pipe,
                                     const struct pipe_vertex_buffer* buffers)
 {
     struct r300_context* r300 = r300_context(pipe);
-    struct pipe_vertex_buffer *vbo;
+    const struct pipe_vertex_buffer *vbo;
     unsigned i, max_index = (1 << 24) - 1;
     boolean any_user_buffer = FALSE;
+    boolean any_nonuser_buffer = FALSE;
     struct pipe_vertex_buffer dummy_vb = {0};
 
     /* There must be at least one vertex buffer set, otherwise it locks up. */
     if (!count) {
         dummy_vb.buffer = r300->dummy_vb;
-        dummy_vb.max_index = r300->dummy_vb->width0 / 4;
         buffers = &dummy_vb;
         count = 1;
     }
@@ -1482,35 +1500,41 @@ static void r300_set_vertex_buffers(struct pipe_context* pipe,
         }
 
         for (i = 0; i < count; i++) {
-            /* Why, yes, I AM casting away constness. How did you know? */
-            vbo = (struct pipe_vertex_buffer*)&buffers[i];
+            vbo = &buffers[i];
 
             /* Skip NULL buffers */
-            if (!buffers[i].buffer) {
+            if (!vbo->buffer) {
                 continue;
             }
 
-            if (r300_buffer_is_user_buffer(vbo->buffer)) {
+            /* User buffers have no info about maximum index,
+             * we will have to compute it in draw_vbo. */
+            if (r300_is_user_buffer(vbo->buffer)) {
                 any_user_buffer = TRUE;
+                continue;
             }
+            any_nonuser_buffer = TRUE;
 
             /* The stride of zero means we will be fetching only the first
              * vertex, so don't care about max_index. */
             if (!vbo->stride)
                 continue;
 
-            if (vbo->max_index == ~0) {
-                vbo->max_index =
-                        (vbo->buffer->width0 - vbo->buffer_offset) / vbo->stride;
+            /* Update the maximum index. */
+            {
+                unsigned vbo_max_index =
+                      (vbo->buffer->width0 - vbo->buffer_offset) / vbo->stride;
+                max_index = MIN2(max_index, vbo_max_index);
             }
-
-            max_index = MIN2(vbo->max_index, max_index);
         }
 
         r300->any_user_vbs = any_user_buffer;
         r300->vertex_buffer_max_index = max_index;
-        r300->aos_dirty = TRUE;
-        r300->validate_buffers = TRUE;
+        r300->vertex_arrays_dirty = TRUE;
+        if (any_nonuser_buffer)
+            r300->validate_buffers = TRUE;
+        if (!any_user_buffer)
+            r300->upload_vb_validated = FALSE;
     } else {
         /* SW TCL. */
         draw_set_vertex_buffers(r300->draw, count, buffers);
@@ -1518,16 +1542,25 @@ static void r300_set_vertex_buffers(struct pipe_context* pipe,
 
     /* Common code. */
     for (i = 0; i < count; i++) {
+        vbo = &buffers[i];
+
         /* Reference our buffer. */
-        pipe_resource_reference(&r300->vertex_buffer[i].buffer, buffers[i].buffer);
+        pipe_resource_reference(&r300->vertex_buffer[i].buffer, vbo->buffer);
+        if (vbo->buffer && r300_is_user_buffer(vbo->buffer)) {
+            pipe_resource_reference(&r300->valid_vertex_buffer[i], NULL);
+        } else {
+            pipe_resource_reference(&r300->valid_vertex_buffer[i], vbo->buffer);
+        }
     }
     for (; i < r300->vertex_buffer_count; i++) {
         /* Dereference any old buffers. */
         pipe_resource_reference(&r300->vertex_buffer[i].buffer, NULL);
+        pipe_resource_reference(&r300->valid_vertex_buffer[i], NULL);
     }
 
     memcpy(r300->vertex_buffer, buffers,
-        sizeof(struct pipe_vertex_buffer) * count);
+           sizeof(struct pipe_vertex_buffer) * count);
+
     r300->vertex_buffer_count = count;
 }
 
@@ -1536,19 +1569,22 @@ static void r300_set_index_buffer(struct pipe_context* pipe,
 {
     struct r300_context* r300 = r300_context(pipe);
 
-    if (ib) {
+    if (ib && ib->buffer) {
         pipe_resource_reference(&r300->index_buffer.buffer, ib->buffer);
         memcpy(&r300->index_buffer, ib, sizeof(r300->index_buffer));
+
+        if (r300->screen->caps.has_tcl &&
+            !r300_is_user_buffer(ib->buffer)) {
+            r300->validate_buffers = TRUE;
+            r300->upload_ib_validated = FALSE;
+        }
     }
     else {
         pipe_resource_reference(&r300->index_buffer.buffer, NULL);
         memset(&r300->index_buffer, 0, sizeof(r300->index_buffer));
     }
 
-    if (r300->screen->caps.has_tcl) {
-        r300->validate_buffers = TRUE;
-    }
-    else {
+    if (!r300->screen->caps.has_tcl) {
         draw_set_index_buffer(r300->draw, ib);
     }
 }
@@ -1717,7 +1753,7 @@ static void r300_bind_vertex_elements_state(struct pipe_context *pipe,
 
     UPDATE_STATE(&velems->vertex_stream, r300->vertex_stream_state);
     r300->vertex_stream_state.size = (1 + velems->vertex_stream.count) * 2;
-    r300->aos_dirty = TRUE;
+    r300->vertex_arrays_dirty = TRUE;
 }
 
 static void r300_delete_vertex_elements_state(struct pipe_context *pipe, void *state)
@@ -1809,6 +1845,7 @@ static void r300_set_constant_buffer(struct pipe_context *pipe,
 {
     struct r300_context* r300 = r300_context(pipe);
     struct r300_constant_buffer *cbuf;
+    struct r300_buffer *rbuf = r300_buffer(buf);
     uint32_t *mapped;
 
     switch (shader) {
@@ -1822,14 +1859,18 @@ static void r300_set_constant_buffer(struct pipe_context *pipe,
             return;
     }
 
-    if (buf == NULL || buf->width0 == 0 ||
-        (mapped = (uint32_t*)r300_buffer(buf)->constant_buffer) == NULL) {
+    if (buf == NULL || buf->width0 == 0)
+        return;
+
+    if (rbuf->user_buffer)
+        mapped = (uint32_t*)rbuf->user_buffer;
+    else if (rbuf->constant_buffer)
+        mapped = (uint32_t*)rbuf->constant_buffer;
+    else
         return;
-    }
 
     if (shader == PIPE_SHADER_FRAGMENT ||
         (shader == PIPE_SHADER_VERTEX && r300->screen->caps.has_tcl)) {
-        assert((buf->width0 % (4 * sizeof(float))) == 0);
         cbuf->ptr = mapped;
     }
 
diff --git a/src/gallium/drivers/r300/r300_state_derived.c b/src/gallium/drivers/r300/r300_state_derived.c
index d5fc8ece252..d3985c11aa8 100644
--- a/src/gallium/drivers/r300/r300_state_derived.c
+++ b/src/gallium/drivers/r300/r300_state_derived.c
@@ -645,6 +645,10 @@ static uint32_t r300_get_border_color(enum pipe_format format,
     }
 
     switch (desc->channel[0].size) {
+        case 2:
+            util_pack_color(border_swizzled, PIPE_FORMAT_B2G3R3_UNORM, &uc);
+            break;
+
         case 4:
             util_pack_color(border_swizzled, PIPE_FORMAT_B4G4R4A4_UNORM, &uc);
             break;
diff --git a/src/gallium/drivers/r300/r300_texture.c b/src/gallium/drivers/r300/r300_texture.c
index 24f1d68f4a7..6fdc504ed54 100644
--- a/src/gallium/drivers/r300/r300_texture.c
+++ b/src/gallium/drivers/r300/r300_texture.c
@@ -244,6 +244,11 @@ uint32_t r300_translate_texformat(enum pipe_format format,
                     desc->channel[2].size == 6) {
                     return R300_TX_FORMAT_Z6Y5X5 | result;
                 }
+                if (desc->channel[0].size == 2 &&
+                    desc->channel[1].size == 3 &&
+                    desc->channel[2].size == 3) {
+                    return R300_TX_FORMAT_Z3Y3X2 | result;
+                }
                 return ~0; /* Unsupported/unknown. */
 
             case 4:
diff --git a/src/gallium/drivers/r300/r300_transfer.c b/src/gallium/drivers/r300/r300_transfer.c
index 3b95af79bcf..ae93fab554e 100644
--- a/src/gallium/drivers/r300/r300_transfer.c
+++ b/src/gallium/drivers/r300/r300_transfer.c
@@ -120,7 +120,6 @@ r300_texture_get_transfer(struct pipe_context *ctx,
             base.format = texture->format;
             base.width0 = box->width;
             base.height0 = box->height;
-            /* XXX: was depth0 = 0 */
             base.depth0 = 1;
             base.array_size = 1;
             base.last_level = 0;
diff --git a/src/gallium/drivers/r300/r300_winsys.h b/src/gallium/drivers/r300/r300_winsys.h
index b8324afe511..460da77a4fb 100644
--- a/src/gallium/drivers/r300/r300_winsys.h
+++ b/src/gallium/drivers/r300/r300_winsys.h
@@ -42,8 +42,8 @@ struct r300_winsys_buffer;      /* for map/unmap etc. */
 struct r300_winsys_cs_buffer;   /* for write_reloc etc. */
 
 struct r300_winsys_cs {
-    unsigned cdw;       /* Number of used dwords. */
-    uint32_t *buf;      /* The command buffer. */
+    unsigned cdw;                           /* Number of used dwords. */
+    uint32_t buf[R300_MAX_CMDBUF_DWORDS];   /* The command buffer. */
 };
 
 enum r300_value_id {
@@ -239,23 +239,22 @@ struct r300_winsys_screen {
     void (*cs_destroy)(struct r300_winsys_cs *cs);
 
     /**
-     * Add a buffer object to the list of buffers to validate.
+     * Add a new buffer relocation. Every relocation must first be added
+     * before it can be written.
      *
-     * \param cs        A command stream to add buffer for validation against.
-     * \param buf       A winsys buffer to validate.
-     * \param rd        A read domain containing a bitmask
-     *                  of the R300_DOMAIN_* flags.
-     * \param wd        A write domain containing a bitmask
-     *                  of the R300_DOMAIN_* flags.
+     * \param cs  A command stream to add buffer for validation against.
+     * \param buf A winsys buffer to validate.
+     * \param rd  A read domain containing a bitmask of the R300_DOMAIN_* flags.
+     * \param wd  A write domain containing a bitmask of the R300_DOMAIN_* flags.
      */
-    void (*cs_add_buffer)(struct r300_winsys_cs *cs,
-                          struct r300_winsys_cs_buffer *buf,
-                          enum r300_buffer_domain rd,
-                          enum r300_buffer_domain wd);
+    void (*cs_add_reloc)(struct r300_winsys_cs *cs,
+                         struct r300_winsys_cs_buffer *buf,
+                         enum r300_buffer_domain rd,
+                         enum r300_buffer_domain wd);
 
     /**
-     * Revalidate all currently set up winsys buffers.
-     * Returns TRUE if a flush is required.
+     * Return TRUE if there is enough memory in VRAM and GTT for the relocs
+     * added so far.
      *
      * \param cs        A command stream to validate.
      */
@@ -270,9 +269,7 @@ struct r300_winsys_screen {
      * \param wd        A write domain containing a bitmask of the R300_DOMAIN_* flags.
      */
     void (*cs_write_reloc)(struct r300_winsys_cs *cs,
-                           struct r300_winsys_cs_buffer *buf,
-                           enum r300_buffer_domain rd,
-                           enum r300_buffer_domain wd);
+                           struct r300_winsys_cs_buffer *buf);
 
     /**
      * Flush a command stream.
@@ -294,14 +291,6 @@ struct r300_winsys_screen {
                          void *user);
 
     /**
-     * Reset the list of buffer objects to validate, usually called
-     * prior to adding buffer objects for validation.
-     *
-     * \param cs        A command stream to reset buffers for.
-     */
-    void (*cs_reset_buffers)(struct r300_winsys_cs *cs);
-
-    /**
      * Return TRUE if a buffer is referenced by a command stream or by hardware
      * (i.e. is busy), based on the domain parameter.
      *
diff --git a/src/gallium/drivers/r600/eg_asm.c b/src/gallium/drivers/r600/eg_asm.c
index 1bb4c6b2afb..1881e633d54 100644
--- a/src/gallium/drivers/r600/eg_asm.c
+++ b/src/gallium/drivers/r600/eg_asm.c
@@ -39,15 +39,15 @@ int eg_bc_cf_build(struct r600_bc *bc, struct r600_bc_cf *cf)
 	case (EG_V_SQ_CF_ALU_WORD1_SQ_CF_INST_ALU_PUSH_BEFORE << 3):
 		assert(!end_of_program);
 		bc->bytecode[id++] = S_SQ_CF_ALU_WORD0_ADDR(cf->addr >> 1) |
-			S_SQ_CF_ALU_WORD0_KCACHE_MODE0(cf->kcache0_mode) |
-			S_SQ_CF_ALU_WORD0_KCACHE_BANK0(cf->kcache0_bank) |
-			S_SQ_CF_ALU_WORD0_KCACHE_BANK1(cf->kcache1_bank);
+			S_SQ_CF_ALU_WORD0_KCACHE_MODE0(cf->kcache[0].mode) |
+			S_SQ_CF_ALU_WORD0_KCACHE_BANK0(cf->kcache[0].bank) |
+			S_SQ_CF_ALU_WORD0_KCACHE_BANK1(cf->kcache[1].bank);
 		bc->bytecode[id++] = S_SQ_CF_ALU_WORD1_CF_INST(cf->inst >> 3) |
-			S_SQ_CF_ALU_WORD1_KCACHE_MODE1(cf->kcache1_mode) |
-			S_SQ_CF_ALU_WORD1_KCACHE_ADDR0(cf->kcache0_addr) |
-			S_SQ_CF_ALU_WORD1_KCACHE_ADDR1(cf->kcache1_addr) |
-					S_SQ_CF_ALU_WORD1_BARRIER(cf->barrier) |
-					S_SQ_CF_ALU_WORD1_COUNT((cf->ndw / 2) - 1);
+			S_SQ_CF_ALU_WORD1_KCACHE_MODE1(cf->kcache[1].mode) |
+			S_SQ_CF_ALU_WORD1_KCACHE_ADDR0(cf->kcache[0].addr) |
+			S_SQ_CF_ALU_WORD1_KCACHE_ADDR1(cf->kcache[1].addr) |
+			S_SQ_CF_ALU_WORD1_BARRIER(cf->barrier) |
+			S_SQ_CF_ALU_WORD1_COUNT((cf->ndw / 2) - 1);
 		break;
 	case EG_V_SQ_CF_WORD1_SQ_CF_INST_TEX:
 	case EG_V_SQ_CF_WORD1_SQ_CF_INST_VTX:
diff --git a/src/gallium/drivers/r600/evergreen_state.c b/src/gallium/drivers/r600/evergreen_state.c
index c6f3669c9a3..94eef77945b 100644
--- a/src/gallium/drivers/r600/evergreen_state.c
+++ b/src/gallium/drivers/r600/evergreen_state.c
@@ -842,6 +842,7 @@ static void evergreen_set_constant_buffer(struct pipe_context *ctx, uint shader,
 {
 	struct r600_pipe_context *rctx = (struct r600_pipe_context *)ctx;
 	struct r600_resource *rbuffer = (struct r600_resource*)buffer;
+	uint32_t offset;
 
 	/* Note that the state tracker can unbind constant buffers by
 	 * passing NULL here.
@@ -850,6 +851,8 @@ static void evergreen_set_constant_buffer(struct pipe_context *ctx, uint shader,
 		return;
 	}
 
+	r600_upload_const_buffer(rctx, buffer, &offset);
+
 	switch (shader) {
 	case PIPE_SHADER_VERTEX:
 		rctx->vs_const_buffer.nregs = 0;
@@ -859,7 +862,7 @@ static void evergreen_set_constant_buffer(struct pipe_context *ctx, uint shader,
 					0xFFFFFFFF, NULL);
 		r600_pipe_state_add_reg(&rctx->vs_const_buffer,
 					R_028980_ALU_CONST_CACHE_VS_0,
-					(r600_bo_offset(rbuffer->bo)) >> 8, 0xFFFFFFFF, rbuffer->bo);
+					(r600_bo_offset(rbuffer->bo) + offset) >> 8, 0xFFFFFFFF, rbuffer->bo);
 		r600_context_pipe_state_set(&rctx->ctx, &rctx->vs_const_buffer);
 		break;
 	case PIPE_SHADER_FRAGMENT:
@@ -870,7 +873,7 @@ static void evergreen_set_constant_buffer(struct pipe_context *ctx, uint shader,
 					0xFFFFFFFF, NULL);
 		r600_pipe_state_add_reg(&rctx->ps_const_buffer,
 					R_028940_ALU_CONST_CACHE_PS_0,
-					(r600_bo_offset(rbuffer->bo)) >> 8, 0xFFFFFFFF, rbuffer->bo);
+					(r600_bo_offset(rbuffer->bo) + offset) >> 8, 0xFFFFFFFF, rbuffer->bo);
 		r600_context_pipe_state_set(&rctx->ctx, &rctx->ps_const_buffer);
 		break;
 	default:
@@ -1067,12 +1070,76 @@ void evergreen_init_config(struct r600_pipe_context *rctx)
 		num_hs_stack_entries = 42;
 		num_ls_stack_entries = 42;
 		break;
+	case CHIP_BARTS:
+		num_ps_gprs = 93;
+		num_vs_gprs = 46;
+		num_temp_gprs = 4;
+		num_gs_gprs = 31;
+		num_es_gprs = 31;
+		num_hs_gprs = 23;
+		num_ls_gprs = 23;
+		num_ps_threads = 128;
+		num_vs_threads = 20;
+		num_gs_threads = 20;
+		num_es_threads = 20;
+		num_hs_threads = 20;
+		num_ls_threads = 20;
+		num_ps_stack_entries = 85;
+		num_vs_stack_entries = 85;
+		num_gs_stack_entries = 85;
+		num_es_stack_entries = 85;
+		num_hs_stack_entries = 85;
+		num_ls_stack_entries = 85;
+		break;
+	case CHIP_TURKS:
+		num_ps_gprs = 93;
+		num_vs_gprs = 46;
+		num_temp_gprs = 4;
+		num_gs_gprs = 31;
+		num_es_gprs = 31;
+		num_hs_gprs = 23;
+		num_ls_gprs = 23;
+		num_ps_threads = 128;
+		num_vs_threads = 20;
+		num_gs_threads = 20;
+		num_es_threads = 20;
+		num_hs_threads = 20;
+		num_ls_threads = 20;
+		num_ps_stack_entries = 42;
+		num_vs_stack_entries = 42;
+		num_gs_stack_entries = 42;
+		num_es_stack_entries = 42;
+		num_hs_stack_entries = 42;
+		num_ls_stack_entries = 42;
+		break;
+	case CHIP_CAICOS:
+		num_ps_gprs = 93;
+		num_vs_gprs = 46;
+		num_temp_gprs = 4;
+		num_gs_gprs = 31;
+		num_es_gprs = 31;
+		num_hs_gprs = 23;
+		num_ls_gprs = 23;
+		num_ps_threads = 128;
+		num_vs_threads = 10;
+		num_gs_threads = 10;
+		num_es_threads = 10;
+		num_hs_threads = 10;
+		num_ls_threads = 10;
+		num_ps_stack_entries = 42;
+		num_vs_stack_entries = 42;
+		num_gs_stack_entries = 42;
+		num_es_stack_entries = 42;
+		num_hs_stack_entries = 42;
+		num_ls_stack_entries = 42;
+		break;
 	}
 
 	tmp = 0x00000000;
 	switch (family) {
 	case CHIP_CEDAR:
 	case CHIP_PALM:
+	case CHIP_CAICOS:
 		break;
 	default:
 		tmp |= S_008C00_VC_ENABLE(1);
diff --git a/src/gallium/drivers/r600/r600.h b/src/gallium/drivers/r600/r600.h
index 85e29665053..335f282b06f 100644
--- a/src/gallium/drivers/r600/r600.h
+++ b/src/gallium/drivers/r600/r600.h
@@ -35,7 +35,7 @@
 #define RADEON_CTX_MAX_PM4	(64 * 1024 / 4)
 
 #define R600_ERR(fmt, args...) \
-	fprintf(stderr, "EE %s/%s:%d - "fmt, __FILE__, __func__, __LINE__, ##args)
+	fprintf(stderr, "EE %s:%d %s - "fmt, __FILE__, __LINE__, __func__, ##args)
 
 typedef uint64_t		u64;
 typedef uint32_t		u32;
@@ -92,6 +92,9 @@ enum radeon_family {
 	CHIP_CYPRESS,
 	CHIP_HEMLOCK,
 	CHIP_PALM,
+	CHIP_BARTS,
+	CHIP_TURKS,
+	CHIP_CAICOS,
 	CHIP_LAST,
 };
 
diff --git a/src/gallium/drivers/r600/r600_asm.c b/src/gallium/drivers/r600/r600_asm.c
index bee1c941e5d..b15758adc33 100644
--- a/src/gallium/drivers/r600/r600_asm.c
+++ b/src/gallium/drivers/r600/r600_asm.c
@@ -161,6 +161,9 @@ int r600_bc_init(struct r600_bc *bc, enum radeon_family family)
 	case CHIP_CYPRESS:
 	case CHIP_HEMLOCK:
 	case CHIP_PALM:
+	case CHIP_BARTS:
+	case CHIP_TURKS:
+	case CHIP_CAICOS:
 		bc->chiprev = CHIPREV_EVERGREEN;
 		break;
 	default:
@@ -473,9 +476,20 @@ static int is_cfile(unsigned sel)
 	return (sel > 255 && sel < 512);
 }
 
+/* CB constants start at 512, and get translated to a kcache index when ALU
+ * clauses are constructed. Note that we handle kcache constants the same way
+ * as (the now gone) cfile constants, is that really required? */
+static int is_cb_const(int sel)
+{
+	if (sel > 511 && sel < 4607)
+		return 1;
+	return 0;
+}
+
 static int is_const(int sel)
 {
 	return is_cfile(sel) ||
+		is_cb_const(sel) ||
 		(sel >= V_SQ_ALU_SRC_0 &&
 		sel <= V_SQ_ALU_SRC_LITERAL);
 }
@@ -837,6 +851,120 @@ static int merge_inst_groups(struct r600_bc *bc, struct r600_bc_alu *slots[5], s
 	return 0;
 }
 
+/* This code handles kcache lines as single blocks of 32 constants. We could
+ * probably do slightly better by recognizing that we actually have two
+ * consecutive lines of 16 constants, but the resulting code would also be
+ * somewhat more complicated. */
+static int r600_bc_alloc_kcache_lines(struct r600_bc *bc, struct r600_bc_alu *alu, int type)
+{
+	struct r600_bc_kcache *kcache = bc->cf_last->kcache;
+	unsigned int required_lines;
+	unsigned int free_lines = 0;
+	unsigned int cache_line[3];
+	unsigned int count = 0;
+	unsigned int i, j;
+	int r;
+
+	/* Collect required cache lines. */
+	for (i = 0; i < 3; ++i) {
+		bool found = false;
+		unsigned int line;
+
+		if (alu->src[i].sel < 512)
+			continue;
+
+		line = ((alu->src[i].sel - 512) / 32) * 2;
+
+		for (j = 0; j < count; ++j) {
+			if (cache_line[j] == line) {
+				found = true;
+				break;
+			}
+		}
+
+		if (!found)
+			cache_line[count++] = line;
+	}
+
+	/* This should never actually happen. */
+	if (count >= 3) return -ENOMEM;
+
+	for (i = 0; i < 2; ++i) {
+		if (kcache[i].mode == V_SQ_CF_KCACHE_NOP) {
+			++free_lines;
+		}
+	}
+
+	/* Filter lines pulled in by previous intructions. Note that this is
+	 * only for the required_lines count, we can't remove these from the
+	 * cache_line array since we may have to start a new ALU clause. */
+	for (i = 0, required_lines = count; i < count; ++i) {
+		for (j = 0; j < 2; ++j) {
+			if (kcache[j].mode == V_SQ_CF_KCACHE_LOCK_2 &&
+			    kcache[j].addr == cache_line[i]) {
+				--required_lines;
+				break;
+			}
+		}
+	}
+
+	/* Start a new ALU clause if needed. */
+	if (required_lines > free_lines) {
+		if ((r = r600_bc_add_cf(bc))) {
+			return r;
+		}
+		bc->cf_last->inst = (type << 3);
+		kcache = bc->cf_last->kcache;
+	}
+
+	/* Setup the kcache lines. */
+	for (i = 0; i < count; ++i) {
+		bool found = false;
+
+		for (j = 0; j < 2; ++j) {
+			if (kcache[j].mode == V_SQ_CF_KCACHE_LOCK_2 &&
+			    kcache[j].addr == cache_line[i]) {
+				found = true;
+				break;
+			}
+		}
+
+		if (found) continue;
+
+		for (j = 0; j < 2; ++j) {
+			if (kcache[j].mode == V_SQ_CF_KCACHE_NOP) {
+				kcache[j].bank = 0;
+				kcache[j].addr = cache_line[i];
+				kcache[j].mode = V_SQ_CF_KCACHE_LOCK_2;
+				break;
+			}
+		}
+	}
+
+	/* Alter the src operands to refer to the kcache. */
+	for (i = 0; i < 3; ++i) {
+		static const unsigned int base[] = {128, 160, 256, 288};
+		unsigned int line;
+
+		if (alu->src[i].sel < 512)
+			continue;
+
+		alu->src[i].sel -= 512;
+		line = (alu->src[i].sel / 32) * 2;
+
+		for (j = 0; j < 2; ++j) {
+			if (kcache[j].mode == V_SQ_CF_KCACHE_LOCK_2 &&
+			    kcache[j].addr == line) {
+				alu->src[i].sel &= 0x1f;
+				alu->src[i].sel += base[j];
+				break;
+			}
+		}
+	}
+
+	return 0;
+}
+
 int r600_bc_add_alu_type(struct r600_bc *bc, const struct r600_bc_alu *alu, int type)
 {
 	struct r600_bc_alu *nalu = r600_bc_alu();
@@ -870,12 +998,20 @@ int r600_bc_add_alu_type(struct r600_bc *bc, const struct r600_bc_alu *alu, int
 		}
 	}
 	bc->cf_last->inst = (type << 3);
+
+	/* Setup the kcache for this ALU instruction. This will start a new
+	 * ALU clause if needed. */
+	if ((r = r600_bc_alloc_kcache_lines(bc, nalu, type))) {
+		free(nalu);
+		return r;
+	}
+
 	if (!bc->cf_last->curr_bs_head) {
 		bc->cf_last->curr_bs_head = nalu;
 	}
 	/* at most 128 slots, one add alu can add 5 slots + 4 constants(2 slots)
 	 * worst case */
-	if (alu->last && (bc->cf_last->ndw >> 1) >= 120) {
+	if (nalu->last && (bc->cf_last->ndw >> 1) >= 120) {
 		bc->force_add_cf = 1;
 	}
 	/* replace special constants */
@@ -890,10 +1026,8 @@ int r600_bc_add_alu_type(struct r600_bc *bc, const struct r600_bc_alu *alu, int
 	bc->cf_last->ndw += 2;
 	bc->ndw += 2;
 
-	bc->cf_last->kcache0_mode = 2;
-
 	/* process cur ALU instructions for bank swizzle */
-	if (alu->last) {
+	if (nalu->last) {
 		struct r600_bc_alu *slots[5];
 		r = assign_alu_units(bc->cf_last->curr_bs_head, slots);
 		if (r)
@@ -1182,14 +1316,14 @@ static int r600_bc_cf_build(struct r600_bc *bc, struct r600_bc_cf *cf)
 	case CF_CLASS_ALU:
 		assert(!end_of_program);
 		bc->bytecode[id++] = S_SQ_CF_ALU_WORD0_ADDR(cf->addr >> 1) |
-			S_SQ_CF_ALU_WORD0_KCACHE_MODE0(cf->kcache0_mode) |
-			S_SQ_CF_ALU_WORD0_KCACHE_BANK0(cf->kcache0_bank) |
-			S_SQ_CF_ALU_WORD0_KCACHE_BANK1(cf->kcache1_bank);
+			S_SQ_CF_ALU_WORD0_KCACHE_MODE0(cf->kcache[0].mode) |
+			S_SQ_CF_ALU_WORD0_KCACHE_BANK0(cf->kcache[0].bank) |
+			S_SQ_CF_ALU_WORD0_KCACHE_BANK1(cf->kcache[1].bank);
 
 		bc->bytecode[id++] = S_SQ_CF_ALU_WORD1_CF_INST(cf->inst >> 3) |
-			S_SQ_CF_ALU_WORD1_KCACHE_MODE1(cf->kcache1_mode) |
-			S_SQ_CF_ALU_WORD1_KCACHE_ADDR0(cf->kcache0_addr) |
-			S_SQ_CF_ALU_WORD1_KCACHE_ADDR1(cf->kcache1_addr) |
+			S_SQ_CF_ALU_WORD1_KCACHE_MODE1(cf->kcache[1].mode) |
+			S_SQ_CF_ALU_WORD1_KCACHE_ADDR0(cf->kcache[0].addr) |
+			S_SQ_CF_ALU_WORD1_KCACHE_ADDR1(cf->kcache[1].addr) |
 			S_SQ_CF_ALU_WORD1_BARRIER(cf->barrier) |
 			S_SQ_CF_ALU_WORD1_USES_WATERFALL(bc->chiprev == CHIPREV_R600 ? cf->r6xx_uses_waterfall : 0) |
 			S_SQ_CF_ALU_WORD1_COUNT((cf->ndw / 2) - 1);
diff --git a/src/gallium/drivers/r600/r600_asm.h b/src/gallium/drivers/r600/r600_asm.h
index 6059e45737f..519245f3af2 100644
--- a/src/gallium/drivers/r600/r600_asm.h
+++ b/src/gallium/drivers/r600/r600_asm.h
@@ -117,6 +117,12 @@ struct r600_bc_output {
 	unsigned			burst_count;
 };
 
+struct r600_bc_kcache {
+	unsigned			bank;
+	unsigned			mode;
+	unsigned			addr;
+};
+
 struct r600_bc_cf {
 	struct list_head		list;
 	unsigned			inst;
@@ -127,12 +133,7 @@ struct r600_bc_cf {
 	unsigned			pop_count;
 	unsigned			cf_addr; /* control flow addr */
 	unsigned			barrier;
-	unsigned			kcache0_mode;
-	unsigned			kcache1_mode;
-	unsigned			kcache0_addr;
-	unsigned			kcache1_addr;
-	unsigned			kcache0_bank;
-	unsigned			kcache1_bank;
+	struct r600_bc_kcache		kcache[2];
 	unsigned			r6xx_uses_waterfall;
 	struct list_head		alu;
 	struct list_head		tex;
diff --git a/src/gallium/drivers/r600/r600_blit.c b/src/gallium/drivers/r600/r600_blit.c
index 0f04136fb2a..b9ec9592e35 100644
--- a/src/gallium/drivers/r600/r600_blit.c
+++ b/src/gallium/drivers/r600/r600_blit.c
@@ -78,7 +78,7 @@ static void r600_blitter_end(struct pipe_context *ctx)
 	r600_context_queries_resume(&rctx->ctx);
 }
 
-int r600_blit_uncompress_depth(struct pipe_context *ctx, struct r600_resource_texture *texture)
+void r600_blit_uncompress_depth(struct pipe_context *ctx, struct r600_resource_texture *texture)
 {
 	struct r600_pipe_context *rctx = (struct r600_pipe_context *)ctx;
 	struct pipe_surface *zsurf, *cbsurf, surf_tmpl;
@@ -107,9 +107,6 @@ int r600_blit_uncompress_depth(struct pipe_context *ctx, struct r600_resource_te
 
 	pipe_surface_reference(&zsurf, NULL);
 	pipe_surface_reference(&cbsurf, NULL);
-
-
-	return 0;
 }
 
 static void r600_clear(struct pipe_context *ctx, unsigned buffers,
diff --git a/src/gallium/drivers/r600/r600_buffer.c b/src/gallium/drivers/r600/r600_buffer.c
index a17c54d6eeb..469c8195fe9 100644
--- a/src/gallium/drivers/r600/r600_buffer.c
+++ b/src/gallium/drivers/r600/r600_buffer.c
@@ -250,3 +250,29 @@ int r600_upload_user_buffers(struct r600_pipe_context *rctx)
 	}
 	return ret;
 }
+
+
+int r600_upload_const_buffer(struct r600_pipe_context *rctx, struct pipe_resource *cbuffer,
+			     uint32_t *const_offset)
+{
+	if (r600_buffer_is_user_buffer(cbuffer)) {
+		struct r600_resource_buffer *rbuffer = r600_buffer(cbuffer);
+		unsigned upload_offset;
+		int ret = 0;
+
+		ret = r600_upload_buffer(rctx->rupload_const,
+					 0, cbuffer->width0,
+					 rbuffer,
+					 &upload_offset,
+					 &rbuffer->r.bo_size,
+					 &rbuffer->r.bo);
+		if (ret)
+			return ret;
+		rbuffer->uploaded = TRUE;
+		*const_offset = upload_offset;
+		return 0;
+	}
+
+	*const_offset = 0;
+	return 0;
+}
diff --git a/src/gallium/drivers/r600/r600_pipe.c b/src/gallium/drivers/r600/r600_pipe.c
index 69cb5f7751f..ad14dbe14f4 100644
--- a/src/gallium/drivers/r600/r600_pipe.c
+++ b/src/gallium/drivers/r600/r600_pipe.c
@@ -70,6 +70,7 @@ static void r600_flush(struct pipe_context *ctx, unsigned flags,
 	r600_context_flush(&rctx->ctx);
 
 	r600_upload_flush(rctx->rupload_vb);
+	r600_upload_flush(rctx->rupload_const);
 }
 
 static void r600_destroy_context(struct pipe_context *context)
@@ -89,6 +90,7 @@ static void r600_destroy_context(struct pipe_context *context)
 	}
 
 	r600_upload_destroy(rctx->rupload_vb);
+	r600_upload_destroy(rctx->rupload_const);
 
 	if (rctx->tran.translate_cache)
 		translate_cache_destroy(rctx->tran.translate_cache);
@@ -149,6 +151,9 @@ static struct pipe_context *r600_create_context(struct pipe_screen *screen, void
 	case CHIP_CYPRESS:
 	case CHIP_HEMLOCK:
 	case CHIP_PALM:
+	case CHIP_BARTS:
+	case CHIP_TURKS:
+	case CHIP_CAICOS:
 		rctx->context.draw_vbo = evergreen_draw;
 		evergreen_init_state_functions(rctx);
 		if (evergreen_context_init(&rctx->ctx, rctx->radeon)) {
@@ -169,6 +174,12 @@ static struct pipe_context *r600_create_context(struct pipe_screen *screen, void
 		return NULL;
 	}
 
+	rctx->rupload_const = r600_upload_create(rctx, 128 * 1024, 256);
+	if (rctx->rupload_const == NULL) {
+		r600_destroy_context(&rctx->context);
+		return NULL;
+	}
+
 	rctx->blitter = util_blitter_create(&rctx->context);
 	if (rctx->blitter == NULL) {
 		FREE(rctx);
@@ -199,8 +210,6 @@ static struct pipe_context *r600_create_context(struct pipe_screen *screen, void
 	else
 		rctx->custom_dsa_flush = evergreen_create_db_flush_dsa(rctx);
 
-	r600_blit_uncompress_depth_ptr = r600_blit_uncompress_depth;
-
 	return &rctx->context;
 }
 
@@ -233,6 +242,9 @@ static const char *r600_get_family_name(enum radeon_family family)
 	case CHIP_CYPRESS: return "AMD CYPRESS";
 	case CHIP_HEMLOCK: return "AMD HEMLOCK";
 	case CHIP_PALM: return "AMD PALM";
+	case CHIP_BARTS: return "AMD BARTS";
+	case CHIP_TURKS: return "AMD TURKS";
+	case CHIP_CAICOS: return "AMD CAICOS";
 	default: return "AMD unknown";
 	}
 }
diff --git a/src/gallium/drivers/r600/r600_pipe.h b/src/gallium/drivers/r600/r600_pipe.h
index d9c35a44f18..2112a40f696 100644
--- a/src/gallium/drivers/r600/r600_pipe.h
+++ b/src/gallium/drivers/r600/r600_pipe.h
@@ -169,6 +169,7 @@ struct r600_pipe_context {
 	struct r600_textures_info	ps_samplers;
 	unsigned			vb_max_index;
 	struct r600_translate_context	tran;
+	struct r600_upload		*rupload_const;
 };
 
 struct r600_drawl {
@@ -196,7 +197,7 @@ void evergreen_vertex_buffer_update(struct r600_pipe_context *rctx);
 
 /* r600_blit.c */
 void r600_init_blit_functions(struct r600_pipe_context *rctx);
-int r600_blit_uncompress_depth(struct pipe_context *ctx, struct r600_resource_texture *texture);
+void r600_blit_uncompress_depth(struct pipe_context *ctx, struct r600_resource_texture *texture);
 
 /* r600_buffer.c */
 struct pipe_resource *r600_buffer_create(struct pipe_screen *screen,
diff --git a/src/gallium/drivers/r600/r600_resource.h b/src/gallium/drivers/r600/r600_resource.h
index 8ca27699206..28b3e1e5e40 100644
--- a/src/gallium/drivers/r600/r600_resource.h
+++ b/src/gallium/drivers/r600/r600_resource.h
@@ -106,7 +106,6 @@ static INLINE boolean r600_buffer_is_user_buffer(struct pipe_resource *buffer)
 }
 
 int r600_texture_depth_flush(struct pipe_context *ctx, struct pipe_resource *texture);
-int (*r600_blit_uncompress_depth_ptr)(struct pipe_context *ctx, struct r600_resource_texture *texture);
 
 /* r600_texture.c texture transfer functions. */
 struct pipe_transfer* r600_texture_get_transfer(struct pipe_context *ctx,
@@ -132,4 +131,5 @@ int r600_upload_buffer(struct r600_upload *upload, unsigned offset,
 			unsigned *out_offset, unsigned *out_size,
 			struct r600_bo **out_buffer);
 
+int r600_upload_const_buffer(struct r600_pipe_context *rctx, struct pipe_resource *cbuffer, uint32_t *offset);
 #endif
diff --git a/src/gallium/drivers/r600/r600_shader.c b/src/gallium/drivers/r600/r600_shader.c
index d78e249ae95..95367d7c536 100644
--- a/src/gallium/drivers/r600/r600_shader.c
+++ b/src/gallium/drivers/r600/r600_shader.c
@@ -507,7 +507,9 @@ int r600_shader_from_tgsi(const struct tgsi_token *tokens, struct r600_shader *s
 	/* Values [0,127] correspond to GPR[0..127].
 	 * Values [128,159] correspond to constant buffer bank 0
 	 * Values [160,191] correspond to constant buffer bank 1
-	 * Values [256,511] correspond to cfile constants c[0..255].
+	 * Values [256,511] correspond to cfile constants c[0..255]. (Gone on EG)
+	 * Values [256,287] correspond to constant buffer bank 2 (EG)
+	 * Values [288,319] correspond to constant buffer bank 3 (EG)
 	 * Other special values are shown in the list below.
 	 * 244  ALU_SRC_1_DBL_L: special constant 1.0 double-float, LSW. (RV670+)
 	 * 245  ALU_SRC_1_DBL_M: special constant 1.0 double-float, MSW. (RV670+)
@@ -541,7 +543,9 @@ int r600_shader_from_tgsi(const struct tgsi_token *tokens, struct r600_shader *s
 	ctx.file_offset[TGSI_FILE_TEMPORARY] = ctx.file_offset[TGSI_FILE_OUTPUT] +
 						ctx.info.file_count[TGSI_FILE_OUTPUT];
 
-	ctx.file_offset[TGSI_FILE_CONSTANT] = 128;
+	/* Outside the GPR range. This will be translated to one of the
+	 * kcache banks later. */
+	ctx.file_offset[TGSI_FILE_CONSTANT] = 512;
 
 	ctx.file_offset[TGSI_FILE_IMMEDIATE] = V_SQ_ALU_SRC_LITERAL;
 	ctx.temp_reg = ctx.file_offset[TGSI_FILE_TEMPORARY] +
@@ -587,6 +591,8 @@ int r600_shader_from_tgsi(const struct tgsi_token *tokens, struct r600_shader *s
 			if (r)
 				goto out_err;
 			break;
+		case TGSI_TOKEN_TYPE_PROPERTY:
+			break;
 		default:
 			R600_ERR("unsupported token type %d\n", ctx.parse.FullToken.Token.Type);
 			r = -EINVAL;
diff --git a/src/gallium/drivers/r600/r600_sq.h b/src/gallium/drivers/r600/r600_sq.h
index 2401d47e2a2..56ed35e8b32 100644
--- a/src/gallium/drivers/r600/r600_sq.h
+++ b/src/gallium/drivers/r600/r600_sq.h
@@ -74,6 +74,10 @@
 #define   S_SQ_CF_ALU_WORD0_KCACHE_MODE0(x)                          (((x) & 0x3) << 30)
 #define   G_SQ_CF_ALU_WORD0_KCACHE_MODE0(x)                          (((x) >> 30) & 0x3)
 #define   C_SQ_CF_ALU_WORD0_KCACHE_MODE0                             0x3FFFFFFF
+#define     V_SQ_CF_KCACHE_NOP                                       0x00000000
+#define     V_SQ_CF_KCACHE_LOCK_1                                    0x00000001
+#define     V_SQ_CF_KCACHE_LOCK_2                                    0x00000002
+#define     V_SQ_CF_KCACHE_LOCK_LOOP_INDEX                           0x00000003
 #define P_SQ_CF_ALU_WORD1
 #define   S_SQ_CF_ALU_WORD1_KCACHE_MODE1(x)                          (((x) & 0x3) << 0)
 #define   G_SQ_CF_ALU_WORD1_KCACHE_MODE1(x)                          (((x) >> 0) & 0x3)
diff --git a/src/gallium/drivers/r600/r600_state.c b/src/gallium/drivers/r600/r600_state.c
index 8b001e11f4a..9572ff9a1a2 100644
--- a/src/gallium/drivers/r600/r600_state.c
+++ b/src/gallium/drivers/r600/r600_state.c
@@ -1120,6 +1120,7 @@ static void r600_set_constant_buffer(struct pipe_context *ctx, uint shader, uint
 {
 	struct r600_pipe_context *rctx = (struct r600_pipe_context *)ctx;
 	struct r600_resource *rbuffer = (struct r600_resource*)buffer;
+	uint32_t offset;
 
 	/* Note that the state tracker can unbind constant buffers by
 	 * passing NULL here.
@@ -1128,6 +1129,8 @@ static void r600_set_constant_buffer(struct pipe_context *ctx, uint shader, uint
 		return;
 	}
 
+	r600_upload_const_buffer(rctx, buffer, &offset);
+
 	switch (shader) {
 	case PIPE_SHADER_VERTEX:
 		rctx->vs_const_buffer.nregs = 0;
@@ -1137,7 +1140,7 @@ static void r600_set_constant_buffer(struct pipe_context *ctx, uint shader, uint
 					0xFFFFFFFF, NULL);
 		r600_pipe_state_add_reg(&rctx->vs_const_buffer,
 					R_028980_ALU_CONST_CACHE_VS_0,
-					r600_bo_offset(rbuffer->bo) >> 8, 0xFFFFFFFF, rbuffer->bo);
+					(r600_bo_offset(rbuffer->bo) + offset) >> 8, 0xFFFFFFFF, rbuffer->bo);
 		r600_context_pipe_state_set(&rctx->ctx, &rctx->vs_const_buffer);
 		break;
 	case PIPE_SHADER_FRAGMENT:
@@ -1148,7 +1151,7 @@ static void r600_set_constant_buffer(struct pipe_context *ctx, uint shader, uint
 					0xFFFFFFFF, NULL);
 		r600_pipe_state_add_reg(&rctx->ps_const_buffer,
 					R_028940_ALU_CONST_CACHE_PS_0,
-					r600_bo_offset(rbuffer->bo) >> 8, 0xFFFFFFFF, rbuffer->bo);
+					(r600_bo_offset(rbuffer->bo) + offset) >> 8, 0xFFFFFFFF, rbuffer->bo);
 		r600_context_pipe_state_set(&rctx->ctx, &rctx->ps_const_buffer);
 		break;
 	default:
diff --git a/src/gallium/drivers/r600/r600_texture.c b/src/gallium/drivers/r600/r600_texture.c
index 23e239c0068..6add92e6d4c 100644
--- a/src/gallium/drivers/r600/r600_texture.c
+++ b/src/gallium/drivers/r600/r600_texture.c
@@ -443,8 +443,6 @@ static unsigned int r600_texture_is_referenced(struct pipe_context *context,
 	return PIPE_REFERENCED_FOR_READ | PIPE_REFERENCED_FOR_WRITE;
 }
 
-int (*r600_blit_uncompress_depth_ptr)(struct pipe_context *ctx, struct r600_resource_texture *texture);
-
 int r600_texture_depth_flush(struct pipe_context *ctx,
 			     struct pipe_resource *texture)
 {
@@ -476,7 +474,7 @@ int r600_texture_depth_flush(struct pipe_context *ctx,
 out:
 	/* XXX: only do this if the depth texture has actually changed:
 	 */
-	r600_blit_uncompress_depth_ptr(ctx, rtex);
+	r600_blit_uncompress_depth(ctx, rtex);
 	return 0;
 }
 
diff --git a/src/gallium/drivers/softpipe/sp_context.c b/src/gallium/drivers/softpipe/sp_context.c
index e935ce6d21b..f3489c1c793 100644
--- a/src/gallium/drivers/softpipe/sp_context.c
+++ b/src/gallium/drivers/softpipe/sp_context.c
@@ -129,6 +129,10 @@ softpipe_destroy( struct pipe_context *pipe )
       }
    }
 
+   for (i = 0; i < softpipe->num_vertex_buffers; i++) {
+      pipe_resource_reference(&softpipe->vertex_buffer[i].buffer, NULL);
+   }
+
    tgsi_exec_machine_destroy(softpipe->fs_machine);
 
    FREE( softpipe );
diff --git a/src/gallium/drivers/softpipe/sp_quad_blend.c b/src/gallium/drivers/softpipe/sp_quad_blend.c
index 6af1b2d0618..76cfc0bf51c 100644
--- a/src/gallium/drivers/softpipe/sp_quad_blend.c
+++ b/src/gallium/drivers/softpipe/sp_quad_blend.c
@@ -35,6 +35,7 @@
 #include "util/u_memory.h"
 #include "util/u_format.h"
 #include "sp_context.h"
+#include "sp_state.h"
 #include "sp_quad.h"
 #include "sp_tile_cache.h"
 #include "sp_quad_pipe.h"
@@ -794,6 +795,9 @@ blend_fallback(struct quad_stage *qs,
    struct softpipe_context *softpipe = qs->softpipe;
    const struct pipe_blend_state *blend = softpipe->blend;
    unsigned cbuf;
+   boolean write_all;
+
+   write_all = softpipe->fs->color0_writes_all_cbufs;
 
    for (cbuf = 0; cbuf < softpipe->framebuffer.nr_cbufs; cbuf++) 
    {
@@ -806,15 +810,19 @@ blend_fallback(struct quad_stage *qs,
                               quads[0]->input.y0);
       boolean has_dst_alpha
          = util_format_has_alpha(softpipe->framebuffer.cbufs[cbuf]->format);
-      uint q, i, j;
+      uint q, i, j, qbuf;
+
+      qbuf = write_all ? 0 : cbuf;
 
       for (q = 0; q < nr; q++) {
          struct quad_header *quad = quads[q];
-         float (*quadColor)[4] = quad->output.color[cbuf];
+         float (*quadColor)[4];
          const int itx = (quad->input.x0 & (TILE_SIZE-1));
          const int ity = (quad->input.y0 & (TILE_SIZE-1));
 
-         /* get/swizzle dest colors 
+         quadColor = quad->output.color[qbuf];
+
+         /* get/swizzle dest colors
           */
          for (j = 0; j < QUAD_SIZE; j++) {
             int x = itx + (j & 1);
diff --git a/src/gallium/drivers/softpipe/sp_state.h b/src/gallium/drivers/softpipe/sp_state.h
index 525bf23734a..bb19f8cff20 100644
--- a/src/gallium/drivers/softpipe/sp_state.h
+++ b/src/gallium/drivers/softpipe/sp_state.h
@@ -74,7 +74,7 @@ struct sp_fragment_shader {
 
    boolean origin_lower_left; /**< fragment shader uses lower left position origin? */
    boolean pixel_center_integer; /**< fragment shader uses integer pixel center? */
-
+   boolean color0_writes_all_cbufs; /**< fragment shader writes color0 to all bound cbufs */
    void (*prepare)( const struct sp_fragment_shader *shader,
 		    struct tgsi_exec_machine *machine,
 		    struct tgsi_sampler **samplers);
diff --git a/src/gallium/drivers/softpipe/sp_state_shader.c b/src/gallium/drivers/softpipe/sp_state_shader.c
index 7fff338ccea..66ddc565722 100644
--- a/src/gallium/drivers/softpipe/sp_state_shader.c
+++ b/src/gallium/drivers/softpipe/sp_state_shader.c
@@ -78,6 +78,8 @@ softpipe_create_fs_state(struct pipe_context *pipe,
          state->origin_lower_left = state->info.properties[i].data[0];
       else if (state->info.properties[i].name == TGSI_PROPERTY_FS_COORD_PIXEL_CENTER)
 	 state->pixel_center_integer = state->info.properties[i].data[0];
+      else if (state->info.properties[i].name == TGSI_PROPERTY_FS_COLOR0_WRITES_ALL_CBUFS)
+	 state->color0_writes_all_cbufs = state->info.properties[i].data[0];
    }
 
    return state;
diff --git a/src/gallium/drivers/softpipe/sp_state_vertex.c b/src/gallium/drivers/softpipe/sp_state_vertex.c
index 7d8055f2baf..5f4d661abde 100644
--- a/src/gallium/drivers/softpipe/sp_state_vertex.c
+++ b/src/gallium/drivers/softpipe/sp_state_vertex.c
@@ -33,6 +33,7 @@
 #include "sp_state.h"
 
 #include "util/u_memory.h"
+#include "util/u_inlines.h"
 #include "draw/draw_context.h"
 
 
@@ -84,8 +85,9 @@ softpipe_set_vertex_buffers(struct pipe_context *pipe,
 
    assert(count <= PIPE_MAX_ATTRIBS);
 
-   memcpy(softpipe->vertex_buffer, buffers, count * sizeof(buffers[0]));
-   softpipe->num_vertex_buffers = count;
+   util_copy_vertex_buffers(softpipe->vertex_buffer,
+                            &softpipe->num_vertex_buffers,
+                            buffers, count);
 
    softpipe->dirty |= SP_NEW_VERTEX;
 
diff --git a/src/gallium/drivers/svga/svga_context.h b/src/gallium/drivers/svga/svga_context.h
index 04e281a506d..d4970908b1e 100644
--- a/src/gallium/drivers/svga/svga_context.h
+++ b/src/gallium/drivers/svga/svga_context.h
@@ -288,6 +288,11 @@ struct svga_sw_state
    boolean need_swvfetch;
    boolean need_pipeline;
    boolean need_swtnl;
+
+   /* Flag to make sure that need sw is on while
+    * updating state within a swtnl call.
+    */
+   boolean in_swtnl_draw;
 };
 
 
diff --git a/src/gallium/drivers/svga/svga_draw.c b/src/gallium/drivers/svga/svga_draw.c
index 81dd4778d0a..97cbac447d6 100644
--- a/src/gallium/drivers/svga/svga_draw.c
+++ b/src/gallium/drivers/svga/svga_draw.c
@@ -315,7 +315,6 @@ enum pipe_error svga_hwtnl_prim( struct svga_hwtnl *hwtnl,
             break;
          }
 
-         assert(!stride || width <= stride);
          if (max_index != ~0) {
             assert(offset + (index_bias + max_index) * stride + width <= size);
          }
diff --git a/src/gallium/drivers/svga/svga_draw_elements.c b/src/gallium/drivers/svga/svga_draw_elements.c
index c7ea014bba5..83527c6ef49 100644
--- a/src/gallium/drivers/svga/svga_draw_elements.c
+++ b/src/gallium/drivers/svga/svga_draw_elements.c
@@ -120,14 +120,17 @@ svga_hwtnl_simple_draw_range_elements( struct svga_hwtnl *hwtnl,
    if (index_buffer && 
        svga_buffer_is_user_buffer(index_buffer)) 
    {
+      boolean flushed;
       assert( index_buffer->width0 >= index_offset + count * index_size );
 
       ret = u_upload_buffer( hwtnl->upload_ib,
+                             0,
                              index_offset,
                              count * index_size,
                              index_buffer,
                              &index_offset,
-                             &upload_buffer );
+                             &upload_buffer,
+                             &flushed );
       if (ret)
          goto done;
 
diff --git a/src/gallium/drivers/svga/svga_state_need_swtnl.c b/src/gallium/drivers/svga/svga_state_need_swtnl.c
index 66fea02a4be..8ba5ac8cdb4 100644
--- a/src/gallium/drivers/svga/svga_state_need_swtnl.c
+++ b/src/gallium/drivers/svga/svga_state_need_swtnl.c
@@ -175,9 +175,17 @@ static int update_need_swtnl( struct svga_context *svga,
       need_swtnl = 1;
    }
 
+   /*
+    * Some state changes the draw module does makes us belive we
+    * we don't need swtnl. This causes the vdecl code to pickup
+    * the wrong buffers and vertex formats. Try trivial/line-wide.
+    */
+   if (svga->state.sw.in_swtnl_draw)
+      need_swtnl = 1;
+
    if (need_swtnl != svga->state.sw.need_swtnl) {
       SVGA_DBG(DEBUG_SWTNL|DEBUG_PERF,
-               "%s need_swvfetch: %s, need_pipeline %s\n",
+               "%s: need_swvfetch %s, need_pipeline %s\n",
                __FUNCTION__,
                svga->state.sw.need_swvfetch ? "true" : "false",
                svga->state.sw.need_pipeline ? "true" : "false");
diff --git a/src/gallium/drivers/svga/svga_state_vdecl.c b/src/gallium/drivers/svga/svga_state_vdecl.c
index 3af7bf2b358..958d00393f2 100644
--- a/src/gallium/drivers/svga/svga_state_vdecl.c
+++ b/src/gallium/drivers/svga/svga_state_vdecl.c
@@ -57,12 +57,14 @@ upload_user_buffers( struct svga_context *svga )
          struct svga_buffer *buffer = svga_buffer(svga->curr.vb[i].buffer);
 
          if (!buffer->uploaded.buffer) {
+            boolean flushed;
             ret = u_upload_buffer( svga->upload_vb,
-                                   0,
+                                   0, 0,
                                    buffer->b.b.width0,
                                    &buffer->b.b,
                                    &buffer->uploaded.offset,
-                                   &buffer->uploaded.buffer );
+                                   &buffer->uploaded.buffer,
+                                   &flushed);
             if (ret)
                return ret;
 
diff --git a/src/gallium/drivers/svga/svga_swtnl_backend.c b/src/gallium/drivers/svga/svga_swtnl_backend.c
index 24646b48f62..d5db6bf641a 100644
--- a/src/gallium/drivers/svga/svga_swtnl_backend.c
+++ b/src/gallium/drivers/svga/svga_swtnl_backend.c
@@ -158,7 +158,7 @@ svga_vbuf_render_set_primitive( struct vbuf_render *render,
 }
 
 static void
-svga_vbuf_sumbit_state( struct svga_vbuf_render *svga_render )
+svga_vbuf_submit_state( struct svga_vbuf_render *svga_render )
 {
    struct svga_context *svga = svga_render->svga;
    SVGA3dVertexDecl vdecl[PIPE_MAX_ATTRIBS];
@@ -221,7 +221,8 @@ svga_vbuf_render_draw_arrays( struct vbuf_render *render,
    unsigned bias = (svga_render->vbuf_offset - svga_render->vdecl_offset) / svga_render->vertex_size;
    enum pipe_error ret = 0;
 
-   svga_vbuf_sumbit_state(svga_render);
+   /* off to hardware */
+   svga_vbuf_submit_state(svga_render);
 
    /* Need to call update_state() again as the draw module may have
     * altered some of our state behind our backs.  Testcase:
@@ -267,9 +268,8 @@ svga_vbuf_render_draw_elements( struct vbuf_render *render,
    pipe_buffer_write_nooverlap(&svga->pipe, svga_render->ibuf,
 			       svga_render->ibuf_offset, 2 * nr_indices, indices);
 
-
    /* off to hardware */
-   svga_vbuf_sumbit_state(svga_render);
+   svga_vbuf_submit_state(svga_render);
 
    /* Need to call update_state() again as the draw module may have
     * altered some of our state behind our backs.  Testcase:
diff --git a/src/gallium/drivers/svga/svga_swtnl_draw.c b/src/gallium/drivers/svga/svga_swtnl_draw.c
index d9039845819..05d86e1fb16 100644
--- a/src/gallium/drivers/svga/svga_swtnl_draw.c
+++ b/src/gallium/drivers/svga/svga_swtnl_draw.c
@@ -51,6 +51,9 @@ svga_swtnl_draw_vbo(struct svga_context *svga,
    assert(svga->state.sw.need_swtnl);
    assert(draw);
 
+   /* Make sure that the need_swtnl flag does not go away */
+   svga->state.sw.in_swtnl_draw = TRUE;
+
    ret = svga_update_state(svga, SVGA_STATE_SWTNL_DRAW);
    if (ret) {
       svga_context_flush(svga, NULL);
@@ -119,6 +122,9 @@ svga_swtnl_draw_vbo(struct svga_context *svga,
       pipe_buffer_unmap(&svga->pipe, cb_transfer);
    }
 
+   /* Now safe to remove the need_swtnl flag in any update_state call */
+   svga->state.sw.in_swtnl_draw = FALSE;
+
    return ret;
 }
 
diff --git a/src/gallium/include/pipe/p_format.h b/src/gallium/include/pipe/p_format.h
index 119d304d927..3aa11be4b5b 100644
--- a/src/gallium/include/pipe/p_format.h
+++ b/src/gallium/include/pipe/p_format.h
@@ -202,6 +202,12 @@ enum pipe_format {
    PIPE_FORMAT_X24S8_USCALED           = 143,
    PIPE_FORMAT_S8X24_USCALED           = 144,
    PIPE_FORMAT_X32_S8X24_USCALED       = 145,
+
+   PIPE_FORMAT_B2G3R3_UNORM            = 146,
+   PIPE_FORMAT_L16A16_UNORM            = 147,
+   PIPE_FORMAT_A16_UNORM               = 148,
+   PIPE_FORMAT_I16_UNORM               = 149,
+
    PIPE_FORMAT_COUNT
 };
 
diff --git a/src/gallium/include/pipe/p_shader_tokens.h b/src/gallium/include/pipe/p_shader_tokens.h
index ba433b2bd2a..0a9e14154d7 100644
--- a/src/gallium/include/pipe/p_shader_tokens.h
+++ b/src/gallium/include/pipe/p_shader_tokens.h
@@ -177,7 +177,8 @@ union tgsi_immediate_data
 #define TGSI_PROPERTY_GS_MAX_OUTPUT_VERTICES 2
 #define TGSI_PROPERTY_FS_COORD_ORIGIN        3
 #define TGSI_PROPERTY_FS_COORD_PIXEL_CENTER  4
-#define TGSI_PROPERTY_COUNT                  5
+#define TGSI_PROPERTY_FS_COLOR0_WRITES_ALL_CBUFS 5
+#define TGSI_PROPERTY_COUNT                  6
 
 struct tgsi_property {
    unsigned Type         : 4;  /**< TGSI_TOKEN_TYPE_PROPERTY */
diff --git a/src/gallium/state_trackers/egl/common/egl_g3d_api.c b/src/gallium/state_trackers/egl/common/egl_g3d_api.c
index bce901ac9a9..c9f94a3c498 100644
--- a/src/gallium/state_trackers/egl/common/egl_g3d_api.c
+++ b/src/gallium/state_trackers/egl/common/egl_g3d_api.c
@@ -158,17 +158,17 @@ egl_g3d_choose_config(_EGLDriver *drv, _EGLDisplay *dpy, const EGLint *attribs,
          (_EGLArrayForEach) egl_g3d_match_config, (void *) &criteria);
 
    /* perform sorting of configs */
-   if (tmp_configs && tmp_size) {
+   if (configs && tmp_size) {
       _eglSortConfigs((const _EGLConfig **) tmp_configs, tmp_size,
             egl_g3d_compare_config, (void *) &criteria);
-      size = MIN2(tmp_size, size);
-      for (i = 0; i < size; i++)
+      tmp_size = MIN2(tmp_size, size);
+      for (i = 0; i < tmp_size; i++)
          configs[i] = _eglGetConfigHandle(tmp_configs[i]);
    }
 
    FREE(tmp_configs);
 
-   *num_configs = size;
+   *num_configs = tmp_size;
 
    return EGL_TRUE;
 }
diff --git a/src/gallium/state_trackers/egl/common/egl_g3d_image.c b/src/gallium/state_trackers/egl/common/egl_g3d_image.c
index e6ff100de04..78c035a2af0 100644
--- a/src/gallium/state_trackers/egl/common/egl_g3d_image.c
+++ b/src/gallium/state_trackers/egl/common/egl_g3d_image.c
@@ -35,7 +35,6 @@
 
 #include "native.h"
 #include "egl_g3d.h"
-#include "egl_g3d_api.h"
 #include "egl_g3d_image.h"
 
 /* for struct winsys_handle */
diff --git a/src/gallium/state_trackers/xorg/xorg_crtc.c b/src/gallium/state_trackers/xorg/xorg_crtc.c
index 28e30e09ff3..71f7b8c21d0 100644
--- a/src/gallium/state_trackers/xorg/xorg_crtc.c
+++ b/src/gallium/state_trackers/xorg/xorg_crtc.c
@@ -133,6 +133,16 @@ crtc_set_mode_major(xf86CrtcPtr crtc, DisplayModePtr mode,
     if (ret)
 	return FALSE;
 
+    /* Only set gamma when needed, to avoid unneeded delays. */
+#if defined(XF86_CRTC_VERSION) && XF86_CRTC_VERSION >= 3
+    if (!crtc->active)
+#endif
+	crtc->funcs->gamma_set(crtc, crtc->gamma_red, crtc->gamma_green,
+			       crtc->gamma_blue, crtc->gamma_size);
+
+#if defined(XF86_CRTC_VERSION) && XF86_CRTC_VERSION >= 3
+    crtc->active = TRUE;
+#endif
     crtc->x = x;
     crtc->y = y;
     crtc->mode = *mode;
@@ -145,7 +155,10 @@ static void
 crtc_gamma_set(xf86CrtcPtr crtc, CARD16 * red, CARD16 * green, CARD16 * blue,
 	       int size)
 {
-    /* XXX: hockup */
+    modesettingPtr ms = modesettingPTR(crtc->scrn);
+    struct crtc_private *crtcp = crtc->driver_private;
+
+    drmModeCrtcSetGamma(ms->fd, crtcp->drm_crtc->crtc_id, size, red, green, blue);
 }
 
 #if 0 /* Implement and enable to enable rotation and reflection. */
diff --git a/src/gallium/state_trackers/xorg/xorg_driver.c b/src/gallium/state_trackers/xorg/xorg_driver.c
index 66685ecec64..33bcacdcc2e 100644
--- a/src/gallium/state_trackers/xorg/xorg_driver.c
+++ b/src/gallium/state_trackers/xorg/xorg_driver.c
@@ -44,6 +44,7 @@
 #include "xf86Crtc.h"
 #include "miscstruct.h"
 #include "dixstruct.h"
+#include "xf86cmap.h"
 #include "xf86xv.h"
 #include "xorgVersion.h"
 #ifndef XSERVER_LIBPCIACCESS
@@ -414,6 +415,7 @@ drv_pre_init(ScrnInfoPtr pScrn, int flags)
 	return FALSE;
 
     switch (pScrn->depth) {
+    case 8:
     case 15:
     case 16:
     case 24:
@@ -677,6 +679,65 @@ drv_set_master(ScrnInfoPtr pScrn)
 }
 
 
+static void drv_load_palette(ScrnInfoPtr pScrn, int numColors,
+			     int *indices, LOCO *colors, VisualPtr pVisual)
+{
+    xf86CrtcConfigPtr xf86_config = XF86_CRTC_CONFIG_PTR(pScrn);
+    modesettingPtr ms = modesettingPTR(pScrn);
+    int index, j, i;
+    int c;
+
+    switch(pScrn->depth) {
+    case 15:
+	for (i = 0; i < numColors; i++) {
+	    index = indices[i];
+	    for (j = 0; j < 8; j++) {
+		ms->lut_r[index * 8 + j] = colors[index].red << 8;
+		ms->lut_g[index * 8 + j] = colors[index].green << 8;
+		ms->lut_b[index * 8 + j] = colors[index].blue << 8;
+	    }
+	}
+	break;
+    case 16:
+	for (i = 0; i < numColors; i++) {
+	    index = indices[i];
+
+	    if (index < 32) {
+		for (j = 0; j < 8; j++) {
+		    ms->lut_r[index * 8 + j] = colors[index].red << 8;
+		    ms->lut_b[index * 8 + j] = colors[index].blue << 8;
+		}
+	    }
+
+	    for (j = 0; j < 4; j++) {
+		ms->lut_g[index * 4 + j] = colors[index].green << 8;
+	    }
+	}
+	break;
+    default:
+	for (i = 0; i < numColors; i++) {
+	    index = indices[i];
+	    ms->lut_r[index] = colors[index].red << 8;
+	    ms->lut_g[index] = colors[index].green << 8;
+	    ms->lut_b[index] = colors[index].blue << 8;
+	}
+	break;
+    }
+
+    for (c = 0; c < xf86_config->num_crtc; c++) {
+	xf86CrtcPtr crtc = xf86_config->crtc[c];
+
+	/* Make the change through RandR */
+#ifdef RANDR_12_INTERFACE
+	if (crtc->randr_crtc)
+	    RRCrtcGammaSet(crtc->randr_crtc, ms->lut_r, ms->lut_g, ms->lut_b);
+	else
+#endif
+	    crtc->funcs->gamma_set(crtc, ms->lut_r, ms->lut_g, ms->lut_b, 256);
+    }
+}
+
+
 static Bool
 drv_screen_init(int scrnIndex, ScreenPtr pScreen, int argc, char **argv)
 {
@@ -816,6 +877,10 @@ drv_screen_init(int scrnIndex, ScreenPtr pScreen, int argc, char **argv)
 
     if (!miCreateDefColormap(pScreen))
 	return FALSE;
+    if (!xf86HandleColormaps(pScreen, 256, 8, drv_load_palette, NULL,
+			     CMAP_PALETTED_TRUECOLOR |
+			     CMAP_RELOAD_ON_MODE_SWITCH))
+	return FALSE;
 
     xf86DPMSInit(pScreen, xf86DPMSSet, 0);
 
diff --git a/src/gallium/state_trackers/xorg/xorg_tracker.h b/src/gallium/state_trackers/xorg/xorg_tracker.h
index 56397b8fea8..664e8c75730 100644
--- a/src/gallium/state_trackers/xorg/xorg_tracker.h
+++ b/src/gallium/state_trackers/xorg/xorg_tracker.h
@@ -129,6 +129,7 @@ typedef struct _modesettingRec
     /* kms */
     struct kms_driver *kms;
     struct kms_bo *root_bo;
+    uint16_t lut_r[256], lut_g[256], lut_b[256];
 
     /* gallium */
     struct pipe_screen *screen;
diff --git a/src/gallium/targets/dri-nouveau/Makefile b/src/gallium/targets/dri-nouveau/Makefile
index 9dfe86c8949..26c927e0a81 100644
--- a/src/gallium/targets/dri-nouveau/Makefile
+++ b/src/gallium/targets/dri-nouveau/Makefile
@@ -10,6 +10,7 @@ PIPE_DRIVERS = \
 	$(TOP)/src/gallium/drivers/rbug/librbug.a \
 	$(TOP)/src/gallium/drivers/nvfx/libnvfx.a \
 	$(TOP)/src/gallium/drivers/nv50/libnv50.a \
+	$(TOP)/src/gallium/drivers/nvc0/libnvc0.a \
 	$(TOP)/src/gallium/drivers/softpipe/libsoftpipe.a \
 	$(TOP)/src/gallium/drivers/nouveau/libnouveau.a
 
diff --git a/src/gallium/targets/dri-vmwgfx/SConscript b/src/gallium/targets/dri-vmwgfx/SConscript
index 7888e4f2c8b..17dd0210512 100644
--- a/src/gallium/targets/dri-vmwgfx/SConscript
+++ b/src/gallium/targets/dri-vmwgfx/SConscript
@@ -2,14 +2,14 @@ Import('*')
 
 env = drienv.Clone()
 
-env.Append(CPPDEFINES = ['GALLIUM_RBUG', 'GALLIUM_TRACE'])
+if True:
+    env.Append(CPPDEFINES = ['GALLIUM_TRACE', 'GALLIUM_RBUG', 'GALLIUM_GALAHAD', 'GALLIUM_SOFTPIPE'])
+    env.Prepend(LIBS = [trace, rbug, galahad, softpipe, ws_wrapper])
 
 env.Prepend(LIBS = [
     st_dri,
     svgadrm,
     svga,
-    trace,
-    rbug,
     mesa,
     glsl,
     gallium,
@@ -22,4 +22,4 @@ module = env.LoadableModule(
     SHLIBPREFIX = '',
 )
 
-env.Alias('dri-vmwgfx', module)
\ No newline at end of file
+env.Alias('dri-vmwgfx', module)
diff --git a/src/gallium/targets/dri-vmwgfx/target.c b/src/gallium/targets/dri-vmwgfx/target.c
index 15089d6db26..1362851d6be 100644
--- a/src/gallium/targets/dri-vmwgfx/target.c
+++ b/src/gallium/targets/dri-vmwgfx/target.c
@@ -1,4 +1,5 @@
 
+#include "target-helpers/inline_wrapper_sw_helper.h"
 #include "target-helpers/inline_debug_helper.h"
 #include "state_tracker/drm_driver.h"
 #include "svga/drm/svga_drm_public.h"
@@ -18,6 +19,8 @@ create_screen(int fd)
    if (!screen)
       return NULL;
 
+   screen = sw_screen_wrap(screen);
+
    screen = debug_screen_wrap(screen);
 
    return screen;
diff --git a/src/gallium/targets/graw-null/graw_util.c b/src/gallium/targets/graw-null/graw_util.c
index e5cf526d33a..09cba895d2a 100644
--- a/src/gallium/targets/graw-null/graw_util.c
+++ b/src/gallium/targets/graw-null/graw_util.c
@@ -1,6 +1,7 @@
 
 #include "pipe/p_compiler.h"
 #include "pipe/p_context.h"
+#include "pipe/p_shader_tokens.h"
 #include "pipe/p_state.h"
 #include "tgsi/tgsi_text.h"
 #include "util/u_debug.h"
diff --git a/src/gallium/targets/libgl-xlib/xlib.c b/src/gallium/targets/libgl-xlib/xlib.c
index 9a3e0e07b0d..1a5892b94a0 100644
--- a/src/gallium/targets/libgl-xlib/xlib.c
+++ b/src/gallium/targets/libgl-xlib/xlib.c
@@ -35,7 +35,6 @@
 #include "state_tracker/xlib_sw_winsys.h"
 #include "xm_public.h"
 
-#include "state_tracker/st_api.h"
 #include "state_tracker/st_gl_api.h"
 #include "target-helpers/inline_sw_helper.h"
 #include "target-helpers/inline_debug_helper.h"
diff --git a/src/gallium/winsys/nouveau/drm/nouveau_drm_winsys.c b/src/gallium/winsys/nouveau/drm/nouveau_drm_winsys.c
index d4bf124ce6f..648d6c8a8e2 100644
--- a/src/gallium/winsys/nouveau/drm/nouveau_drm_winsys.c
+++ b/src/gallium/winsys/nouveau/drm/nouveau_drm_winsys.c
@@ -50,6 +50,9 @@ nouveau_drm_screen_create(int fd)
 	case 0xa0:
 		init = nv50_screen_create;
 		break;
+	case 0xc0:
+		init = nvc0_screen_create;
+		break;
 	default:
 		debug_printf("%s: unknown chipset nv%02x\n", __func__,
 			     dev->chipset);
diff --git a/src/gallium/winsys/r600/drm/r600_drm.c b/src/gallium/winsys/r600/drm/r600_drm.c
index 58aacb77c9e..36ec583ccee 100644
--- a/src/gallium/winsys/r600/drm/r600_drm.c
+++ b/src/gallium/winsys/r600/drm/r600_drm.c
@@ -177,6 +177,9 @@ static struct radeon *radeon_new(int fd, unsigned device)
 	case CHIP_CYPRESS:
 	case CHIP_HEMLOCK:
 	case CHIP_PALM:
+	case CHIP_BARTS:
+	case CHIP_TURKS:
+	case CHIP_CAICOS:
 		radeon->chip_class = EVERGREEN;
 		/* set default group bytes, overridden by tiling info ioctl */
 		radeon->tiling_info.group_bytes = 512;
diff --git a/src/gallium/winsys/r600/drm/radeon_pciid.c b/src/gallium/winsys/r600/drm/radeon_pciid.c
index 92560a488ae..e2622abd468 100644
--- a/src/gallium/winsys/r600/drm/radeon_pciid.c
+++ b/src/gallium/winsys/r600/drm/radeon_pciid.c
@@ -445,6 +445,42 @@ struct pci_id radeon_pci_id[] = {
 	{0x1002, 0x9803, CHIP_PALM},
 	{0x1002, 0x9804, CHIP_PALM},
 	{0x1002, 0x9805, CHIP_PALM},
+	{0x1002, 0x6720, CHIP_BARTS},
+	{0x1002, 0x6721, CHIP_BARTS},
+	{0x1002, 0x6722, CHIP_BARTS},
+	{0x1002, 0x6723, CHIP_BARTS},
+	{0x1002, 0x6724, CHIP_BARTS},
+	{0x1002, 0x6725, CHIP_BARTS},
+	{0x1002, 0x6726, CHIP_BARTS},
+	{0x1002, 0x6727, CHIP_BARTS},
+	{0x1002, 0x6728, CHIP_BARTS},
+	{0x1002, 0x6729, CHIP_BARTS},
+	{0x1002, 0x6738, CHIP_BARTS},
+	{0x1002, 0x6739, CHIP_BARTS},
+	{0x1002, 0x6740, CHIP_TURKS},
+	{0x1002, 0x6741, CHIP_TURKS},
+	{0x1002, 0x6742, CHIP_TURKS},
+	{0x1002, 0x6743, CHIP_TURKS},
+	{0x1002, 0x6744, CHIP_TURKS},
+	{0x1002, 0x6745, CHIP_TURKS},
+	{0x1002, 0x6746, CHIP_TURKS},
+	{0x1002, 0x6747, CHIP_TURKS},
+	{0x1002, 0x6748, CHIP_TURKS},
+	{0x1002, 0x6749, CHIP_TURKS},
+	{0x1002, 0x6750, CHIP_TURKS},
+	{0x1002, 0x6758, CHIP_TURKS},
+	{0x1002, 0x6759, CHIP_TURKS},
+	{0x1002, 0x6760, CHIP_CAICOS},
+	{0x1002, 0x6761, CHIP_CAICOS},
+	{0x1002, 0x6762, CHIP_CAICOS},
+	{0x1002, 0x6763, CHIP_CAICOS},
+	{0x1002, 0x6764, CHIP_CAICOS},
+	{0x1002, 0x6765, CHIP_CAICOS},
+	{0x1002, 0x6766, CHIP_CAICOS},
+	{0x1002, 0x6767, CHIP_CAICOS},
+	{0x1002, 0x6768, CHIP_CAICOS},
+	{0x1002, 0x6770, CHIP_CAICOS},
+	{0x1002, 0x6779, CHIP_CAICOS},
 	{0, 0},
 };
 
diff --git a/src/gallium/winsys/radeon/drm/Makefile b/src/gallium/winsys/radeon/drm/Makefile
index aa73edde34e..7e339a2ecfe 100644
--- a/src/gallium/winsys/radeon/drm/Makefile
+++ b/src/gallium/winsys/radeon/drm/Makefile
@@ -6,6 +6,7 @@ LIBNAME = radeonwinsys
 
 C_SOURCES = \
 	radeon_drm_buffer.c \
+	radeon_drm_cs.c \
 	radeon_drm_common.c \
 	radeon_r300.c
 
diff --git a/src/gallium/winsys/radeon/drm/SConscript b/src/gallium/winsys/radeon/drm/SConscript
index 2dbf61a7ba3..80816621848 100644
--- a/src/gallium/winsys/radeon/drm/SConscript
+++ b/src/gallium/winsys/radeon/drm/SConscript
@@ -4,6 +4,7 @@ env = env.Clone()
 
 radeon_sources = [
     'radeon_drm_buffer.c',
+    'radeon_drm_cs.c',
     'radeon_drm_common.c',
     'radeon_r300.c',
 ]
diff --git a/src/gallium/winsys/radeon/drm/radeon_drm_buffer.c b/src/gallium/winsys/radeon/drm/radeon_drm_buffer.c
index dc4b51747f4..5e14287ec2d 100644
--- a/src/gallium/winsys/radeon/drm/radeon_drm_buffer.c
+++ b/src/gallium/winsys/radeon/drm/radeon_drm_buffer.c
@@ -1,5 +1,5 @@
-#include "radeon_cs_gem.h"
 #include "radeon_drm_buffer.h"
+#include "radeon_drm_cs.h"
 
 #include "util/u_hash_table.h"
 #include "util/u_memory.h"
@@ -155,7 +155,7 @@ radeon_drm_buffer_map_internal(struct pb_buffer *_buf,
      * we cannot flush. */
     assert(cs || !radeon_bo_is_referenced_by_cs(buf->bo, NULL));
 
-    if (cs && radeon_bo_is_referenced_by_cs(buf->bo, cs->cs)) {
+    if (cs && radeon_bo_is_referenced_by_cs(buf->bo, NULL)) {
         cs->flush_cs(cs->flush_data);
     }
 
@@ -463,35 +463,6 @@ static void radeon_drm_buffer_set_tiling(struct r300_winsys_screen *ws,
     radeon_bo_set_tiling(buf->bo, flags, pitch);
 }
 
-static void radeon_drm_bufmgr_add_buffer(struct r300_winsys_cs *rcs,
-                                         struct r300_winsys_cs_buffer *_buf,
-                                         enum r300_buffer_domain rd,
-                                         enum r300_buffer_domain wd)
-{
-    struct radeon_drm_cs *cs = radeon_drm_cs(rcs);
-    struct radeon_bo *bo = (struct radeon_bo*)_buf;
-
-    radeon_cs_space_add_persistent_bo(cs->cs, bo, rd, wd);
-}
-
-static void radeon_drm_bufmgr_write_reloc(struct r300_winsys_cs *rcs,
-                                          struct r300_winsys_cs_buffer *_buf,
-                                          enum r300_buffer_domain rd,
-                                          enum r300_buffer_domain wd)
-{
-    struct radeon_drm_cs *cs = radeon_drm_cs(rcs);
-    struct radeon_bo *bo = (struct radeon_bo*)_buf;
-    int retval;
-
-    cs->cs->cdw = cs->base.cdw;
-    retval = radeon_cs_write_reloc(cs->cs, bo, rd, wd, 0);
-    cs->base.cdw = cs->cs->cdw;
-    if (retval) {
-        fprintf(stderr, "radeon: Relocation of %p (%d, %d, %d) failed!\n",
-                bo, rd, wd, 0);
-    }
-}
-
 static struct r300_winsys_cs_buffer *radeon_drm_get_cs_handle(
         struct r300_winsys_screen *rws,
         struct r300_winsys_buffer *_buf)
@@ -505,12 +476,11 @@ static boolean radeon_drm_is_buffer_referenced(struct r300_winsys_cs *rcs,
                                                struct r300_winsys_cs_buffer *_buf,
                                                enum r300_reference_domain domain)
 {
-    struct radeon_drm_cs *cs = radeon_drm_cs(rcs);
     struct radeon_bo *bo = (struct radeon_bo*)_buf;
     uint32_t tmp;
 
     if (domain & R300_REF_CS) {
-        if (radeon_bo_is_referenced_by_cs(bo, cs->cs)) {
+        if (radeon_bo_is_referenced_by_cs(bo, NULL)) {
             return TRUE;
         }
     }
@@ -559,6 +529,4 @@ void radeon_drm_bufmgr_init_functions(struct radeon_drm_winsys *ws)
     ws->base.buffer_unmap = radeon_drm_buffer_unmap;
     ws->base.buffer_wait = radeon_drm_buffer_wait;
     ws->base.cs_is_buffer_referenced = radeon_drm_is_buffer_referenced;
-    ws->base.cs_add_buffer = radeon_drm_bufmgr_add_buffer;
-    ws->base.cs_write_reloc = radeon_drm_bufmgr_write_reloc;
 }
diff --git a/src/gallium/winsys/radeon/drm/radeon_drm_common.c b/src/gallium/winsys/radeon/drm/radeon_drm_common.c
index f38ab6e3b43..fe71f080592 100644
--- a/src/gallium/winsys/radeon/drm/radeon_drm_common.c
+++ b/src/gallium/winsys/radeon/drm/radeon_drm_common.c
@@ -31,13 +31,12 @@
 
 #include "radeon_winsys.h"
 #include "radeon_drm_buffer.h"
+#include "radeon_drm_cs.h"
 #include "radeon_drm_public.h"
 
 #include "pipebuffer/pb_bufmgr.h"
 #include "util/u_memory.h"
 
-#include "state_tracker/drm_driver.h"
-
 #include <radeon_drm.h>
 #include <radeon_bo_gem.h>
 #include <radeon_cs_gem.h>
@@ -187,7 +186,6 @@ static void radeon_winsys_destroy(struct r300_winsys_screen *rws)
     ws->kman->destroy(ws->kman);
 
     radeon_bo_manager_gem_dtor(ws->bom);
-    radeon_cs_manager_gem_dtor(ws->csm);
     FREE(rws);
 }
 
@@ -209,9 +207,6 @@ struct r300_winsys_screen *r300_drm_winsys_screen_create(int fd)
     ws->bom = radeon_bo_manager_gem_ctor(fd);
     if (!ws->bom)
 	goto fail;
-    ws->csm = radeon_cs_manager_gem_ctor(fd);
-    if (!ws->csm)
-	goto fail;
     ws->kman = radeon_drm_bufmgr_create(ws);
     if (!ws->kman)
 	goto fail;
@@ -223,6 +218,7 @@ struct r300_winsys_screen *r300_drm_winsys_screen_create(int fd)
     ws->base.destroy = radeon_winsys_destroy;
 
     radeon_drm_bufmgr_init_functions(ws);
+    radeon_drm_cs_init_functions(ws);
     radeon_winsys_init_functions(ws);
 
     return &ws->base;
@@ -230,8 +226,6 @@ struct r300_winsys_screen *r300_drm_winsys_screen_create(int fd)
 fail:
     if (ws->bom)
 	radeon_bo_manager_gem_dtor(ws->bom);
-    if (ws->csm)
-	radeon_cs_manager_gem_dtor(ws->csm);
 
     if (ws->cman)
 	ws->cman->destroy(ws->cman);
diff --git a/src/gallium/winsys/radeon/drm/radeon_drm_cs.c b/src/gallium/winsys/radeon/drm/radeon_drm_cs.c
new file mode 100644
index 00000000000..60bc36b0929
--- /dev/null
+++ b/src/gallium/winsys/radeon/drm/radeon_drm_cs.c
@@ -0,0 +1,360 @@
+/*
+ * Copyright © 2008 Jérôme Glisse
+ * Copyright © 2010 Marek Olšák <[email protected]>
+ * All Rights Reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sub license, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NON-INFRINGEMENT. IN NO EVENT SHALL THE COPYRIGHT HOLDERS, AUTHORS
+ * AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
+ * USE OR OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * The above copyright notice and this permission notice (including the
+ * next paragraph) shall be included in all copies or substantial portions
+ * of the Software.
+ */
+/*
+ * Authors:
+ *      Marek Olšák <[email protected]>
+ *
+ * Based on work from libdrm_radeon by:
+ *      Aapo Tahkola <[email protected]>
+ *      Nicolai Haehnle <[email protected]>
+ *      Jérôme Glisse <[email protected]>
+ */
+
+/*
+    This file replaces libdrm's radeon_cs_gem with our own implemention.
+    It's optimized specifically for r300g, but r600g could use it as well.
+    Reloc writes and space checking are faster and simpler than their
+    counterparts in libdrm (the time complexity of all the functions
+    is O(1) in nearly all scenarios, thanks to hashing).
+
+    It works like this:
+
+    cs_add_reloc(cs, buf, read_domain, write_domain) adds a new relocation and
+    also adds the size of 'buf' to the used_gart and used_vram winsys variables
+    based on the domains, which are simply or'd for the accounting purposes.
+    The adding is skipped if the reloc is already present in the list, but it
+    accounts any newly-referenced domains.
+
+    cs_validate is then called, which just checks:
+        used_vram/gart < vram/gart_size * 0.8
+    The 0.8 number allows for some memory fragmentation. If the validation
+    fails, the pipe driver flushes CS and tries do the validation again,
+    i.e. it validates only that one operation. If it fails again, it drops
+    the operation on the floor and prints some nasty message to stderr.
+    (done in the pipe driver)
+
+    cs_write_reloc(cs, buf) just writes a reloc that has been added using
+    cs_add_reloc. The read_domain and write_domain parameters have been removed,
+    because we already specify them in cs_add_reloc.
+*/
+
+#include "radeon_drm_cs.h"
+#include "radeon_drm_buffer.h"
+
+#include "util/u_memory.h"
+
+#include <stdlib.h>
+#include <stdint.h>
+#include <radeon_bo.h>
+#include <xf86drm.h>
+
+#define RELOC_DWORDS (sizeof(struct drm_radeon_cs_reloc) / sizeof(uint32_t))
+
+static struct r300_winsys_cs *radeon_drm_cs_create(struct r300_winsys_screen *rws)
+{
+    struct radeon_drm_winsys *ws = radeon_drm_winsys(rws);
+    struct radeon_drm_cs *cs;
+
+    cs = CALLOC_STRUCT(radeon_drm_cs);
+    if (!cs) {
+        return NULL;
+    }
+
+    cs->ws = ws;
+    cs->nrelocs = 256;
+    cs->relocs_bo = (struct radeon_bo**)
+                     CALLOC(1, cs->nrelocs * sizeof(struct radeon_bo*));
+    if (!cs->relocs_bo) {
+        FREE(cs);
+        return NULL;
+    }
+
+    cs->relocs = (struct drm_radeon_cs_reloc*)
+                 CALLOC(1, cs->nrelocs * sizeof(struct drm_radeon_cs_reloc));
+    if (!cs->relocs) {
+        FREE(cs->relocs_bo);
+        FREE(cs);
+        return NULL;
+    }
+
+    cs->chunks[0].chunk_id = RADEON_CHUNK_ID_IB;
+    cs->chunks[0].length_dw = 0;
+    cs->chunks[0].chunk_data = (uint64_t)(uintptr_t)cs->base.buf;
+    cs->chunks[1].chunk_id = RADEON_CHUNK_ID_RELOCS;
+    cs->chunks[1].length_dw = 0;
+    cs->chunks[1].chunk_data = (uint64_t)(uintptr_t)cs->relocs;
+    return &cs->base;
+}
+
+#define OUT_CS(cs, value) (cs)->buf[(cs)->cdw++] = (value)
+
+static inline void update_domains(struct drm_radeon_cs_reloc *reloc,
+                                  enum r300_buffer_domain rd,
+                                  enum r300_buffer_domain wd,
+                                  enum r300_buffer_domain *added_domains)
+{
+    *added_domains = (rd | wd) & ~(reloc->read_domains | reloc->write_domain);
+
+    if (reloc->read_domains & wd) {
+        reloc->read_domains = rd;
+        reloc->write_domain = wd;
+    } else if (rd & reloc->write_domain) {
+        reloc->read_domains = rd;
+        reloc->write_domain |= wd;
+    } else {
+        reloc->read_domains |= rd;
+        reloc->write_domain |= wd;
+    }
+}
+
+static int radeon_get_reloc(struct radeon_drm_cs *cs,
+                            struct radeon_bo *bo)
+{
+    struct drm_radeon_cs_reloc *reloc;
+    unsigned i;
+    unsigned hash = bo->handle & (sizeof(cs->is_handle_added)-1);
+
+    if (cs->is_handle_added[hash]) {
+        reloc = cs->relocs_hashlist[hash];
+        if (reloc->handle == bo->handle) {
+            return cs->reloc_indices_hashlist[hash];
+        }
+
+        /* Hash collision, look for the BO in the list of relocs linearly. */
+        for (i = cs->crelocs; i != 0;) {
+            --i;
+            reloc = &cs->relocs[i];
+            if (reloc->handle == bo->handle) {
+                /* Put this reloc in the hash list.
+                 * This will prevent additional hash collisions if there are
+                 * several subsequent get_reloc calls of the same buffer.
+                 *
+                 * Example: Assuming buffers A,B,C collide in the hash list,
+                 * the following sequence of relocs:
+                 *         AAAAAAAAAAABBBBBBBBBBBBBBCCCCCCCC
+                 * will collide here: ^ and here:   ^,
+                 * meaning that we should get very few collisions in the end. */
+                cs->relocs_hashlist[hash] = reloc;
+                cs->reloc_indices_hashlist[hash] = i;
+                /*printf("write_reloc collision, hash: %i, handle: %i\n", hash, bo->handle);*/
+                return i;
+            }
+        }
+    }
+
+    return -1;
+}
+
+static void radeon_add_reloc(struct radeon_drm_cs *cs,
+                             struct radeon_bo *bo,
+                             enum r300_buffer_domain rd,
+                             enum r300_buffer_domain wd,
+                             enum r300_buffer_domain *added_domains)
+{
+    struct drm_radeon_cs_reloc *reloc;
+    unsigned i;
+    unsigned hash = bo->handle & (sizeof(cs->is_handle_added)-1);
+
+    if (cs->is_handle_added[hash]) {
+        reloc = cs->relocs_hashlist[hash];
+        if (reloc->handle == bo->handle) {
+            update_domains(reloc, rd, wd, added_domains);
+            return;
+        }
+
+        /* Hash collision, look for the BO in the list of relocs linearly. */
+        for (i = cs->crelocs; i != 0;) {
+            --i;
+            reloc = &cs->relocs[i];
+            if (reloc->handle == bo->handle) {
+                update_domains(reloc, rd, wd, added_domains);
+
+                cs->relocs_hashlist[hash] = reloc;
+                cs->reloc_indices_hashlist[hash] = i;
+                /*printf("write_reloc collision, hash: %i, handle: %i\n", hash, bo->handle);*/
+                return;
+            }
+        }
+    }
+
+    /* New relocation, check if the backing array is large enough. */
+    if (cs->crelocs >= cs->nrelocs) {
+        uint32_t size;
+        cs->nrelocs += 10;
+
+        size = cs->nrelocs * sizeof(struct radeon_bo*);
+        cs->relocs_bo = (struct radeon_bo**)realloc(cs->relocs_bo, size);
+
+        size = cs->nrelocs * sizeof(struct drm_radeon_cs_reloc);
+        cs->relocs = (struct drm_radeon_cs_reloc*)realloc(cs->relocs, size);
+
+        cs->chunks[1].chunk_data = (uint64_t)(uintptr_t)cs->relocs;
+    }
+
+    /* Initialize the new relocation. */
+    radeon_bo_ref(bo);
+    cs->relocs_bo[cs->crelocs] = bo;
+    reloc = &cs->relocs[cs->crelocs];
+    reloc->handle = bo->handle;
+    reloc->read_domains = rd;
+    reloc->write_domain = wd;
+    reloc->flags = 0;
+
+    cs->is_handle_added[hash] = TRUE;
+    cs->relocs_hashlist[hash] = reloc;
+    cs->reloc_indices_hashlist[hash] = cs->crelocs;
+
+    cs->chunks[1].length_dw += RELOC_DWORDS;
+    cs->crelocs++;
+
+    *added_domains = rd | wd;
+}
+
+static void radeon_drm_cs_add_reloc(struct r300_winsys_cs *rcs,
+                                    struct r300_winsys_cs_buffer *buf,
+                                    enum r300_buffer_domain rd,
+                                    enum r300_buffer_domain wd)
+{
+    struct radeon_drm_cs *cs = radeon_drm_cs(rcs);
+    struct radeon_bo *bo = (struct radeon_bo*)buf;
+    enum r300_buffer_domain added_domains;
+
+    radeon_add_reloc(cs, bo, rd, wd, &added_domains);
+
+    if (!added_domains)
+        return;
+
+    if (added_domains & R300_DOMAIN_GTT)
+        cs->used_gart += bo->size;
+    if (added_domains & R300_DOMAIN_VRAM)
+        cs->used_vram += bo->size;
+}
+
+static boolean radeon_drm_cs_validate(struct r300_winsys_cs *rcs)
+{
+    struct radeon_drm_cs *cs = radeon_drm_cs(rcs);
+
+    return cs->used_gart < cs->ws->gart_size * 0.8 &&
+           cs->used_vram < cs->ws->vram_size * 0.8;
+}
+
+static void radeon_drm_cs_write_reloc(struct r300_winsys_cs *rcs,
+                                      struct r300_winsys_cs_buffer *buf)
+{
+    struct radeon_drm_cs *cs = radeon_drm_cs(rcs);
+    struct radeon_bo *bo = (struct radeon_bo*)buf;
+
+    unsigned index = radeon_get_reloc(cs, bo);
+
+    if (index == -1) {
+        fprintf(stderr, "r300: Cannot get a relocation in %s.\n", __func__);
+        return;
+    }
+
+    OUT_CS(&cs->base, 0xc0001000);
+    OUT_CS(&cs->base, index * RELOC_DWORDS);
+}
+
+static void radeon_drm_cs_emit(struct r300_winsys_cs *rcs)
+{
+    struct radeon_drm_cs *cs = radeon_drm_cs(rcs);
+    uint64_t chunk_array[2];
+    unsigned i;
+    int r;
+
+    if (cs->base.cdw) {
+        /* Unmap buffers. */
+        radeon_drm_bufmgr_flush_maps(cs->ws->kman);
+
+        /* Prepare the arguments. */
+        cs->chunks[0].length_dw = cs->base.cdw;
+
+        chunk_array[0] = (uint64_t)(uintptr_t)&cs->chunks[0];
+        chunk_array[1] = (uint64_t)(uintptr_t)&cs->chunks[1];
+
+        cs->cs.num_chunks = 2;
+        cs->cs.chunks = (uint64_t)(uintptr_t)chunk_array;
+
+        /* Emit. */
+        r = drmCommandWriteRead(cs->ws->fd, DRM_RADEON_CS,
+                                &cs->cs, sizeof(struct drm_radeon_cs));
+        if (r) {
+            if (debug_get_bool_option("RADEON_DUMP_CS", FALSE)) {
+                fprintf(stderr, "radeon: The kernel rejected CS, dumping...\n");
+                fprintf(stderr, "VENDORID:DEVICEID 0x%04X:0x%04X\n", 0x1002,
+                        cs->ws->pci_id);
+                for (i = 0; i < cs->base.cdw; i++) {
+                    fprintf(stderr, "0x%08X\n", cs->base.buf[i]);
+                }
+            } else {
+                fprintf(stderr, "radeon: The kernel rejected CS, "
+                                "see dmesg for more information.\n");
+            }
+        }
+    }
+
+    /* Unreference buffers, cleanup. */
+    for (i = 0; i < cs->crelocs; i++) {
+        radeon_bo_unref((struct radeon_bo*)cs->relocs_bo[i]);
+        cs->relocs_bo[i] = NULL;
+    }
+
+    cs->base.cdw = 0;
+    cs->crelocs = 0;
+    cs->chunks[0].length_dw = 0;
+    cs->chunks[1].length_dw = 0;
+    cs->used_gart = 0;
+    cs->used_vram = 0;
+    memset(cs->is_handle_added, 0, sizeof(cs->is_handle_added));
+}
+
+static void radeon_drm_cs_destroy(struct r300_winsys_cs *rcs)
+{
+    struct radeon_drm_cs *cs = radeon_drm_cs(rcs);
+    FREE(cs->relocs_bo);
+    FREE(cs->relocs);
+    FREE(cs);
+}
+
+static void radeon_drm_cs_set_flush(struct r300_winsys_cs *rcs,
+                             void (*flush)(void *), void *user)
+{
+    struct radeon_drm_cs *cs = radeon_drm_cs(rcs);
+    cs->flush_cs = flush;
+    cs->flush_data = user;
+}
+
+void radeon_drm_cs_init_functions(struct radeon_drm_winsys *ws)
+{
+    ws->base.cs_create = radeon_drm_cs_create;
+    ws->base.cs_destroy = radeon_drm_cs_destroy;
+    ws->base.cs_add_reloc = radeon_drm_cs_add_reloc;
+    ws->base.cs_validate = radeon_drm_cs_validate;
+    ws->base.cs_write_reloc = radeon_drm_cs_write_reloc;
+    ws->base.cs_flush = radeon_drm_cs_emit;
+    ws->base.cs_set_flush = radeon_drm_cs_set_flush;
+}
diff --git a/src/gallium/winsys/radeon/drm/radeon_drm_cs.h b/src/gallium/winsys/radeon/drm/radeon_drm_cs.h
new file mode 100644
index 00000000000..76046534b65
--- /dev/null
+++ b/src/gallium/winsys/radeon/drm/radeon_drm_cs.h
@@ -0,0 +1,42 @@
+#ifndef RADEON_DRM_CS_H
+#define RADEON_DRM_CS_H
+
+#include "radeon_winsys.h"
+#include <radeon_drm.h>
+
+struct radeon_drm_cs {
+    struct r300_winsys_cs base;
+
+    /* The winsys. */
+    struct radeon_drm_winsys *ws;
+
+    /* Flush CS. */
+    void (*flush_cs)(void *);
+    void *flush_data;
+
+    /* Relocs. */
+    unsigned                    crelocs;
+    unsigned                    nrelocs;
+    struct drm_radeon_cs_reloc  *relocs;
+    struct radeon_bo            **relocs_bo;
+    struct drm_radeon_cs        cs;
+    struct drm_radeon_cs_chunk  chunks[2];
+
+    unsigned used_vram;
+    unsigned used_gart;
+
+    /* 0 = BO not added, 1 = BO added */
+    char                        is_handle_added[256];
+    struct drm_radeon_cs_reloc  *relocs_hashlist[256];
+    unsigned                    reloc_indices_hashlist[256];
+};
+
+static INLINE struct radeon_drm_cs *
+radeon_drm_cs(struct r300_winsys_cs *base)
+{
+    return (struct radeon_drm_cs*)base;
+}
+
+void radeon_drm_cs_init_functions(struct radeon_drm_winsys *ws);
+
+#endif
diff --git a/src/gallium/winsys/radeon/drm/radeon_r300.c b/src/gallium/winsys/radeon/drm/radeon_r300.c
index ae7020a0639..bacf181b47c 100644
--- a/src/gallium/winsys/radeon/drm/radeon_r300.c
+++ b/src/gallium/winsys/radeon/drm/radeon_r300.c
@@ -25,7 +25,6 @@
 #include "util/u_memory.h"
 #include "pipebuffer/pb_bufmgr.h"
 
-#include "radeon_cs_gem.h"
 #include "state_tracker/drm_driver.h"
 
 static unsigned get_pb_usage_from_create_flags(unsigned bind, unsigned usage,
@@ -134,65 +133,6 @@ static boolean radeon_r300_winsys_buffer_get_handle(struct r300_winsys_screen *r
     return radeon_drm_bufmgr_get_handle(_buf, whandle);
 }
 
-static void radeon_r300_winsys_cs_set_flush(struct r300_winsys_cs *rcs,
-                                            void (*flush)(void *),
-                                            void *user)
-{
-    struct radeon_drm_cs *cs = radeon_drm_cs(rcs);
-    cs->flush_cs = flush;
-    cs->flush_data = user;
-    radeon_cs_space_set_flush(cs->cs, flush, user);
-}
-
-static boolean radeon_r300_winsys_cs_validate(struct r300_winsys_cs *rcs)
-{
-    struct radeon_drm_cs *cs = radeon_drm_cs(rcs);
-
-    return radeon_cs_space_check(cs->cs) >= 0;
-}
-
-static void radeon_r300_winsys_cs_reset_buffers(struct r300_winsys_cs *rcs)
-{
-    struct radeon_drm_cs *cs = radeon_drm_cs(rcs);
-    radeon_cs_space_reset_bos(cs->cs);
-}
-
-static void radeon_r300_winsys_cs_flush(struct r300_winsys_cs *rcs)
-{
-    struct radeon_drm_cs *cs = radeon_drm_cs(rcs);
-    int retval;
-
-    /* Don't flush a zero-sized CS. */
-    if (!cs->base.cdw) {
-        return;
-    }
-
-    cs->cs->cdw = cs->base.cdw;
-
-    radeon_drm_bufmgr_flush_maps(cs->ws->kman);
-
-    /* Emit the CS. */
-    retval = radeon_cs_emit(cs->cs);
-    if (retval) {
-        if (debug_get_bool_option("RADEON_DUMP_CS", FALSE)) {
-            fprintf(stderr, "radeon: The kernel rejected CS, dumping...\n");
-            radeon_cs_print(cs->cs, stderr);
-        } else {
-            fprintf(stderr, "radeon: The kernel rejected CS, "
-                            "see dmesg for more information.\n");
-        }
-    }
-
-    /* Reset CS.
-     * Someday, when we care about performance, we should really find a way
-     * to rotate between two or three CS objects so that the GPU can be
-     * spinning through one CS while another one is being filled. */
-    radeon_cs_erase(cs->cs);
-
-    cs->base.buf = cs->cs->packets;
-    cs->base.cdw = cs->cs->cdw;
-}
-
 static uint32_t radeon_get_value(struct r300_winsys_screen *rws,
                                  enum r300_value_id id)
 {
@@ -219,39 +159,6 @@ static uint32_t radeon_get_value(struct r300_winsys_screen *rws,
     return 0;
 }
 
-static struct r300_winsys_cs *radeon_r300_winsys_cs_create(struct r300_winsys_screen *rws)
-{
-    struct radeon_drm_winsys *ws = radeon_drm_winsys(rws);
-    struct radeon_drm_cs *cs = CALLOC_STRUCT(radeon_drm_cs);
-
-    if (!cs)
-        return NULL;
-
-    /* Size limit on IBs is 64 kibibytes. */
-    cs->cs = radeon_cs_create(ws->csm, 1024 * 64 / 4);
-    if (!cs->cs) {
-        FREE(cs);
-        return NULL;
-    }
-
-    radeon_cs_set_limit(cs->cs,
-            RADEON_GEM_DOMAIN_GTT, ws->gart_size);
-    radeon_cs_set_limit(cs->cs,
-            RADEON_GEM_DOMAIN_VRAM, ws->vram_size);
-
-    cs->ws = ws;
-    cs->base.buf = cs->cs->packets;
-    cs->base.cdw = cs->cs->cdw;
-    return &cs->base;
-}
-
-static void radeon_r300_winsys_cs_destroy(struct r300_winsys_cs *rcs)
-{
-    struct radeon_drm_cs *cs = radeon_drm_cs(rcs);
-    radeon_cs_destroy(cs->cs);
-    FREE(cs);
-}
-
 void radeon_winsys_init_functions(struct radeon_drm_winsys *ws)
 {
     ws->base.get_value = radeon_get_value;
@@ -259,10 +166,4 @@ void radeon_winsys_init_functions(struct radeon_drm_winsys *ws)
     ws->base.buffer_reference = radeon_r300_winsys_buffer_reference;
     ws->base.buffer_from_handle = radeon_r300_winsys_buffer_from_handle;
     ws->base.buffer_get_handle = radeon_r300_winsys_buffer_get_handle;
-    ws->base.cs_create = radeon_r300_winsys_cs_create;
-    ws->base.cs_destroy = radeon_r300_winsys_cs_destroy;
-    ws->base.cs_validate = radeon_r300_winsys_cs_validate;
-    ws->base.cs_flush = radeon_r300_winsys_cs_flush;
-    ws->base.cs_reset_buffers = radeon_r300_winsys_cs_reset_buffers;
-    ws->base.cs_set_flush = radeon_r300_winsys_cs_set_flush;
 }
diff --git a/src/gallium/winsys/radeon/drm/radeon_winsys.h b/src/gallium/winsys/radeon/drm/radeon_winsys.h
index 6f232143f6a..492edfef8c3 100644
--- a/src/gallium/winsys/radeon/drm/radeon_winsys.h
+++ b/src/gallium/winsys/radeon/drm/radeon_winsys.h
@@ -55,31 +55,8 @@ struct radeon_drm_winsys {
     boolean drm_2_8_0;
     /* Hyper-Z user */
     boolean hyperz;
-
-    /* Radeon CS manager. */
-    struct radeon_cs_manager *csm;
 };
 
-struct radeon_drm_cs {
-    struct r300_winsys_cs base;
-
-    /* The winsys. */
-    struct radeon_drm_winsys *ws;
-
-    /* The libdrm command stream. */
-    struct radeon_cs *cs;
-
-    /* Flush CS. */
-    void (*flush_cs)(void *);
-    void *flush_data;
-};
-
-static INLINE struct radeon_drm_cs *
-radeon_drm_cs(struct r300_winsys_cs *base)
-{
-    return (struct radeon_drm_cs*)base;
-}
-
 static INLINE struct radeon_drm_winsys *
 radeon_drm_winsys(struct r300_winsys_screen *base)
 {
diff --git a/src/gallium/winsys/svga/drm/vmw_screen.c b/src/gallium/winsys/svga/drm/vmw_screen.c
index 51a4c55e5a2..cc3003d252f 100644
--- a/src/gallium/winsys/svga/drm/vmw_screen.c
+++ b/src/gallium/winsys/svga/drm/vmw_screen.c
@@ -45,8 +45,6 @@ vmw_winsys_create( int fd, boolean use_old_scanout_flag )
 
    vws->ioctl.drm_fd = fd;
    vws->use_old_scanout_flag = use_old_scanout_flag;
-   debug_printf("%s: use_old_scanout_flag == %s\n", __FUNCTION__,
-		use_old_scanout_flag ? "true" : "false");
 
    if (!vmw_ioctl_init(vws))
       goto out_no_ioctl;
diff --git a/src/gallium/winsys/sw/wrapper/wrapper_sw_winsys.c b/src/gallium/winsys/sw/wrapper/wrapper_sw_winsys.c
index 8f9a90858de..4ecfdbf3915 100644
--- a/src/gallium/winsys/sw/wrapper/wrapper_sw_winsys.c
+++ b/src/gallium/winsys/sw/wrapper/wrapper_sw_winsys.c
@@ -61,10 +61,8 @@ struct wrapper_sw_displaytarget
    struct pipe_resource *tex;
    struct pipe_transfer *transfer;
 
-   unsigned width;
-   unsigned height;
    unsigned map_count;
-   unsigned stride; /**< because we give stride at create */
+   unsigned stride; /**< because we get stride at create */
    void *ptr;
 };
 
@@ -95,7 +93,7 @@ wsw_dt_get_stride(struct wrapper_sw_displaytarget *wdt, unsigned *stride)
 
    tr = pipe_get_transfer(pipe, tex, 0, 0,
                           PIPE_TRANSFER_READ_WRITE,
-                          0, 0, wdt->width, wdt->height);
+                          0, 0, wdt->tex->width0, wdt->tex->height0);
    if (!tr)
       return FALSE;
 
@@ -208,7 +206,7 @@ wsw_dt_map(struct sw_winsys *ws,
 
       tr = pipe_get_transfer(pipe, tex, 0, 0,
                              PIPE_TRANSFER_READ_WRITE,
-                             0, 0, wdt->width, wdt->height);
+                             0, 0, wdt->tex->width0, wdt->tex->height0);
       if (!tr)
          return NULL;
 
@@ -248,6 +246,7 @@ wsw_dt_unmap(struct sw_winsys *ws,
 
    pipe->transfer_unmap(pipe, wdt->transfer);
    pipe->transfer_destroy(pipe, wdt->transfer);
+   pipe->flush(pipe, 0, NULL);
    wdt->transfer = NULL;
 }
 
diff --git a/src/glsl/ast.h b/src/glsl/ast.h
index a77b522705c..cd933cfc588 100644
--- a/src/glsl/ast.h
+++ b/src/glsl/ast.h
@@ -349,7 +349,11 @@ struct ast_type_qualifier {
 	  * qualifier is used.
 	  */
 	 unsigned explicit_location:1;
-      } q;
+      }
+      /** \brief Set of flags, accessed by name. */
+      q;
+
+      /** \brief Set of flags, accessed as a bitmask. */
       unsigned i;
    } flags;
 
diff --git a/src/glsl/ast_to_hir.cpp b/src/glsl/ast_to_hir.cpp
index 741cd19e9d3..a8dcae6e810 100644
--- a/src/glsl/ast_to_hir.cpp
+++ b/src/glsl/ast_to_hir.cpp
@@ -60,7 +60,7 @@ void
 _mesa_ast_to_hir(exec_list *instructions, struct _mesa_glsl_parse_state *state)
 {
    _mesa_glsl_initialize_variables(instructions, state);
-   _mesa_glsl_initialize_functions(instructions, state);
+   _mesa_glsl_initialize_functions(state);
 
    state->symbols->language_version = state->language_version;
 
@@ -1564,18 +1564,38 @@ ast_expression::hir(exec_list *instructions,
 	 }
       }
 
-      /* From section 4.1.7 of the GLSL 1.30 spec:
+      /* From page 23 (29 of the PDF) of the GLSL 1.30 spec:
+       *
        *    "Samplers aggregated into arrays within a shader (using square
        *    brackets [ ]) can only be indexed with integral constant
        *    expressions [...]."
+       *
+       * This restriction was added in GLSL 1.30.  Shaders using earlier version
+       * of the language should not be rejected by the compiler front-end for
+       * using this construct.  This allows useful things such as using a loop
+       * counter as the index to an array of samplers.  If the loop in unrolled,
+       * the code should compile correctly.  Instead, emit a warning.
        */
       if (array->type->is_array() &&
           array->type->element_type()->is_sampler() &&
           const_index == NULL) {
 
-         _mesa_glsl_error(&loc, state, "sampler arrays can only be indexed "
-                          "with constant expressions");
-         error_emitted = true;
+	 if (state->language_version == 100) {
+	    _mesa_glsl_warning(&loc, state,
+			       "sampler arrays indexed with non-constant "
+			       "expressions is optional in GLSL ES 1.00");
+	 } else if (state->language_version < 130) {
+	    _mesa_glsl_warning(&loc, state,
+			       "sampler arrays indexed with non-constant "
+			       "expressions is forbidden in GLSL 1.30 and "
+			       "later");
+	 } else {
+	    _mesa_glsl_error(&loc, state,
+			     "sampler arrays indexed with non-constant "
+			     "expressions is forbidden in GLSL 1.30 and "
+			     "later");
+	    error_emitted = true;
+	 }
       }
 
       if (error_emitted)
@@ -1832,6 +1852,23 @@ apply_type_qualifier_to_variable(const struct ast_type_qualifier *qual,
    else if (qual->flags.q.uniform)
       var->mode = ir_var_uniform;
 
+   if (state->all_invariant && (state->current_function == NULL)) {
+      switch (state->target) {
+      case vertex_shader:
+	 if (var->mode == ir_var_out)
+	    var->invariant = true;
+	 break;
+      case geometry_shader:
+	 if ((var->mode == ir_var_in) || (var->mode == ir_var_out))
+	    var->invariant = true;
+	 break;
+      case fragment_shader:
+	 if (var->mode == ir_var_in)
+	    var->invariant = true;
+	 break;
+      }
+   }
+
    if (qual->flags.q.flat)
       var->interpolation = ir_var_flat;
    else if (qual->flags.q.noperspective)
@@ -2158,6 +2195,25 @@ ast_declarator_list::hir(exec_list *instructions,
 	 }
       }
 
+      /* Integer vertex outputs must be qualified with 'flat'.
+       *
+       * From section 4.3.6 of the GLSL 1.30 spec:
+       *    "If a vertex output is a signed or unsigned integer or integer
+       *    vector, then it must be qualified with the interpolation qualifier
+       *    flat."
+       */
+      if (state->language_version >= 130
+          && state->target == vertex_shader
+          && state->current_function == NULL
+          && var->type->is_integer()
+          && var->mode == ir_var_out
+          && var->interpolation != ir_var_flat) {
+
+         _mesa_glsl_error(&loc, state, "If a vertex output is an integer, "
+                          "then it must be qualified with 'flat'");
+      }
+
+
       /* Process the initializer and add its instructions to a temporary
        * list.  This list will be added to the instruction stream (below) after
        * the declaration is added.  This is done because in some cases (such as
@@ -2344,6 +2400,27 @@ ast_declarator_list::hir(exec_list *instructions,
 	     */
 	    earlier->origin_upper_left = var->origin_upper_left;
 	    earlier->pixel_center_integer = var->pixel_center_integer;
+
+	 /* According to section 4.3.7 of the GLSL 1.30 spec,
+	  * the following built-in varaibles can be redeclared with an
+	  * interpolation qualifier:
+	  *    * gl_FrontColor
+	  *    * gl_BackColor
+	  *    * gl_FrontSecondaryColor
+	  *    * gl_BackSecondaryColor
+	  *    * gl_Color
+	  *    * gl_SecondaryColor
+	  */
+	 } else if (state->language_version >= 130
+	            && (strcmp(var->name, "gl_FrontColor") == 0
+                        || strcmp(var->name, "gl_BackColor") == 0
+                        || strcmp(var->name, "gl_FrontSecondaryColor") == 0
+                        || strcmp(var->name, "gl_BackSecondaryColor") == 0
+                        || strcmp(var->name, "gl_Color") == 0
+                        || strcmp(var->name, "gl_SecondaryColor") == 0)
+	            && earlier->type == var->type
+	            && earlier->mode == var->mode) {
+	    earlier->interpolation = var->interpolation;
 	 } else {
 	    YYLTYPE loc = this->get_location();
 	    _mesa_glsl_error(&loc, state, "`%s' redeclared", decl->identifier);
diff --git a/src/glsl/builtin_function.cpp b/src/glsl/builtin_function.cpp
index 1c6d59d5a21..b9db141b3c2 100644
--- a/src/glsl/builtin_function.cpp
+++ b/src/glsl/builtin_function.cpp
@@ -13523,7 +13523,6 @@ _mesa_glsl_release_functions(void)
 
 static void
 _mesa_read_profile(struct _mesa_glsl_parse_state *state,
-		   exec_list *instructions,
                    int profile_index,
 		   const char *prototypes,
 		   const char **functions,
@@ -13542,8 +13541,7 @@ _mesa_read_profile(struct _mesa_glsl_parse_state *state,
 }
 
 void
-_mesa_glsl_initialize_functions(exec_list *instructions,
-                                struct _mesa_glsl_parse_state *state)
+_mesa_glsl_initialize_functions(struct _mesa_glsl_parse_state *state)
 {
    if (builtin_mem_ctx == NULL) {
       builtin_mem_ctx = talloc_init("GLSL built-in functions");
@@ -13553,84 +13551,84 @@ _mesa_glsl_initialize_functions(exec_list *instructions,
    state->num_builtins_to_link = 0;
 
    if (state->target == fragment_shader && state->language_version == 100) {
-      _mesa_read_profile(state, instructions, 0,
+      _mesa_read_profile(state, 0,
                          prototypes_for_100_frag,
                          functions_for_100_frag,
                          Elements(functions_for_100_frag));
    }
 
    if (state->target == vertex_shader && state->language_version == 100) {
-      _mesa_read_profile(state, instructions, 1,
+      _mesa_read_profile(state, 1,
                          prototypes_for_100_vert,
                          functions_for_100_vert,
                          Elements(functions_for_100_vert));
    }
 
    if (state->target == fragment_shader && state->language_version == 110) {
-      _mesa_read_profile(state, instructions, 2,
+      _mesa_read_profile(state, 2,
                          prototypes_for_110_frag,
                          functions_for_110_frag,
                          Elements(functions_for_110_frag));
    }
 
    if (state->target == vertex_shader && state->language_version == 110) {
-      _mesa_read_profile(state, instructions, 3,
+      _mesa_read_profile(state, 3,
                          prototypes_for_110_vert,
                          functions_for_110_vert,
                          Elements(functions_for_110_vert));
    }
 
    if (state->target == fragment_shader && state->language_version == 120) {
-      _mesa_read_profile(state, instructions, 4,
+      _mesa_read_profile(state, 4,
                          prototypes_for_120_frag,
                          functions_for_120_frag,
                          Elements(functions_for_120_frag));
    }
 
    if (state->target == vertex_shader && state->language_version == 120) {
-      _mesa_read_profile(state, instructions, 5,
+      _mesa_read_profile(state, 5,
                          prototypes_for_120_vert,
                          functions_for_120_vert,
                          Elements(functions_for_120_vert));
    }
 
    if (state->target == fragment_shader && state->language_version == 130) {
-      _mesa_read_profile(state, instructions, 6,
+      _mesa_read_profile(state, 6,
                          prototypes_for_130_frag,
                          functions_for_130_frag,
                          Elements(functions_for_130_frag));
    }
 
    if (state->target == vertex_shader && state->language_version == 130) {
-      _mesa_read_profile(state, instructions, 7,
+      _mesa_read_profile(state, 7,
                          prototypes_for_130_vert,
                          functions_for_130_vert,
                          Elements(functions_for_130_vert));
    }
 
    if (state->target == fragment_shader && state->ARB_texture_rectangle_enable) {
-      _mesa_read_profile(state, instructions, 8,
+      _mesa_read_profile(state, 8,
                          prototypes_for_ARB_texture_rectangle_frag,
                          functions_for_ARB_texture_rectangle_frag,
                          Elements(functions_for_ARB_texture_rectangle_frag));
    }
 
    if (state->target == vertex_shader && state->ARB_texture_rectangle_enable) {
-      _mesa_read_profile(state, instructions, 9,
+      _mesa_read_profile(state, 9,
                          prototypes_for_ARB_texture_rectangle_vert,
                          functions_for_ARB_texture_rectangle_vert,
                          Elements(functions_for_ARB_texture_rectangle_vert));
    }
 
    if (state->target == fragment_shader && state->EXT_texture_array_enable) {
-      _mesa_read_profile(state, instructions, 10,
+      _mesa_read_profile(state, 10,
                          prototypes_for_EXT_texture_array_frag,
                          functions_for_EXT_texture_array_frag,
                          Elements(functions_for_EXT_texture_array_frag));
    }
 
    if (state->target == vertex_shader && state->EXT_texture_array_enable) {
-      _mesa_read_profile(state, instructions, 11,
+      _mesa_read_profile(state, 11,
                          prototypes_for_EXT_texture_array_vert,
                          functions_for_EXT_texture_array_vert,
                          Elements(functions_for_EXT_texture_array_vert));
diff --git a/src/glsl/builtins/tools/builtin_function.cpp b/src/glsl/builtins/tools/builtin_function.cpp
index c44804f2ef7..00b32949158 100644
--- a/src/glsl/builtins/tools/builtin_function.cpp
+++ b/src/glsl/builtins/tools/builtin_function.cpp
@@ -33,7 +33,6 @@ _mesa_glsl_release_functions(void)
 }
 
 void
-_mesa_glsl_initialize_functions(exec_list *instructions,
-			        struct _mesa_glsl_parse_state *state)
+_mesa_glsl_initialize_functions(_mesa_glsl_parse_state *state)
 {
 }
diff --git a/src/glsl/builtins/tools/generate_builtins.py b/src/glsl/builtins/tools/generate_builtins.py
index 5ea4b5c48f4..e2de9dbcdca 100755
--- a/src/glsl/builtins/tools/generate_builtins.py
+++ b/src/glsl/builtins/tools/generate_builtins.py
@@ -200,7 +200,6 @@ _mesa_glsl_release_functions(void)
 
 static void
 _mesa_read_profile(struct _mesa_glsl_parse_state *state,
-		   exec_list *instructions,
                    int profile_index,
 		   const char *prototypes,
 		   const char **functions,
@@ -219,8 +218,7 @@ _mesa_read_profile(struct _mesa_glsl_parse_state *state,
 }
 
 void
-_mesa_glsl_initialize_functions(exec_list *instructions,
-                                struct _mesa_glsl_parse_state *state)
+_mesa_glsl_initialize_functions(struct _mesa_glsl_parse_state *state)
 {
    if (builtin_mem_ctx == NULL) {
       builtin_mem_ctx = talloc_init("GLSL built-in functions");
@@ -244,7 +242,7 @@ _mesa_glsl_initialize_functions(exec_list *instructions,
             check += 'state->' + version + '_enable'
 
         print '   if (' + check + ') {'
-        print '      _mesa_read_profile(state, instructions, %d,' % i
+        print '      _mesa_read_profile(state, %d,' % i
         print '                         prototypes_for_' + profile + ','
         print '                         functions_for_' + profile + ','
         print '                         Elements(functions_for_' + profile + '));'
diff --git a/src/glsl/glcpp/tests/092-redefine-macro-error-2.c b/src/glsl/glcpp/tests/092-redefine-macro-error-2.c
new file mode 100644
index 00000000000..3c161a5c501
--- /dev/null
+++ b/src/glsl/glcpp/tests/092-redefine-macro-error-2.c
@@ -0,0 +1,5 @@
+#define A
+#define A 1
+
+#define B 1
+#define B
diff --git a/src/glsl/glcpp/tests/092-redefine-macro-error-2.c.expected b/src/glsl/glcpp/tests/092-redefine-macro-error-2.c.expected
new file mode 100644
index 00000000000..0026f91f32b
--- /dev/null
+++ b/src/glsl/glcpp/tests/092-redefine-macro-error-2.c.expected
@@ -0,0 +1,10 @@
+0:2(9): preprocessor error: Redefinition of macro A
+
+0:5(9): preprocessor error: Redefinition of macro B
+
+
+
+
+
+
+
diff --git a/src/glsl/glcpp/tests/093-divide-by-zero.c b/src/glsl/glcpp/tests/093-divide-by-zero.c
new file mode 100644
index 00000000000..bf65d4f5271
--- /dev/null
+++ b/src/glsl/glcpp/tests/093-divide-by-zero.c
@@ -0,0 +1,2 @@
+#if (1 / 0)
+#endif
diff --git a/src/glsl/glcpp/tests/094-divide-by-zero-short-circuit.c b/src/glsl/glcpp/tests/094-divide-by-zero-short-circuit.c
new file mode 100644
index 00000000000..a9c6f36def8
--- /dev/null
+++ b/src/glsl/glcpp/tests/094-divide-by-zero-short-circuit.c
@@ -0,0 +1,2 @@
+#if 1 || (1 / 0)
+#endif
diff --git a/src/glsl/glcpp/tests/095-recursive-define.c b/src/glsl/glcpp/tests/095-recursive-define.c
new file mode 100644
index 00000000000..801d90ce2e3
--- /dev/null
+++ b/src/glsl/glcpp/tests/095-recursive-define.c
@@ -0,0 +1,3 @@
+#define A(a, b) B(a, b)
+#define C A(0, C)
+C
diff --git a/src/glsl/glsl_lexer.cpp b/src/glsl/glsl_lexer.cpp
index 39c119001fd..7c0a51b99bf 100644
--- a/src/glsl/glsl_lexer.cpp
+++ b/src/glsl/glsl_lexer.cpp
@@ -54,7 +54,6 @@ typedef int flex_int32_t;
 typedef unsigned char flex_uint8_t; 
 typedef unsigned short int flex_uint16_t;
 typedef unsigned int flex_uint32_t;
-#endif /* ! C99 */
 
 /* Limits of integral types. */
 #ifndef INT8_MIN
@@ -85,6 +84,8 @@ typedef unsigned int flex_uint32_t;
 #define UINT32_MAX             (4294967295U)
 #endif
 
+#endif /* ! C99 */
+
 #endif /* ! FLEXINT_H */
 
 #ifdef __cplusplus
@@ -158,7 +159,15 @@ typedef void* yyscan_t;
 
 /* Size of default input buffer. */
 #ifndef YY_BUF_SIZE
+#ifdef __ia64__
+/* On IA-64, the buffer size is 16k, not 8k.
+ * Moreover, YY_BUF_SIZE is 2*YY_READ_BUF_SIZE in the general case.
+ * Ditto for the __ia64__ case accordingly.
+ */
+#define YY_BUF_SIZE 32768
+#else
 #define YY_BUF_SIZE 16384
+#endif /* __ia64__ */
 #endif
 
 /* The state buf must be large enough to hold one state per character in the main buffer.
@@ -349,8 +358,8 @@ static void yy_fatal_error (yyconst char msg[] ,yyscan_t yyscanner );
 	*yy_cp = '\0'; \
 	yyg->yy_c_buf_p = yy_cp;
 
-#define YY_NUM_RULES 209
-#define YY_END_OF_BUFFER 210
+#define YY_NUM_RULES 210
+#define YY_END_OF_BUFFER 211
 /* This struct is not used in this scanner,
    but its presence is necessary. */
 struct yy_trans_info
@@ -358,98 +367,100 @@ struct yy_trans_info
 	flex_int32_t yy_verify;
 	flex_int32_t yy_nxt;
 	};
-static yyconst flex_int16_t yy_accept[813] =
+static yyconst flex_int16_t yy_accept[836] =
     {   0,
-        0,    0,   15,   15,    0,    0,  210,  208,    1,   20,
-      208,  208,  208,  208,  208,  208,  208,  208,  119,  117,
-      208,  208,  208,  207,  208,  207,  207,  207,  207,  207,
-      207,  207,  207,  207,  207,  207,  207,  207,  207,  207,
-      207,  207,  207,  207,  207,  208,    1,  208,  209,   15,
-       19,  209,   18,   16,   17,   13,   12,    1,  101,  110,
-      102,  113,  107,   96,  109,   97,  116,  121,  108,  122,
-      119,    0,    0,  124,  119,    0,  117,  117,  105,   98,
-      100,   99,  106,  207,  114,  104,  207,  207,  207,  207,
-      207,  207,  207,  207,  207,  207,  207,  207,   29,  207,
-
-      207,  207,  207,  207,  207,  207,  207,  207,  207,  207,
-      207,  207,   33,  207,  207,   60,  207,  207,  207,  207,
-      207,  207,  207,  207,  207,  207,  207,  207,  207,  207,
-      207,  207,  207,  207,  207,  207,  207,  207,  207,  207,
-      207,  207,  207,  207,  207,  207,  207,  207,  207,  115,
-      103,    1,    0,    0,    2,    0,    0,    0,    0,   15,
-       14,   18,   17,    0,  121,  120,    0,  122,    0,  123,
-      118,  111,  112,  207,  127,  207,  207,  207,  207,  207,
-      207,  207,  207,  207,  207,  207,  207,  207,  207,  207,
-      207,  207,  207,  207,  207,  207,   32,  207,  207,  207,
-
-      207,  207,  207,  207,  207,  207,  207,   25,  207,  207,
-      207,  207,  207,  207,  207,  207,  207,  207,  207,   61,
-      207,  207,  207,  207,  207,  207,  207,  207,  207,  207,
-      207,  207,  207,  207,  207,  207,  207,  207,  207,  207,
-      207,  207,  207,  207,  207,  207,  207,  207,  207,  207,
-        0,    0,    0,    0,   14,    0,  121,    0,  120,    0,
-      122,  123,  118,  207,  207,   23,  207,  207,  174,  167,
-      207,  207,  207,  207,  207,  207,  207,  207,  207,   31,
-      130,  207,  207,  207,  207,   67,  207,  207,  135,  149,
-      207,  207,  207,  207,  207,  207,  207,  207,  207,  207,
-
-      207,  207,  146,  170,   48,   49,   50,  207,  207,  207,
-      207,  207,  207,  207,  207,  207,  207,  207,  207,  207,
-      207,  207,  207,  207,  207,  207,  207,  133,  125,  207,
-      207,   26,  207,  207,  207,  207,  207,  207,  207,   45,
-       46,   47,   94,  207,  207,    0,    0,    0,    0,    0,
-      120,  207,  207,   27,   36,   37,   38,  207,  128,  207,
-       22,  207,  207,  207,  207,  157,  158,  159,  207,  126,
-      207,  150,   24,  160,  161,  162,  172,  154,  155,  156,
-      207,  207,  207,   62,  152,  207,  207,  207,   39,   40,
-       41,  207,  207,  207,  207,  207,  207,  207,  207,  207,
-
-      207,  207,  207,  207,  207,  207,  207,  147,  207,  207,
-      207,  207,  207,  207,  207,  207,  207,  207,  129,  207,
-      207,  169,   42,   43,   44,  207,  207,   30,    0,    0,
-        0,    0,  177,  207,  207,  175,  207,  207,  207,  148,
-      143,  180,  207,  207,  207,  207,  207,  207,  138,  207,
-      207,  207,   95,   51,   52,   53,   54,   55,   56,   57,
-       58,   59,  207,  207,  207,  207,  153,  134,  207,  207,
-      141,   35,  207,  207,  166,   68,  142,   93,  178,  136,
-      207,  207,  207,  207,  207,  207,  207,  207,    0,    0,
-        0,    0,  207,  207,  207,  137,   34,  207,  207,  207,
-
-      207,  207,  207,  181,  182,  183,  207,  207,  207,  207,
-      207,  171,  207,  207,  207,  207,  207,  207,  207,  207,
-      131,  207,  207,  207,  207,  207,   63,  207,  207,   64,
-      207,    0,    0,    0,    0,    0,  207,   65,   28,  144,
-      185,  186,  187,  207,  207,  207,  207,  207,  207,  207,
-      207,  207,  207,  207,  207,  139,  207,  207,  207,  207,
-      207,  207,  207,  207,  207,  132,  189,  190,  191,  207,
-      207,  151,  207,  140,    0,    0,    6,    0,    0,    0,
-       11,    3,   21,  207,  207,  207,  207,  207,  207,  207,
-      207,  207,  184,  145,   66,  207,  207,  207,  207,  168,
-
-      207,  176,  173,  206,   70,   71,   72,  207,  207,  207,
-      207,  207,  207,  207,  207,  207,  207,    0,    0,    0,
-        0,    0,    0,  207,  207,  207,  188,  207,  207,  207,
-      207,  207,   81,   82,   83,  207,  207,  207,  207,  207,
-      207,  207,  207,  207,  207,  207,  207,  207,  192,   87,
-       88,   89,  207,    4,    0,    5,    0,    0,    0,    0,
-        0,  207,  207,  207,  207,  207,  207,  207,  203,  207,
-      207,  207,  207,  207,  207,  207,  207,  207,  207,  207,
-       73,  207,  207,  207,  207,  207,  207,    0,    0,    0,
-      207,  207,  204,  193,  207,  194,  207,  207,  207,   84,
-
-      207,  207,  207,  207,  207,  207,  207,  207,  207,  207,
-      207,  205,  207,  207,   90,    0,    0,  195,  196,  207,
-      199,  207,  200,  207,  207,   69,  207,  207,  207,  163,
-      207,  164,  179,  207,  197,  198,  207,  207,    0,    0,
-      207,  207,  207,  207,   74,  207,   75,  207,  207,  207,
-      207,  207,    0,    0,    0,  207,  207,   85,   86,  207,
-       76,  207,  207,   77,  207,   91,   92,    0,    0,    0,
-      207,  207,  207,  207,  207,  207,    0,    0,    0,  207,
-      207,  207,  207,  207,   78,    0,    0,    7,    0,    0,
-      201,  202,  207,  207,  207,    0,    8,    0,    0,  207,
-
-      207,  165,    0,    0,   79,   80,    0,    0,    9,    0,
-       10,    0
+        0,    0,   16,   16,    0,    0,  211,  209,    1,   21,
+      209,  209,  209,  209,  209,  209,  209,  209,  120,  118,
+      209,  209,  209,  208,  209,  208,  208,  208,  208,  208,
+      208,  208,  208,  208,  208,  208,  208,  208,  208,  208,
+      208,  208,  208,  208,  208,  209,    1,  209,  210,   16,
+       20,  210,   19,   17,   18,   14,   13,    1,  102,  111,
+      103,  114,  108,   97,  110,   98,  117,  122,  109,  123,
+      120,    0,    0,  125,  120,    0,  118,  118,  106,   99,
+      101,  100,  107,  208,  115,  105,  208,  208,  208,  208,
+      208,  208,  208,  208,  208,  208,  208,  208,   30,  208,
+
+      208,  208,  208,  208,  208,  208,  208,  208,  208,  208,
+      208,  208,   34,  208,  208,   61,  208,  208,  208,  208,
+      208,  208,  208,  208,  208,  208,  208,  208,  208,  208,
+      208,  208,  208,  208,  208,  208,  208,  208,  208,  208,
+      208,  208,  208,  208,  208,  208,  208,  208,  208,  116,
+      104,    1,    0,    0,    2,    0,    0,    0,    0,   16,
+       15,   19,   18,    0,  122,  121,    0,  123,    0,  124,
+      119,  112,  113,  208,  128,  208,  208,  208,  208,  208,
+      208,  208,  208,  208,  208,  208,  208,  208,  208,  208,
+      208,  208,  208,  208,  208,  208,   33,  208,  208,  208,
+
+      208,  208,  208,  208,  208,  208,  208,   26,  208,  208,
+      208,  208,  208,  208,  208,  208,  208,  208,  208,   62,
+      208,  208,  208,  208,  208,  208,  208,  208,  208,  208,
+      208,  208,  208,  208,  208,  208,  208,  208,  208,  208,
+      208,  208,  208,  208,  208,  208,  208,  208,  208,  208,
+        0,    0,    0,    0,   15,    0,  122,    0,  121,    0,
+      123,  124,  119,  208,  208,   24,  208,  208,  175,  168,
+      208,  208,  208,  208,  208,  208,  208,  208,  208,   32,
+      131,  208,  208,  208,  208,   68,  208,  208,  136,  150,
+      208,  208,  208,  208,  208,  208,  208,  208,  208,  208,
+
+      208,  208,  147,  171,   49,   50,   51,  208,  208,  208,
+      208,  208,  208,  208,  208,  208,  208,  208,  208,  208,
+      208,  208,  208,  208,  208,  208,  208,  134,  126,  208,
+      208,   27,  208,  208,  208,  208,  208,  208,  208,   46,
+       47,   48,   95,  208,  208,    0,    0,    0,    0,    0,
+      121,  208,  208,   28,   37,   38,   39,  208,  129,  208,
+       23,  208,  208,  208,  208,  158,  159,  160,  208,  127,
+      208,  151,   25,  161,  162,  163,  173,  155,  156,  157,
+      208,  208,  208,   63,  153,  208,  208,  208,   40,   41,
+       42,  208,  208,  208,  208,  208,  208,  208,  208,  208,
+
+      208,  208,  208,  208,  208,  208,  208,  148,  208,  208,
+      208,  208,  208,  208,  208,  208,  208,  208,  130,  208,
+      208,  170,   43,   44,   45,  208,  208,   31,    0,    0,
+        0,    0,  178,  208,  208,  176,  208,  208,  208,  149,
+      144,  181,  208,  208,  208,  208,  208,  208,  139,  208,
+      208,  208,   96,   52,   53,   54,   55,   56,   57,   58,
+       59,   60,  208,  208,  208,  208,  154,  135,  208,  208,
+      142,   36,  208,  208,  167,   69,  143,   94,  179,  137,
+      208,  208,  208,  208,  208,  208,  208,  208,    0,    0,
+        0,    0,  208,  208,  208,  138,   35,  208,  208,  208,
+
+      208,  208,  208,  182,  183,  184,  208,  208,  208,  208,
+      208,  172,  208,  208,  208,  208,  208,  208,  208,  208,
+      132,  208,  208,  208,  208,  208,   64,  208,  208,   65,
+      208,    0,    0,    0,    0,    0,  208,   66,   29,  145,
+      186,  187,  188,  208,  208,  208,  208,  208,  208,  208,
+      208,  208,  208,  208,  208,  140,  208,  208,  208,  208,
+      208,  208,  208,  208,  208,  133,  190,  191,  192,  208,
+      208,  152,  208,  141,    0,    0,    6,    0,    0,    0,
+       12,    3,   22,  208,  208,  208,  208,  208,  208,  208,
+      208,  208,  185,  146,   67,  208,  208,  208,  208,  169,
+
+      208,  177,  174,  207,   71,   72,   73,  208,  208,  208,
+      208,  208,  208,  208,  208,  208,  208,    0,    0,    0,
+        0,    0,    0,    0,  208,  208,  208,  189,  208,  208,
+      208,  208,  208,   82,   83,   84,  208,  208,  208,  208,
+      208,  208,  208,  208,  208,  208,  208,  208,  208,  193,
+       88,   89,   90,  208,    4,    0,    5,    0,    0,    0,
+        0,    0,    0,  208,  208,  208,  208,  208,  208,  208,
+      204,  208,  208,  208,  208,  208,  208,  208,  208,  208,
+      208,  208,   74,  208,  208,  208,  208,  208,  208,    0,
+        0,    0,    0,  208,  208,  205,  194,  208,  195,  208,
+
+      208,  208,   85,  208,  208,  208,  208,  208,  208,  208,
+      208,  208,  208,  208,  206,  208,  208,   91,    0,    0,
+        0,  196,  197,  208,  200,  208,  201,  208,  208,   70,
+      208,  208,  208,  164,  208,  165,  180,  208,  198,  199,
+      208,  208,    0,    0,    0,  208,  208,  208,  208,   75,
+      208,   76,  208,  208,  208,  208,  208,    0,    0,    0,
+        0,  208,  208,   86,   87,  208,   77,  208,  208,   78,
+      208,   92,   93,    0,    0,    0,    0,  208,  208,  208,
+      208,  208,  208,    0,    0,    0,    0,  208,  208,  208,
+      208,  208,   79,    0,    0,    0,    7,    0,    0,  202,
+
+      203,  208,  208,  208,    0,    0,    8,    0,    0,  208,
+      208,  166,    0,    0,    0,   80,   81,    0,    0,    0,
+        9,    0,    0,   10,    0,    0,    0,    0,    0,    0,
+        0,    0,    0,   11,    0
     } ;
 
 static yyconst flex_int32_t yy_ec[256] =
@@ -461,13 +472,13 @@ static yyconst flex_int32_t yy_ec[256] =
        10,   11,   12,    1,   13,   14,   15,   16,   17,   18,
        19,   20,   21,   21,   21,   22,   22,   23,    1,   24,
        25,   26,    1,    1,   27,   28,   29,   30,   31,   32,
-       33,   33,   33,   33,   33,   33,   33,   33,   33,   33,
-       33,   34,   35,   33,   36,   33,   33,   37,   33,   33,
-        1,    1,    1,   38,   39,    1,   40,   41,   42,   43,
+       33,   34,   34,   34,   34,   35,   34,   34,   34,   34,
+       34,   36,   37,   38,   39,   34,   34,   40,   34,   34,
+        1,    1,    1,   41,   42,    1,   43,   44,   45,   46,
 
-       44,   45,   46,   47,   48,   49,   50,   51,   52,   53,
-       54,   55,   33,   56,   57,   58,   59,   60,   61,   62,
-       63,   64,    1,   65,    1,    1,    1,    1,    1,    1,
+       47,   48,   49,   50,   51,   52,   53,   54,   55,   56,
+       57,   58,   34,   59,   60,   61,   62,   63,   64,   65,
+       66,   67,    1,   68,    1,    1,    1,    1,    1,    1,
         1,    1,    1,    1,    1,    1,    1,    1,    1,    1,
         1,    1,    1,    1,    1,    1,    1,    1,    1,    1,
         1,    1,    1,    1,    1,    1,    1,    1,    1,    1,
@@ -484,350 +495,357 @@ static yyconst flex_int32_t yy_ec[256] =
         1,    1,    1,    1,    1
     } ;
 
-static yyconst flex_int32_t yy_meta[66] =
+static yyconst flex_int32_t yy_meta[69] =
     {   0,
-        1,    2,    3,    1,    1,    1,    1,    1,    1,    1,
-        1,    1,    1,    1,    1,    4,    4,    4,    4,    4,
-        4,    5,    1,    1,    1,    1,    6,    6,    6,    6,
-        5,    5,    7,    7,    7,    7,    8,    1,    7,    6,
-        6,    6,    6,    5,    5,    7,    7,    7,    7,    7,
-        7,    7,    7,    7,    7,    7,    7,    7,    7,    7,
-        7,    8,    7,    7,    1
+        1,    1,    2,    1,    1,    1,    1,    1,    1,    1,
+        1,    1,    1,    1,    1,    3,    3,    3,    3,    3,
+        3,    3,    1,    1,    1,    1,    4,    4,    4,    4,
+        3,    3,    5,    5,    5,    5,    5,    5,    5,    5,
+        1,    5,    4,    4,    4,    4,    3,    3,    5,    5,
+        5,    5,    5,    5,    5,    5,    5,    5,    5,    5,
+        5,    5,    5,    5,    5,    5,    5,    1
     } ;
 
-static yyconst flex_int16_t yy_base[824] =
+static yyconst flex_int16_t yy_base[845] =
     {   0,
-        0,   64,   70,    0, 1185, 1184, 1186, 1189,   65, 1189,
-     1160, 1159,  128, 1158,  125,  126,  124, 1157,  140,  189,
-      123, 1156,  138,    0,  127,  124,  113,  135,  144,  161,
-      176, 1126,  130,  187,  140,  143,  161, 1120,  182,  174,
-      202,  199,  211,  212, 1131,  130,  263,  255, 1189,  189,
-     1189, 1162,  256, 1189,    0, 1189, 1189,  215, 1189, 1189,
-     1189, 1189, 1189, 1189, 1189, 1189, 1189,  248, 1189,  250,
-      112,  302,  319, 1189, 1189,    0,    0, 1189, 1151, 1189,
-     1189, 1189, 1150,    0, 1189, 1189, 1116, 1121, 1114, 1117,
-     1126, 1125, 1111, 1114, 1126,  144, 1120, 1107, 1104, 1118,
-
-     1104, 1101, 1101, 1107,  175,  191, 1101, 1112, 1097, 1103,
-     1107, 1108,    0, 1099, 1110,  247, 1109, 1104, 1084,  230,
-     1088, 1102, 1092,  241, 1085,  228, 1098, 1100, 1082, 1078,
-     1086, 1083, 1072, 1081,  173, 1079, 1085, 1080, 1083, 1071,
-     1074,  233,  240,  260, 1084, 1071, 1084,  239, 1077, 1189,
-     1189,  307,  301,  323, 1189, 1062, 1075, 1066, 1077,  249,
-        0,  368,    0,  379, 1189,  298,  390, 1189,  397,  404,
-      291, 1189, 1189, 1072,    0, 1063, 1067, 1077, 1074,  270,
-     1057, 1057, 1061,  291, 1072, 1069, 1069, 1067, 1064, 1055,
-     1062, 1048, 1046, 1059, 1044, 1061,    0, 1058, 1045, 1053,
-
-     1050, 1054, 1055, 1048, 1045, 1033, 1032, 1046, 1049, 1036,
-     1045, 1032, 1039, 1029,  335, 1035, 1038, 1028, 1036, 1024,
-     1028, 1019, 1034, 1024, 1015, 1034, 1017, 1015, 1026, 1015,
-     1010, 1008, 1022, 1007, 1009, 1006, 1018, 1017, 1020, 1001,
-      306, 1010, 1005, 1003, 1013,  991,  339, 1010, 1012, 1000,
-      992,  996, 1008,  991,    0,  411,  421,  438, 1189,  451,
-      458, 1189, 1189,  986,  997,    0,  994,  344,    0,    0,
-      987,  985,  987,  982,  991,  979,  997,  985,  350,    0,
-        0,  979,  990,  989,  989,    0,  973,  353,    0,    0,
-      975,  357,  983,  984,  974,  968,  967,  968,  967,  967,
-
-      361,  962,    0,    0,  958,  957,  956,  958,  959,  964,
-      958,  954,  968,  963,  962,  961,  952,  955,  955,  947,
-      950,  945,  954,  959,  944,  957,  947,    0,    0,  954,
-      950,    0,  941,  941,  947,  937,  945,  426,  942,    0,
-        0,    0,    0,  931,  944,  943,  942,  939,  927,  465,
-      475,  939,  941,    0,    0,    0,    0,  927,    0,  927,
-        0,  926,  927,  921,  932,    0,    0,    0,  922,    0,
-      918,    0,    0,    0,    0,    0,    0,    0,    0,    0,
-      929,  481,  928,    0,    0,  926,  922,  918,    0,    0,
-        0,  910,  443,  486,  493,  915,  911,  917,  907,  905,
-
-      919,  903,  903,  917,  905,  917,  912,    0,  910,  907,
-      911,  894,  896,  903,  909,  904,  903,  890,    0,  892,
-      893,    0,    0,    0,    0,  890,  894,    0,  888,  938,
-      887,  890,    0,  878,  888,    0,  876,  876,  890,    0,
-      892,    0,  497,  901,  900,  899,  869,  868,    0,  886,
-      885,  880,    0,    0,    0,    0,    0,    0,    0,    0,
-        0,    0,  868,  882,  868,  865,    0,    0,  871,  870,
-        0,    0,  868,  860,    0,    0,    0,    0,    0,    0,
-      857,  869,  500,  861,  868,  867,  864,  858,  851,  519,
-      867,  852,  847,  861,  859,    0,    0,  851,  871,  870,
-
-      869,  839,  838,  495,  496,    0,  851,  854,  852,  840,
-      836,    0,  849,  846,  845,  834,  833,  832,  515,  841,
-        0,  854,  853,  852,  822,  821,    0,  836,  822,    0,
-      833,  828,  543,  545,  873,  821,  829,    0,    0,    0,
-      845,  844,    0,  825,  828,  812,  820,  810,  818,  819,
-      819,  818,  803,  539,  816,    0,  817,  805,  804,  800,
-      825,  824,  823,  793,  792,    0,  823,  822,    0,  803,
-      806,    0,  552,    0,  792,  557, 1189,  580,    0,  590,
-      499, 1189,    0,  789,  788,  798,  798,  785,  800,  783,
-      798,  793,    0,    0,    0,  806,  805,  804,  774,    0,
-
-      774,    0,    0,    0,  559,  568,  797,  785,  788,  772,
-      771,  781,  781,  794,  793,  792,  762,  767,  552,  613,
-      363,  775,  763,  761,  760,  771,    0,  774,  770,  772,
-      768,  754,  782,  781,    0,  766,  758,  749,  757,  747,
-      758,  754,  756,  754,  754,  741,  740,  751,    0,  767,
-      766,    0,  751, 1189,  391, 1189,  620,    0,  640,  750,
-      732,  749,  748,  731,  723,  731,  721,  729,    0,  726,
-      725,  736,  719,  722,  737,  720,  733,  734,  731,  728,
-      736,  730,  729,  712,  711,  710,  721,  402,  705,  715,
-      699,  698,    0,  725,  698,  723,  696,  700,  699,    0,
-
-      710,  713,  709,  711,  688,  702,  686,  680,  688,  671,
-      662,    0,  640,  639,    0,  648,  641,    0,    0,  645,
-        0,  644,    0,  650,  649,    0,  625,  633,  623,  650,
-      630,    0,    0,  643,    0,    0,  642,  641,  582,  632,
-      639,  638,  614,  613,  635,  608,  633,  607,  590,  607,
-      586,  585,  611,  380,  526,  545,  544,    0,    0,  538,
-        0,  504,  510,    0,  495,    0,    0,  550,  572,  483,
-      470,  454,  462,  449,  445,  404,  419,  616,  619,  391,
-      387,  404,  392,  290,    0,  617,  642, 1189,  644,  570,
-        0,    0,  263,  258,  139,  645, 1189,  643,  618,  107,
-
-       77,    0,   23,  662,    0,    0,  663,  664, 1189,  665,
-     1189, 1189,  697,  702,  707,  712,  714,  716,  722,  729,
-      734,  739,  744
+        0,   67,   73,    0, 1210, 1209, 1211, 1214,   68, 1214,
+     1185, 1184,  134, 1183,  131,  132,  130, 1182,  146,  198,
+      129, 1181,  144,    0,  130,  113,  124,  141,  150,  126,
+      181, 1148,  159,  192,  118,  129,  146, 1142,  147,  174,
+      206,  192,  203,  222, 1153,  203,  221,  231, 1214,  260,
+     1214, 1187,  279, 1214,    0, 1214, 1214,  270, 1214, 1214,
+     1214, 1214, 1214, 1214, 1214, 1214, 1214,  244, 1214,  255,
+      139,  290,  307, 1214, 1214,    0,    0, 1214, 1176, 1214,
+     1214, 1214, 1175,    0, 1214, 1214, 1138, 1143, 1136, 1139,
+     1148, 1147, 1133, 1136, 1148,  144, 1142, 1129, 1126, 1140,
+
+     1126, 1123, 1123, 1129,  219,  193, 1123, 1134, 1119, 1125,
+     1129, 1130,    0, 1121, 1132,  278, 1131, 1126, 1106,  224,
+     1110, 1124, 1114,  232, 1107,  271, 1120, 1122, 1104, 1100,
+     1108, 1105, 1094, 1103,  234, 1101, 1107, 1102, 1105, 1093,
+     1096,  226,  145,  262, 1106, 1093, 1106,  263, 1099, 1214,
+     1214,  338,  331,  343, 1214, 1084, 1097, 1088, 1099,  345,
+        0,  334,    0,  345, 1214,  328,  391, 1214,  352,  398,
+      338, 1214, 1214, 1094,    0, 1085, 1089, 1099, 1096,  332,
+     1079, 1079, 1083,  320, 1094, 1091, 1091, 1089, 1086, 1077,
+     1084, 1070, 1068, 1081, 1066, 1083,    0, 1080, 1067, 1075,
+
+     1072, 1076, 1077, 1070, 1067, 1055, 1054, 1068, 1071, 1058,
+     1067, 1054, 1061, 1051,  364, 1057, 1060, 1050, 1058, 1046,
+     1050, 1041, 1056, 1046, 1037, 1056, 1039, 1037, 1048, 1037,
+     1032, 1030, 1044, 1029, 1031, 1028, 1040, 1039, 1042, 1023,
+      338, 1032, 1027, 1025, 1035, 1013,  403, 1032, 1034, 1022,
+     1014, 1018, 1030, 1013,    0,  415,  422,  439, 1214,  446,
+      455, 1214, 1214, 1008, 1019,    0, 1016,  406,    0,    0,
+     1009, 1007, 1009, 1004, 1013, 1001, 1019, 1007,  409,    0,
+        0, 1001, 1012, 1011, 1011,    0,  995,  429,    0,    0,
+      997,  460, 1005, 1006,  996,  990,  989,  990,  989,  989,
+
+      463,  984,    0,    0,  980,  979,  978,  980,  981,  986,
+      980,  976,  990,  985,  984,  983,  974,  977,  977,  969,
+      972,  967,  976,  981,  966,  979,  969,    0,    0,  976,
+      972,    0,  963,  963,  969,  959,  967,  466,  964,    0,
+        0,    0,    0,  953,  966,  965,  964,  961,  949,  472,
+      479,  961,  963,    0,    0,    0,    0,  949,    0,  949,
+        0,  948,  949,  943,  954,    0,    0,    0,  944,    0,
+      940,    0,    0,    0,    0,    0,    0,    0,    0,    0,
+      951,  487,  950,    0,    0,  948,  944,  940,    0,    0,
+        0,  932,  489,  494,  499,  937,  933,  939,  929,  927,
+
+      941,  925,  925,  939,  927,  939,  934,    0,  932,  929,
+      933,  916,  918,  925,  931,  926,  925,  912,    0,  914,
+      915,    0,    0,    0,    0,  912,  916,    0,  910,  963,
+      909,  912,    0,  900,  910,    0,  898,  898,  912,    0,
+      914,    0,  503,  926,  925,  924,  891,  890,    0,  908,
+      907,  902,    0,    0,    0,    0,    0,    0,    0,    0,
+        0,    0,  890,  904,  890,  887,    0,    0,  893,  892,
+        0,    0,  890,  882,    0,    0,    0,    0,    0,    0,
+      879,  891,  506,  883,  890,  889,  886,  880,  873,  524,
+      889,  874,  869,  883,  881,    0,    0,  873,  896,  895,
+
+      894,  861,  860,  361,  365,    0,  873,  876,  874,  862,
+      858,    0,  871,  868,  867,  856,  855,  854,  519,  863,
+        0,  879,  878,  877,  844,  843,    0,  858,  844,    0,
+      855,  850,  547,  553,  898,  843,  851,    0,    0,    0,
+      870,  869,    0,  847,  850,  834,  842,  832,  840,  841,
+      841,  840,  825,  559,  838,    0,  839,  827,  826,  822,
+      850,  849,  848,  815,  814,    0,  848,  847,    0,  825,
+      828,    0,  562,    0,  814,  580, 1214,  587,    0,  607,
+      584, 1214,    0,  811,  810,  820,  820,  807,  822,  805,
+      820,  815,    0,    0,    0,  831,  830,  829,  796,    0,
+
+      796,    0,    0,    0,  502,  524,  820,  807,  810,  794,
+      793,  803,  803,  819,  818,  817,  784,  789,  615,  640,
+      550,  806,  796,  784,  782,  781,  792,    0,  795,  791,
+      793,  789,  775,  806,  805,    0,  787,  779,  770,  778,
+      768,  779,  775,  777,  775,  775,  762,  761,  772,    0,
+      791,  790,    0,  772, 1214,  555, 1214,  647,    0,  667,
+      785,  770,  752,  769,  768,  751,  743,  751,  741,  749,
+        0,  746,  745,  756,  739,  742,  757,  740,  753,  754,
+      751,  748,  757,  750,  749,  732,  731,  730,  741,  582,
+      754,  724,  734,  718,  717,    0,  745,  717,  743,  715,
+
+      719,  718,    0,  729,  732,  728,  730,  711,  725,  709,
+      710,  718,  701,  700,    0,  706,  705,    0,  728,  713,
+      706,    0,    0,  710,    0,  709,    0,  715,  714,    0,
+      690,  698,  688,  716,  695,    0,    0,  708,    0,    0,
+      707,  706,  746,  611,  696,  703,  702,  678,  677,  705,
+      677,  703,  689,  674,  691,  670,  669,  190,  613,  557,
+      667,  687,  686,    0,    0,  681,    0,  680,  686,    0,
+      671,    0,    0,  671,  590,  343,  672,  645,  644,  654,
+      635,  631,  612,  612,  604,  443,  635,  576,  575,  549,
+       25,   87,    0,  183,  500,  552, 1214,  636,  591,    0,
+
+        0,  196,  258,  254,  272,  609, 1214,  614,  598,  279,
+      284,    0,  336,  348,  671,    0,    0,  362,  672,  688,
+     1214,  394,  689, 1214,  408,  670,  691,  649,  651,  474,
+      476,  693,  694, 1214, 1214,  704,  707,  710,  530,  591,
+      713,  717,  720,  722
     } ;
 
-static yyconst flex_int16_t yy_def[824] =
+static yyconst flex_int16_t yy_def[845] =
     {   0,
-      812,    1,  812,    3,  813,  813,  812,  812,  812,  812,
-      812,  812,  812,  812,  812,  812,  812,  812,  812,  812,
-      812,  812,  812,  814,  812,  814,  814,  814,  814,  814,
-      814,  814,  814,  814,  814,  814,  814,  814,  814,  814,
-      814,  814,  814,  814,  814,  812,  812,  812,  812,  812,
-      812,  812,  812,  812,  815,  812,  812,  812,  812,  812,
-      812,  812,  812,  812,  812,  812,  812,  816,  812,  817,
-       19,  812,  812,  812,  812,  818,   20,  812,  812,  812,
-      812,  812,  812,  814,  812,  812,  814,  814,  814,  814,
-      814,  814,  814,  814,  814,  814,  814,  814,  814,  814,
-
-      814,  814,  814,  814,  814,  814,  814,  814,  814,  814,
-      814,  814,  814,  814,  814,  814,  814,  814,  814,  814,
-      814,  814,  814,  814,  814,  814,  814,  814,  814,  814,
-      814,  814,  814,  814,  814,  814,  814,  814,  814,  814,
-      814,  814,  814,  814,  814,  814,  814,  814,  814,  812,
-      812,  812,  812,  812,  812,  812,  812,  812,  812,  812,
-      819,  812,  815,  812,  812,  817,  812,  812,  812,  812,
-      818,  812,  812,  814,  814,  814,  814,  814,  814,  814,
-      814,  814,  814,  814,  814,  814,  814,  814,  814,  814,
-      814,  814,  814,  814,  814,  814,  814,  814,  814,  814,
-
-      814,  814,  814,  814,  814,  814,  814,  814,  814,  814,
-      814,  814,  814,  814,  814,  814,  814,  814,  814,  814,
-      814,  814,  814,  814,  814,  814,  814,  814,  814,  814,
-      814,  814,  814,  814,  814,  814,  814,  814,  814,  814,
-      814,  814,  814,  814,  814,  814,  814,  814,  814,  814,
-      812,  812,  812,  812,  819,  812,  812,  812,  812,  812,
-      812,  812,  812,  814,  814,  814,  814,  814,  814,  814,
-      814,  814,  814,  814,  814,  814,  814,  814,  814,  814,
-      814,  814,  814,  814,  814,  814,  814,  814,  814,  814,
-      814,  814,  814,  814,  814,  814,  814,  814,  814,  814,
-
-      814,  814,  814,  814,  814,  814,  814,  814,  814,  814,
-      814,  814,  814,  814,  814,  814,  814,  814,  814,  814,
-      814,  814,  814,  814,  814,  814,  814,  814,  814,  814,
-      814,  814,  814,  814,  814,  814,  814,  814,  814,  814,
-      814,  814,  814,  814,  814,  812,  812,  812,  812,  812,
-      812,  814,  814,  814,  814,  814,  814,  814,  814,  814,
-      814,  814,  814,  814,  814,  814,  814,  814,  814,  814,
-      814,  814,  814,  814,  814,  814,  814,  814,  814,  814,
-      814,  814,  814,  814,  814,  814,  814,  814,  814,  814,
-      814,  814,  814,  814,  814,  814,  814,  814,  814,  814,
-
-      814,  814,  814,  814,  814,  814,  814,  814,  814,  814,
-      814,  814,  814,  814,  814,  814,  814,  814,  814,  814,
-      814,  814,  814,  814,  814,  814,  814,  814,  812,  812,
-      812,  812,  814,  814,  814,  814,  814,  814,  814,  814,
-      814,  814,  814,  814,  814,  814,  814,  814,  814,  814,
-      814,  814,  814,  814,  814,  814,  814,  814,  814,  814,
-      814,  814,  814,  814,  814,  814,  814,  814,  814,  814,
-      814,  814,  814,  814,  814,  814,  814,  814,  814,  814,
-      814,  814,  814,  814,  814,  814,  814,  814,  812,  812,
-      812,  812,  814,  814,  814,  814,  814,  814,  814,  814,
-
-      814,  814,  814,  814,  814,  814,  814,  814,  814,  814,
-      814,  814,  814,  814,  814,  814,  814,  814,  814,  814,
-      814,  814,  814,  814,  814,  814,  814,  814,  814,  814,
-      814,  812,  820,  812,  812,  812,  814,  814,  814,  814,
-      814,  814,  814,  814,  814,  814,  814,  814,  814,  814,
-      814,  814,  814,  814,  814,  814,  814,  814,  814,  814,
-      814,  814,  814,  814,  814,  814,  814,  814,  814,  814,
-      814,  814,  814,  814,  812,  812,  812,  812,  821,  812,
-      812,  812,  814,  814,  814,  814,  814,  814,  814,  814,
-      814,  814,  814,  814,  814,  814,  814,  814,  814,  814,
-
-      814,  814,  814,  814,  814,  814,  814,  814,  814,  814,
-      814,  814,  814,  814,  814,  814,  814,  812,  822,  812,
-      821,  812,  812,  814,  814,  814,  814,  814,  814,  814,
-      814,  814,  814,  814,  814,  814,  814,  814,  814,  814,
-      814,  814,  814,  814,  814,  814,  814,  814,  814,  814,
-      814,  814,  814,  812,  812,  812,  812,  823,  812,  812,
-      812,  814,  814,  814,  814,  814,  814,  814,  814,  814,
-      814,  814,  814,  814,  814,  814,  814,  814,  814,  814,
-      814,  814,  814,  814,  814,  814,  814,  823,  812,  812,
-      814,  814,  814,  814,  814,  814,  814,  814,  814,  814,
-
-      814,  814,  814,  814,  814,  814,  814,  814,  814,  814,
-      814,  814,  814,  814,  814,  812,  812,  814,  814,  814,
-      814,  814,  814,  814,  814,  814,  814,  814,  814,  814,
-      814,  814,  814,  814,  814,  814,  814,  814,  812,  812,
-      814,  814,  814,  814,  814,  814,  814,  814,  814,  814,
-      814,  814,  812,  812,  812,  814,  814,  814,  814,  814,
-      814,  814,  814,  814,  814,  814,  814,  812,  812,  812,
-      814,  814,  814,  814,  814,  814,  812,  812,  812,  814,
-      814,  814,  814,  814,  814,  812,  812,  812,  812,  812,
-      814,  814,  814,  814,  814,  812,  812,  812,  812,  814,
-
-      814,  814,  812,  812,  814,  814,  812,  812,  812,  812,
-      812,    0,  812,  812,  812,  812,  812,  812,  812,  812,
-      812,  812,  812
+      835,    1,  835,    3,  836,  836,  835,  835,  835,  835,
+      835,  835,  835,  835,  835,  835,  835,  835,  835,  835,
+      835,  835,  835,  837,  835,  837,  837,  837,  837,  837,
+      837,  837,  837,  837,  837,  837,  837,  837,  837,  837,
+      837,  837,  837,  837,  837,  835,  835,  835,  835,  835,
+      835,  835,  835,  835,  838,  835,  835,  835,  835,  835,
+      835,  835,  835,  835,  835,  835,  835,  839,  835,  840,
+       19,  835,  835,  835,  835,  841,   20,  835,  835,  835,
+      835,  835,  835,  837,  835,  835,  837,  837,  837,  837,
+      837,  837,  837,  837,  837,  837,  837,  837,  837,  837,
+
+      837,  837,  837,  837,  837,  837,  837,  837,  837,  837,
+      837,  837,  837,  837,  837,  837,  837,  837,  837,  837,
+      837,  837,  837,  837,  837,  837,  837,  837,  837,  837,
+      837,  837,  837,  837,  837,  837,  837,  837,  837,  837,
+      837,  837,  837,  837,  837,  837,  837,  837,  837,  835,
+      835,  835,  835,  835,  835,  835,  835,  835,  835,  835,
+      842,  835,  838,  835,  835,  840,  835,  835,  835,  835,
+      841,  835,  835,  837,  837,  837,  837,  837,  837,  837,
+      837,  837,  837,  837,  837,  837,  837,  837,  837,  837,
+      837,  837,  837,  837,  837,  837,  837,  837,  837,  837,
+
+      837,  837,  837,  837,  837,  837,  837,  837,  837,  837,
+      837,  837,  837,  837,  837,  837,  837,  837,  837,  837,
+      837,  837,  837,  837,  837,  837,  837,  837,  837,  837,
+      837,  837,  837,  837,  837,  837,  837,  837,  837,  837,
+      837,  837,  837,  837,  837,  837,  837,  837,  837,  837,
+      835,  835,  835,  835,  842,  835,  835,  835,  835,  835,
+      835,  835,  835,  837,  837,  837,  837,  837,  837,  837,
+      837,  837,  837,  837,  837,  837,  837,  837,  837,  837,
+      837,  837,  837,  837,  837,  837,  837,  837,  837,  837,
+      837,  837,  837,  837,  837,  837,  837,  837,  837,  837,
+
+      837,  837,  837,  837,  837,  837,  837,  837,  837,  837,
+      837,  837,  837,  837,  837,  837,  837,  837,  837,  837,
+      837,  837,  837,  837,  837,  837,  837,  837,  837,  837,
+      837,  837,  837,  837,  837,  837,  837,  837,  837,  837,
+      837,  837,  837,  837,  837,  835,  835,  835,  835,  835,
+      835,  837,  837,  837,  837,  837,  837,  837,  837,  837,
+      837,  837,  837,  837,  837,  837,  837,  837,  837,  837,
+      837,  837,  837,  837,  837,  837,  837,  837,  837,  837,
+      837,  837,  837,  837,  837,  837,  837,  837,  837,  837,
+      837,  837,  837,  837,  837,  837,  837,  837,  837,  837,
+
+      837,  837,  837,  837,  837,  837,  837,  837,  837,  837,
+      837,  837,  837,  837,  837,  837,  837,  837,  837,  837,
+      837,  837,  837,  837,  837,  837,  837,  837,  835,  835,
+      835,  835,  837,  837,  837,  837,  837,  837,  837,  837,
+      837,  837,  837,  837,  837,  837,  837,  837,  837,  837,
+      837,  837,  837,  837,  837,  837,  837,  837,  837,  837,
+      837,  837,  837,  837,  837,  837,  837,  837,  837,  837,
+      837,  837,  837,  837,  837,  837,  837,  837,  837,  837,
+      837,  837,  837,  837,  837,  837,  837,  837,  835,  835,
+      835,  835,  837,  837,  837,  837,  837,  837,  837,  837,
+
+      837,  837,  837,  837,  837,  837,  837,  837,  837,  837,
+      837,  837,  837,  837,  837,  837,  837,  837,  837,  837,
+      837,  837,  837,  837,  837,  837,  837,  837,  837,  837,
+      837,  835,  835,  835,  835,  835,  837,  837,  837,  837,
+      837,  837,  837,  837,  837,  837,  837,  837,  837,  837,
+      837,  837,  837,  837,  837,  837,  837,  837,  837,  837,
+      837,  837,  837,  837,  837,  837,  837,  837,  837,  837,
+      837,  837,  837,  837,  835,  835,  835,  835,  843,  835,
+      835,  835,  837,  837,  837,  837,  837,  837,  837,  837,
+      837,  837,  837,  837,  837,  837,  837,  837,  837,  837,
+
+      837,  837,  837,  837,  837,  837,  837,  837,  837,  837,
+      837,  837,  837,  837,  837,  837,  837,  835,  835,  835,
+      843,  835,  835,  835,  837,  837,  837,  837,  837,  837,
+      837,  837,  837,  837,  837,  837,  837,  837,  837,  837,
+      837,  837,  837,  837,  837,  837,  837,  837,  837,  837,
+      837,  837,  837,  837,  835,  835,  835,  835,  844,  835,
+      835,  835,  835,  837,  837,  837,  837,  837,  837,  837,
+      837,  837,  837,  837,  837,  837,  837,  837,  837,  837,
+      837,  837,  837,  837,  837,  837,  837,  837,  837,  844,
+      835,  835,  835,  837,  837,  837,  837,  837,  837,  837,
+
+      837,  837,  837,  837,  837,  837,  837,  837,  837,  837,
+      837,  837,  837,  837,  837,  837,  837,  837,  835,  835,
+      835,  837,  837,  837,  837,  837,  837,  837,  837,  837,
+      837,  837,  837,  837,  837,  837,  837,  837,  837,  837,
+      837,  837,  835,  835,  835,  837,  837,  837,  837,  837,
+      837,  837,  837,  837,  837,  837,  837,  835,  835,  835,
+      835,  837,  837,  837,  837,  837,  837,  837,  837,  837,
+      837,  837,  837,  835,  835,  835,  835,  837,  837,  837,
+      837,  837,  837,  835,  835,  835,  835,  837,  837,  837,
+      837,  837,  837,  835,  835,  835,  835,  835,  835,  837,
+
+      837,  837,  837,  837,  835,  835,  835,  835,  835,  837,
+      837,  837,  835,  835,  835,  837,  837,  835,  835,  835,
+      835,  835,  835,  835,  835,  835,  835,  835,  835,  835,
+      835,  835,  835,  835,    0,  835,  835,  835,  835,  835,
+      835,  835,  835,  835
     } ;
 
-static yyconst flex_int16_t yy_nxt[1255] =
+static yyconst flex_int16_t yy_nxt[1283] =
     {   0,
         8,    9,   10,    9,   11,    8,   12,   13,    8,    8,
        14,   15,   16,   17,   18,   19,   20,   20,   20,   20,
        20,   20,    8,   21,   22,   23,   24,   24,   24,   24,
-       24,   24,   24,   24,   24,   24,   24,   25,   24,   26,
-       27,   28,   29,   30,   31,   32,   33,   34,   24,   24,
-       35,   36,   37,   38,   39,   40,   41,   42,   43,   44,
-       45,   24,   24,   24,   46,   47,   58,  807,   58,   48,
-       49,   50,   51,   50,   49,   49,   49,   49,   49,   49,
-       49,   49,   49,   49,   52,   49,   53,   53,   53,   53,
-       53,   53,   54,   49,   49,   49,   55,   55,   55,   55,
-
-       55,   55,   55,   55,   55,   55,   55,   49,   55,   55,
+       24,   24,   24,   24,   24,   24,   24,   24,   24,   24,
+       25,   24,   26,   27,   28,   29,   30,   31,   32,   33,
+       34,   24,   24,   35,   36,   37,   38,   39,   40,   41,
+       42,   43,   44,   45,   24,   24,   24,   46,   47,   58,
+      803,   58,   48,   49,   50,   51,   50,   49,   49,   49,
+       49,   49,   49,   49,   49,   49,   49,   52,   49,   53,
+       53,   53,   53,   53,   53,   54,   49,   49,   49,   55,
+
        55,   55,   55,   55,   55,   55,   55,   55,   55,   55,
+       55,   55,   55,   49,   55,   55,   55,   55,   55,   55,
        55,   55,   55,   55,   55,   55,   55,   55,   55,   55,
-       55,   55,   55,   55,   49,   61,   64,  806,   66,   68,
-       68,   68,   68,   68,   68,   68,   79,   80,  812,   65,
-       67,   85,   62,   70,  150,   71,   71,   71,   71,   71,
-       71,   72,   82,   83,   86,   87,   90,  805,   91,  110,
-       73,   74,   92,  812,   93,   75,   76,  111,   94,  119,
-       88,   89,  121,   73,   74,   95,  122,   97,   96,  112,
-      160,   98,  160,  120,  151,  183,  184,   99,   75,  802,
-
-      123,   76,   70,  100,   77,   77,   77,   77,   77,   77,
-       77,  101,  231,  102,  124,  104,   58,  129,   58,   73,
-       74,  126,  103,  105,   78,  193,  106,  130,  232,  107,
-      195,  113,   73,   74,  114,  108,  194,  127,  115,  116,
-      128,  131,  138,  117,  196,  139,  118,   78,  132,  133,
-      160,  146,  160,  134,  140,  147,  154,  155,  142,  135,
-      136,  141,  137,  143,  152,  148,   58,  144,  153,  221,
-      145,  162,  162,  162,  162,  162,  162,  162,  164,  165,
-      167,  168,  213,  222,  239,  240,  248,  241,  218,  249,
-      214,  164,  165,  167,  168,  219,  242,  205,  156,  243,
-
-      206,  207,  154,  155,  208,  157,  209,  244,  152,  158,
-       58,  801,  153,  269,  159,   70,  800,   72,   72,   72,
-       72,   72,   72,   72,  154,  155,  263,  270,  258,  259,
-      169,  169,   73,   74,  170,  170,  170,  170,  170,  170,
-      170,  258,  259,  795,  156,   73,   74,  274,  275,  263,
-      333,  157,  305,  306,  307,  158,  340,  341,  342,  334,
-      159,  355,  356,  357,  576,  577,  156,  366,  367,  368,
-      374,  375,  376,  157,  378,  379,  380,  158,  389,  390,
-      391,  768,  159,  162,  162,  162,  162,  162,  162,  162,
-      256,  256,  655,  656,  257,  257,  257,  257,  257,  257,
-
-      257,  260,  260,  655,  656,  261,  261,  261,  261,  261,
-      261,  261,  170,  170,  170,  170,  170,  170,  170,  170,
-      170,  170,  170,  170,  170,  170,  257,  257,  257,  257,
-      257,  257,  257,  769,  794,  262,  257,  257,  257,  257,
-      257,  257,  257,  423,  424,  425,  793,  792,  262,  350,
-      350,  791,  165,  351,  351,  351,  351,  351,  351,  351,
-      454,  455,  456,  786,  785,  165,  261,  261,  261,  261,
-      261,  261,  261,  261,  261,  261,  261,  261,  261,  261,
-      351,  351,  351,  351,  351,  351,  351,  784,  783,  168,
-      351,  351,  351,  351,  351,  351,  351,  444,  445,  446,
-
-      581,  782,  168,  457,  458,  459,  259,  781,  447,  448,
-      460,  461,  462,  499,  500,  501,  522,  523,  524,  259,
-      490,  546,  548,  780,  502,  503,  779,  525,  526,  547,
-      549,  561,  562,  563,  533,  534,  534,  534,  534,  534,
-      534,  622,  564,  565,  576,  577,  576,  577,  776,  775,
-      774,  768,  623,  655,  656,  596,  597,  598,  576,  577,
-      580,  580,  580,  580,  580,  580,  580,  599,  614,  615,
-      616,  798,  619,  620,  620,  620,  620,  620,  620,  579,
-      617,  576,  577,  753,  773,  638,  772,  771,  658,  770,
-      754,  576,  577,  639,  640,  578,  578,  578,  578,  578,
-
-      578,  641,  642,  769,  579,  580,  580,  580,  580,  580,
-      580,  580,  753,  658,  655,  656,  777,  787,  796,  754,
-      789,  655,  656,  799,  778,  788,  797,  790,  659,  659,
-      659,  659,  659,  659,  659,  657,  657,  657,  657,  657,
-      657,  655,  656,  787,  798,  789,  796,  767,  766,  765,
-      764,  788,  790,  763,  797,  659,  659,  659,  659,  659,
-      659,  659,  803,  808,  810,  808,  810,  762,  761,  760,
-      804,  809,  811,  809,  811,  759,  758,  757,  756,  755,
-      752,  751,  750,  749,  748,  747,  746,  745,  744,  743,
-      742,  741,  740,  739,  738,  737,  799,   56,   56,   56,
-
-       56,   56,   56,   56,   56,   84,   84,   84,   84,   84,
-      163,  163,  163,  163,  163,   68,   68,  166,  166,  171,
-      171,  171,  255,  255,  736,  255,  255,  255,  255,  255,
-      578,  578,  578,  735,  734,  733,  578,  621,  621,  621,
-      657,  657,  657,  732,  731,  730,  657,  688,  688,  688,
-      729,  728,  727,  726,  725,  724,  723,  722,  721,  720,
-      719,  718,  717,  716,  715,  714,  713,  712,  711,  710,
-      709,  708,  707,  706,  705,  704,  703,  702,  701,  700,
-      699,  698,  697,  696,  695,  694,  693,  692,  691,  690,
-      689,  687,  686,  685,  684,  683,  682,  681,  680,  679,
-
-      678,  677,  676,  675,  674,  673,  672,  671,  670,  669,
-      668,  667,  666,  665,  664,  663,  662,  661,  660,  654,
-      653,  652,  651,  650,  649,  648,  647,  646,  645,  644,
-      643,  637,  636,  635,  634,  633,  632,  631,  630,  629,
-      628,  627,  626,  625,  624,  618,  613,  612,  611,  610,
-      609,  608,  607,  606,  605,  604,  603,  602,  601,  600,
-      595,  594,  593,  592,  591,  590,  589,  588,  587,  586,
-      585,  584,  583,  582,  581,  575,  574,  573,  572,  571,
-      570,  569,  568,  567,  566,  560,  559,  558,  557,  556,
-      555,  554,  553,  552,  551,  550,  545,  544,  543,  542,
-
-      541,  540,  539,  538,  537,  536,  535,  532,  531,  530,
-      529,  528,  527,  521,  520,  519,  518,  517,  516,  515,
-      514,  513,  512,  511,  510,  509,  508,  507,  506,  505,
-      504,  498,  497,  496,  495,  494,  493,  492,  491,  490,
-      489,  488,  487,  486,  485,  484,  483,  482,  481,  480,
-      479,  478,  477,  476,  475,  474,  473,  472,  471,  470,
-      469,  468,  467,  466,  465,  464,  463,  453,  452,  451,
-      450,  449,  443,  442,  441,  440,  439,  438,  437,  436,
-      435,  434,  433,  432,  431,  430,  429,  428,  427,  426,
-      422,  421,  420,  419,  418,  417,  416,  415,  414,  413,
-
-      412,  411,  410,  409,  408,  407,  406,  405,  404,  403,
-      402,  401,  400,  399,  398,  397,  396,  395,  394,  393,
-      392,  388,  387,  386,  385,  384,  383,  382,  381,  377,
-      373,  372,  371,  370,  369,  365,  364,  363,  362,  361,
-      360,  359,  358,  354,  353,  352,  349,  348,  347,  346,
-      345,  344,  343,  339,  338,  337,  336,  335,  332,  331,
-      330,  329,  328,  327,  326,  325,  324,  323,  322,  321,
-      320,  319,  318,  317,  316,  315,  314,  313,  312,  311,
-      310,  309,  308,  304,  303,  302,  301,  300,  299,  298,
-      297,  296,  295,  294,  293,  292,  291,  290,  289,  288,
-
-      287,  286,  285,  284,  283,  282,  281,  280,  279,  278,
-      277,  276,  273,  272,  271,  268,  267,  266,  265,  264,
-      254,  253,  252,  251,  250,  247,  246,  245,  238,  237,
-      236,  235,  234,  233,  230,  229,  228,  227,  226,  225,
-      224,  223,  220,  217,  216,  215,  212,  211,  210,  204,
-      203,  202,  201,  200,  199,  198,  197,  192,  191,  190,
-      189,  188,  187,  186,  185,  182,  181,  180,  179,  178,
-      177,  176,  175,  174,  173,  172,  161,  149,  125,  109,
-       81,   69,   63,   60,   59,  812,   57,   57,    7,  812,
-      812,  812,  812,  812,  812,  812,  812,  812,  812,  812,
-
-      812,  812,  812,  812,  812,  812,  812,  812,  812,  812,
-      812,  812,  812,  812,  812,  812,  812,  812,  812,  812,
-      812,  812,  812,  812,  812,  812,  812,  812,  812,  812,
-      812,  812,  812,  812,  812,  812,  812,  812,  812,  812,
-      812,  812,  812,  812,  812,  812,  812,  812,  812,  812,
-      812,  812,  812,  812
+       55,   55,   55,   55,   55,   55,   55,   55,   55,   55,
+       49,   61,   64,  804,   66,   68,   68,   68,   68,   68,
+       68,   68,   79,   80,   85,   65,   67,   87,   62,   70,
+      119,   71,   71,   71,   71,   71,   71,   72,   82,   83,
+       86,  121,   88,   89,  120,  122,   73,   74,  835,  101,
+       90,  102,   91,   93,   75,   76,   92,   94,  123,  126,
+      103,  758,   73,   74,   95,  241,   97,   96,  183,  184,
+
+       98,  110,  124,  835,  242,  127,   99,   75,  128,  111,
+       76,   70,  100,   77,   77,   77,   77,   77,   77,   77,
+      129,  112,  152,  104,   58,  805,  153,  150,   73,   74,
+      130,  105,  154,  155,  106,  195,   78,  107,  138,  113,
+      774,  139,  114,  108,   73,   74,  115,  116,  131,  196,
+      140,  117,  810,  142,  118,  132,  133,  141,  143,   78,
+      134,  160,  144,  160,  146,  145,  135,  136,  147,  137,
+      151,   58,  193,   58,  164,  165,  231,  156,  148,  213,
+      239,  240,  218,  194,  157,  167,  168,  214,  158,  219,
+      164,  165,  232,  159,  162,  162,  162,  162,  162,  162,
+
+      162,  167,  168,   70,  243,   72,   72,   72,   72,   72,
+       72,   72,  244,  248,  811,  221,  249,  812,  169,  169,
+       73,   74,  170,  170,  170,  170,  170,  170,  170,  222,
+      813,  205,  154,  155,  206,  207,   73,   74,  208,  152,
+      209,   58,  816,  153,  154,  155,  160,  817,  160,  162,
+      162,  162,  162,  162,  162,  162,  256,  256,  258,  259,
+      257,  257,  257,  257,  257,  257,  257,  170,  170,  170,
+      170,  170,  170,  170,  258,  259,  263,  156,  269,  274,
+      275,  305,  306,  307,  157,  333,  818,  546,  158,  156,
+      785,  548,  270,  159,  334,  819,  157,  547,  786,  263,
+
+      158,  549,  260,  260,  822,  159,  261,  261,  261,  261,
+      261,  261,  261,  170,  170,  170,  170,  170,  170,  170,
+      340,  341,  342,  355,  356,  357,  366,  367,  368,  262,
+      257,  257,  257,  257,  257,  257,  257,  257,  257,  257,
+      257,  257,  257,  257,  796,  262,  374,  375,  376,  825,
+      350,  350,  797,  165,  351,  351,  351,  351,  351,  351,
+      351,  261,  261,  261,  261,  261,  261,  261,  826,  165,
+      261,  261,  261,  261,  261,  261,  261,  378,  379,  380,
+      389,  390,  391,  423,  424,  425,  168,  351,  351,  351,
+      351,  351,  351,  351,  351,  351,  351,  351,  351,  351,
+
+      351,  806,  168,  444,  445,  446,  454,  455,  456,  807,
+      259,  457,  458,  459,  447,  448,  460,  461,  462,  499,
+      500,  501,  522,  523,  524,  490,  259,  831,  639,  832,
+      502,  503,   68,  525,  526,  561,  562,  563,  640,  533,
+      534,  534,  534,  534,  534,  534,  564,  565,  576,  577,
+      641,  576,  577,  796,  576,  577,  656,  657,  775,  642,
+      643,  797,  578,  578,  578,  578,  578,  578,  580,  580,
+      580,  580,  580,  580,  580,  596,  597,  598,  614,  615,
+      616,  576,  577,  656,  657,  581,  579,  599,  576,  577,
+      617,  775,  808,  166,  802,  619,  620,  620,  620,  620,
+
+      620,  620,  578,  578,  578,  578,  578,  578,  576,  577,
+      806,  579,  759,  776,  759,  808,  656,  657,  807,  760,
+      622,  760,  580,  580,  580,  580,  580,  580,  580,  623,
+      658,  658,  658,  658,  658,  658,  798,  798,  801,  800,
+      624,  656,  657,  799,  799,  814,  776,  809,  656,  657,
+      829,  795,  829,  815,  659,  660,  660,  660,  660,  660,
+      660,  660,  658,  658,  658,  658,  658,  658,  656,  657,
+      809,  827,  820,  823,  794,  793,  792,  791,  828,  659,
+      821,  824,  660,  660,  660,  660,  660,  660,  660,  820,
+      823,  830,  827,  830,  833,  833,  790,  821,  824,  828,
+
+      789,  788,  834,  834,   56,   56,   56,   56,   56,   84,
+       84,   84,  163,  163,  163,  171,  171,  255,  787,  255,
+      255,  255,  621,  621,  690,  690,  784,  783,  782,  781,
+      780,  779,  778,  777,  773,  772,  771,  770,  769,  768,
+      767,  766,  765,  764,  763,  762,  761,  758,  757,  756,
+      755,  754,  753,  752,  751,  750,  749,  748,  747,  746,
+      745,  744,  743,  742,  741,  740,  739,  738,  737,  736,
+      735,  734,  733,  732,  731,  730,  729,  728,  727,  726,
+      725,  724,  723,  722,  721,  720,  719,  718,  717,  716,
+      715,  714,  713,  712,  711,  710,  709,  708,  707,  706,
+
+      705,  704,  703,  702,  701,  700,  699,  698,  697,  696,
+      695,  694,  693,  692,  691,  689,  688,  687,  686,  685,
+      684,  683,  682,  681,  680,  679,  678,  677,  676,  675,
+      674,  673,  672,  671,  670,  669,  668,  667,  666,  665,
+      664,  663,  662,  661,  655,  654,  653,  652,  651,  650,
+      649,  648,  647,  646,  645,  644,  638,  637,  636,  635,
+      634,  633,  632,  631,  630,  629,  628,  627,  626,  625,
+      618,  613,  612,  611,  610,  609,  608,  607,  606,  605,
+      604,  603,  602,  601,  600,  595,  594,  593,  592,  591,
+      590,  589,  588,  587,  586,  585,  584,  583,  582,  581,
+
+      575,  574,  573,  572,  571,  570,  569,  568,  567,  566,
+      560,  559,  558,  557,  556,  555,  554,  553,  552,  551,
+      550,  545,  544,  543,  542,  541,  540,  539,  538,  537,
+      536,  535,  532,  531,  530,  529,  528,  527,  521,  520,
+      519,  518,  517,  516,  515,  514,  513,  512,  511,  510,
+      509,  508,  507,  506,  505,  504,  498,  497,  496,  495,
+      494,  493,  492,  491,  490,  489,  488,  487,  486,  485,
+      484,  483,  482,  481,  480,  479,  478,  477,  476,  475,
+      474,  473,  472,  471,  470,  469,  468,  467,  466,  465,
+      464,  463,  453,  452,  451,  450,  449,  443,  442,  441,
+
+      440,  439,  438,  437,  436,  435,  434,  433,  432,  431,
+      430,  429,  428,  427,  426,  422,  421,  420,  419,  418,
+      417,  416,  415,  414,  413,  412,  411,  410,  409,  408,
+      407,  406,  405,  404,  403,  402,  401,  400,  399,  398,
+      397,  396,  395,  394,  393,  392,  388,  387,  386,  385,
+      384,  383,  382,  381,  377,  373,  372,  371,  370,  369,
+      365,  364,  363,  362,  361,  360,  359,  358,  354,  353,
+      352,  349,  348,  347,  346,  345,  344,  343,  339,  338,
+      337,  336,  335,  332,  331,  330,  329,  328,  327,  326,
+      325,  324,  323,  322,  321,  320,  319,  318,  317,  316,
+
+      315,  314,  313,  312,  311,  310,  309,  308,  304,  303,
+      302,  301,  300,  299,  298,  297,  296,  295,  294,  293,
+      292,  291,  290,  289,  288,  287,  286,  285,  284,  283,
+      282,  281,  280,  279,  278,  277,  276,  273,  272,  271,
+      268,  267,  266,  265,  264,  254,  253,  252,  251,  250,
+      247,  246,  245,  238,  237,  236,  235,  234,  233,  230,
+      229,  228,  227,  226,  225,  224,  223,  220,  217,  216,
+      215,  212,  211,  210,  204,  203,  202,  201,  200,  199,
+      198,  197,  192,  191,  190,  189,  188,  187,  186,  185,
+      182,  181,  180,  179,  178,  177,  176,  175,  174,  173,
+
+      172,  161,  149,  125,  109,   81,   69,   63,   60,   59,
+      835,   57,   57,    7,  835,  835,  835,  835,  835,  835,
+      835,  835,  835,  835,  835,  835,  835,  835,  835,  835,
+      835,  835,  835,  835,  835,  835,  835,  835,  835,  835,
+      835,  835,  835,  835,  835,  835,  835,  835,  835,  835,
+      835,  835,  835,  835,  835,  835,  835,  835,  835,  835,
+      835,  835,  835,  835,  835,  835,  835,  835,  835,  835,
+      835,  835,  835,  835,  835,  835,  835,  835,  835,  835,
+      835,  835
     } ;
 
-static yyconst flex_int16_t yy_chk[1255] =
+static yyconst flex_int16_t yy_chk[1283] =
     {   0,
         1,    1,    1,    1,    1,    1,    1,    1,    1,    1,
         1,    1,    1,    1,    1,    1,    1,    1,    1,    1,
@@ -835,138 +853,141 @@ static yyconst flex_int16_t yy_chk[1255] =
         1,    1,    1,    1,    1,    1,    1,    1,    1,    1,
         1,    1,    1,    1,    1,    1,    1,    1,    1,    1,
         1,    1,    1,    1,    1,    1,    1,    1,    1,    1,
-        1,    1,    1,    1,    1,    2,    9,  803,    9,    2,
-        3,    3,    3,    3,    3,    3,    3,    3,    3,    3,
+        1,    1,    1,    1,    1,    1,    1,    1,    2,    9,
+      791,    9,    2,    3,    3,    3,    3,    3,    3,    3,
         3,    3,    3,    3,    3,    3,    3,    3,    3,    3,
         3,    3,    3,    3,    3,    3,    3,    3,    3,    3,
 
         3,    3,    3,    3,    3,    3,    3,    3,    3,    3,
         3,    3,    3,    3,    3,    3,    3,    3,    3,    3,
         3,    3,    3,    3,    3,    3,    3,    3,    3,    3,
-        3,    3,    3,    3,    3,   13,   15,  801,   16,   17,
-       17,   17,   17,   17,   17,   17,   21,   21,   71,   15,
-       16,   25,   13,   19,   46,   19,   19,   19,   19,   19,
-       19,   19,   23,   23,   25,   26,   27,  800,   27,   33,
-       19,   19,   27,   71,   28,   19,   19,   33,   28,   35,
-       26,   26,   36,   19,   19,   28,   36,   29,   28,   33,
-       50,   29,   50,   35,   46,   96,   96,   29,   19,  795,
-
-       37,   19,   20,   29,   20,   20,   20,   20,   20,   20,
-       20,   30,  135,   30,   37,   31,   58,   40,   58,   20,
-       20,   39,   30,   31,   20,  105,   31,   40,  135,   31,
-      106,   34,   20,   20,   34,   31,  105,   39,   34,   34,
-       39,   41,   42,   34,  106,   42,   34,   20,   41,   41,
-      160,   44,  160,   41,   42,   44,   48,   48,   43,   41,
-       41,   42,   41,   43,   47,   44,   47,   43,   47,  126,
-       43,   53,   53,   53,   53,   53,   53,   53,   68,   68,
-       70,   70,  120,  126,  142,  142,  148,  143,  124,  148,
-      120,   68,   68,   70,   70,  124,  143,  116,   48,  144,
-
-      116,  116,  153,  153,  116,   48,  116,  144,  152,   48,
-      152,  794,  152,  180,   48,   72,  793,   72,   72,   72,
-       72,   72,   72,   72,  154,  154,  171,  180,  166,  166,
-       73,   73,   72,   72,   73,   73,   73,   73,   73,   73,
-       73,  166,  166,  784,  153,   72,   72,  184,  184,  171,
-      241,  153,  215,  215,  215,  153,  247,  247,  247,  241,
-      153,  268,  268,  268,  621,  621,  154,  279,  279,  279,
-      288,  288,  288,  154,  292,  292,  292,  154,  301,  301,
-      301,  754,  154,  162,  162,  162,  162,  162,  162,  162,
-      164,  164,  655,  655,  164,  164,  164,  164,  164,  164,
-
-      164,  167,  167,  688,  688,  167,  167,  167,  167,  167,
-      167,  167,  169,  169,  169,  169,  169,  169,  169,  170,
-      170,  170,  170,  170,  170,  170,  256,  256,  256,  256,
-      256,  256,  256,  754,  783,  170,  257,  257,  257,  257,
-      257,  257,  257,  338,  338,  338,  782,  781,  170,  258,
-      258,  780,  257,  258,  258,  258,  258,  258,  258,  258,
-      393,  393,  393,  777,  776,  257,  260,  260,  260,  260,
-      260,  260,  260,  261,  261,  261,  261,  261,  261,  261,
-      350,  350,  350,  350,  350,  350,  350,  775,  774,  261,
-      351,  351,  351,  351,  351,  351,  351,  382,  382,  382,
-
-      581,  773,  261,  394,  394,  394,  351,  772,  382,  382,
-      395,  395,  395,  443,  443,  443,  483,  483,  483,  351,
-      490,  504,  505,  771,  443,  443,  770,  483,  483,  504,
-      505,  519,  519,  519,  490,  490,  490,  490,  490,  490,
-      490,  581,  519,  519,  533,  533,  534,  534,  765,  763,
-      762,  768,  581,  619,  619,  554,  554,  554,  576,  576,
-      534,  534,  534,  534,  534,  534,  534,  554,  573,  573,
-      573,  790,  576,  576,  576,  576,  576,  576,  576,  533,
-      573,  578,  578,  739,  760,  605,  757,  756,  619,  755,
-      739,  580,  580,  605,  606,  578,  578,  578,  578,  578,
-
-      578,  606,  606,  768,  533,  580,  580,  580,  580,  580,
-      580,  580,  753,  619,  620,  620,  769,  778,  786,  753,
-      779,  657,  657,  790,  769,  778,  786,  779,  620,  620,
-      620,  620,  620,  620,  620,  657,  657,  657,  657,  657,
-      657,  659,  659,  787,  798,  789,  796,  752,  751,  750,
-      749,  787,  789,  748,  796,  659,  659,  659,  659,  659,
-      659,  659,  799,  804,  807,  808,  810,  747,  746,  745,
-      799,  804,  807,  808,  810,  744,  743,  742,  741,  740,
-      738,  737,  734,  731,  730,  729,  728,  727,  725,  724,
-      722,  720,  717,  716,  714,  713,  798,  813,  813,  813,
-
-      813,  813,  813,  813,  813,  814,  814,  814,  814,  814,
-      815,  815,  815,  815,  815,  816,  816,  817,  817,  818,
-      818,  818,  819,  819,  711,  819,  819,  819,  819,  819,
-      820,  820,  820,  710,  709,  708,  820,  821,  821,  821,
-      822,  822,  822,  707,  706,  705,  822,  823,  823,  823,
-      704,  703,  702,  701,  699,  698,  697,  696,  695,  694,
-      692,  691,  690,  689,  687,  686,  685,  684,  683,  682,
-      681,  680,  679,  678,  677,  676,  675,  674,  673,  672,
-      671,  670,  668,  667,  666,  665,  664,  663,  662,  661,
-      660,  653,  651,  650,  648,  647,  646,  645,  644,  643,
-
-      642,  641,  640,  639,  638,  637,  636,  634,  633,  632,
-      631,  630,  629,  628,  626,  625,  624,  623,  622,  618,
-      617,  616,  615,  614,  613,  612,  611,  610,  609,  608,
-      607,  601,  599,  598,  597,  596,  592,  591,  590,  589,
-      588,  587,  586,  585,  584,  575,  571,  570,  568,  567,
-      565,  564,  563,  562,  561,  560,  559,  558,  557,  555,
-      553,  552,  551,  550,  549,  548,  547,  546,  545,  544,
-      542,  541,  537,  536,  535,  532,  531,  529,  528,  526,
-      525,  524,  523,  522,  520,  518,  517,  516,  515,  514,
-      513,  511,  510,  509,  508,  507,  503,  502,  501,  500,
-
-      499,  498,  495,  494,  493,  492,  491,  489,  488,  487,
-      486,  485,  484,  482,  481,  474,  473,  470,  469,  466,
-      465,  464,  463,  452,  451,  450,  448,  447,  446,  445,
-      444,  441,  439,  438,  437,  435,  434,  432,  431,  430,
-      429,  427,  426,  421,  420,  418,  417,  416,  415,  414,
-      413,  412,  411,  410,  409,  407,  406,  405,  404,  403,
-      402,  401,  400,  399,  398,  397,  396,  392,  388,  387,
-      386,  383,  381,  371,  369,  365,  364,  363,  362,  360,
-      358,  353,  352,  349,  348,  347,  346,  345,  344,  339,
-      337,  336,  335,  334,  333,  331,  330,  327,  326,  325,
-
-      324,  323,  322,  321,  320,  319,  318,  317,  316,  315,
-      314,  313,  312,  311,  310,  309,  308,  307,  306,  305,
-      302,  300,  299,  298,  297,  296,  295,  294,  293,  291,
-      287,  285,  284,  283,  282,  278,  277,  276,  275,  274,
-      273,  272,  271,  267,  265,  264,  254,  253,  252,  251,
-      250,  249,  248,  246,  245,  244,  243,  242,  240,  239,
-      238,  237,  236,  235,  234,  233,  232,  231,  230,  229,
-      228,  227,  226,  225,  224,  223,  222,  221,  220,  219,
-      218,  217,  216,  214,  213,  212,  211,  210,  209,  208,
-      207,  206,  205,  204,  203,  202,  201,  200,  199,  198,
-
-      196,  195,  194,  193,  192,  191,  190,  189,  188,  187,
-      186,  185,  183,  182,  181,  179,  178,  177,  176,  174,
-      159,  158,  157,  156,  149,  147,  146,  145,  141,  140,
-      139,  138,  137,  136,  134,  133,  132,  131,  130,  129,
-      128,  127,  125,  123,  122,  121,  119,  118,  117,  115,
-      114,  112,  111,  110,  109,  108,  107,  104,  103,  102,
-      101,  100,   99,   98,   97,   95,   94,   93,   92,   91,
-       90,   89,   88,   87,   83,   79,   52,   45,   38,   32,
-       22,   18,   14,   12,   11,    7,    6,    5,  812,  812,
-      812,  812,  812,  812,  812,  812,  812,  812,  812,  812,
-
-      812,  812,  812,  812,  812,  812,  812,  812,  812,  812,
-      812,  812,  812,  812,  812,  812,  812,  812,  812,  812,
-      812,  812,  812,  812,  812,  812,  812,  812,  812,  812,
-      812,  812,  812,  812,  812,  812,  812,  812,  812,  812,
-      812,  812,  812,  812,  812,  812,  812,  812,  812,  812,
-      812,  812,  812,  812
+        3,    3,    3,    3,    3,    3,    3,    3,    3,    3,
+        3,   13,   15,  792,   16,   17,   17,   17,   17,   17,
+       17,   17,   21,   21,   25,   15,   16,   26,   13,   19,
+       35,   19,   19,   19,   19,   19,   19,   19,   23,   23,
+       25,   36,   26,   26,   35,   36,   19,   19,   71,   30,
+       27,   30,   27,   28,   19,   19,   27,   28,   37,   39,
+       30,  758,   19,   19,   28,  143,   29,   28,   96,   96,
+
+       29,   33,   37,   71,  143,   39,   29,   19,   39,   33,
+       19,   20,   29,   20,   20,   20,   20,   20,   20,   20,
+       40,   33,   47,   31,   47,  794,   47,   46,   20,   20,
+       40,   31,   48,   48,   31,  106,   20,   31,   42,   34,
+      758,   42,   34,   31,   20,   20,   34,   34,   41,  106,
+       42,   34,  802,   43,   34,   41,   41,   42,   43,   20,
+       41,   50,   43,   50,   44,   43,   41,   41,   44,   41,
+       46,   58,  105,   58,   68,   68,  135,   48,   44,  120,
+      142,  142,  124,  105,   48,   70,   70,  120,   48,  124,
+       68,   68,  135,   48,   53,   53,   53,   53,   53,   53,
+
+       53,   70,   70,   72,  144,   72,   72,   72,   72,   72,
+       72,   72,  144,  148,  803,  126,  148,  804,   73,   73,
+       72,   72,   73,   73,   73,   73,   73,   73,   73,  126,
+      805,  116,  153,  153,  116,  116,   72,   72,  116,  152,
+      116,  152,  810,  152,  154,  154,  160,  811,  160,  162,
+      162,  162,  162,  162,  162,  162,  164,  164,  166,  166,
+      164,  164,  164,  164,  164,  164,  164,  169,  169,  169,
+      169,  169,  169,  169,  166,  166,  171,  153,  180,  184,
+      184,  215,  215,  215,  153,  241,  813,  504,  153,  154,
+      776,  505,  180,  153,  241,  814,  154,  504,  776,  171,
+
+      154,  505,  167,  167,  818,  154,  167,  167,  167,  167,
+      167,  167,  167,  170,  170,  170,  170,  170,  170,  170,
+      247,  247,  247,  268,  268,  268,  279,  279,  279,  170,
+      256,  256,  256,  256,  256,  256,  256,  257,  257,  257,
+      257,  257,  257,  257,  786,  170,  288,  288,  288,  822,
+      258,  258,  786,  257,  258,  258,  258,  258,  258,  258,
+      258,  260,  260,  260,  260,  260,  260,  260,  825,  257,
+      261,  261,  261,  261,  261,  261,  261,  292,  292,  292,
+      301,  301,  301,  338,  338,  338,  261,  350,  350,  350,
+      350,  350,  350,  350,  351,  351,  351,  351,  351,  351,
+
+      351,  795,  261,  382,  382,  382,  393,  393,  393,  795,
+      351,  394,  394,  394,  382,  382,  395,  395,  395,  443,
+      443,  443,  483,  483,  483,  490,  351,  830,  605,  831,
+      443,  443,  839,  483,  483,  519,  519,  519,  605,  490,
+      490,  490,  490,  490,  490,  490,  519,  519,  533,  533,
+      606,  621,  621,  796,  534,  534,  656,  656,  760,  606,
+      606,  796,  533,  533,  533,  533,  533,  533,  534,  534,
+      534,  534,  534,  534,  534,  554,  554,  554,  573,  573,
+      573,  576,  576,  690,  690,  581,  533,  554,  578,  578,
+      573,  775,  799,  840,  790,  576,  576,  576,  576,  576,
+
+      576,  576,  578,  578,  578,  578,  578,  578,  580,  580,
+      806,  533,  744,  760,  759,  808,  619,  619,  806,  744,
+      581,  759,  580,  580,  580,  580,  580,  580,  580,  581,
+      619,  619,  619,  619,  619,  619,  787,  798,  789,  788,
+      581,  620,  620,  787,  798,  809,  775,  799,  658,  658,
+      828,  785,  829,  809,  619,  620,  620,  620,  620,  620,
+      620,  620,  658,  658,  658,  658,  658,  658,  660,  660,
+      808,  826,  815,  819,  784,  783,  782,  781,  826,  619,
+      815,  819,  660,  660,  660,  660,  660,  660,  660,  820,
+      823,  828,  827,  829,  832,  833,  780,  820,  823,  827,
+
+      779,  778,  832,  833,  836,  836,  836,  836,  836,  837,
+      837,  837,  838,  838,  838,  841,  841,  842,  777,  842,
+      842,  842,  843,  843,  844,  844,  774,  771,  769,  768,
+      766,  763,  762,  761,  757,  756,  755,  754,  753,  752,
+      751,  750,  749,  748,  747,  746,  745,  743,  742,  741,
+      738,  735,  734,  733,  732,  731,  729,  728,  726,  724,
+      721,  720,  719,  717,  716,  714,  713,  712,  711,  710,
+      709,  708,  707,  706,  705,  704,  702,  701,  700,  699,
+      698,  697,  695,  694,  693,  692,  691,  689,  688,  687,
+      686,  685,  684,  683,  682,  681,  680,  679,  678,  677,
+
+      676,  675,  674,  673,  672,  670,  669,  668,  667,  666,
+      665,  664,  663,  662,  661,  654,  652,  651,  649,  648,
+      647,  646,  645,  644,  643,  642,  641,  640,  639,  638,
+      637,  635,  634,  633,  632,  631,  630,  629,  627,  626,
+      625,  624,  623,  622,  618,  617,  616,  615,  614,  613,
+      612,  611,  610,  609,  608,  607,  601,  599,  598,  597,
+      596,  592,  591,  590,  589,  588,  587,  586,  585,  584,
+      575,  571,  570,  568,  567,  565,  564,  563,  562,  561,
+      560,  559,  558,  557,  555,  553,  552,  551,  550,  549,
+      548,  547,  546,  545,  544,  542,  541,  537,  536,  535,
+
+      532,  531,  529,  528,  526,  525,  524,  523,  522,  520,
+      518,  517,  516,  515,  514,  513,  511,  510,  509,  508,
+      507,  503,  502,  501,  500,  499,  498,  495,  494,  493,
+      492,  491,  489,  488,  487,  486,  485,  484,  482,  481,
+      474,  473,  470,  469,  466,  465,  464,  463,  452,  451,
+      450,  448,  447,  446,  445,  444,  441,  439,  438,  437,
+      435,  434,  432,  431,  430,  429,  427,  426,  421,  420,
+      418,  417,  416,  415,  414,  413,  412,  411,  410,  409,
+      407,  406,  405,  404,  403,  402,  401,  400,  399,  398,
+      397,  396,  392,  388,  387,  386,  383,  381,  371,  369,
+
+      365,  364,  363,  362,  360,  358,  353,  352,  349,  348,
+      347,  346,  345,  344,  339,  337,  336,  335,  334,  333,
+      331,  330,  327,  326,  325,  324,  323,  322,  321,  320,
+      319,  318,  317,  316,  315,  314,  313,  312,  311,  310,
+      309,  308,  307,  306,  305,  302,  300,  299,  298,  297,
+      296,  295,  294,  293,  291,  287,  285,  284,  283,  282,
+      278,  277,  276,  275,  274,  273,  272,  271,  267,  265,
+      264,  254,  253,  252,  251,  250,  249,  248,  246,  245,
+      244,  243,  242,  240,  239,  238,  237,  236,  235,  234,
+      233,  232,  231,  230,  229,  228,  227,  226,  225,  224,
+
+      223,  222,  221,  220,  219,  218,  217,  216,  214,  213,
+      212,  211,  210,  209,  208,  207,  206,  205,  204,  203,
+      202,  201,  200,  199,  198,  196,  195,  194,  193,  192,
+      191,  190,  189,  188,  187,  186,  185,  183,  182,  181,
+      179,  178,  177,  176,  174,  159,  158,  157,  156,  149,
+      147,  146,  145,  141,  140,  139,  138,  137,  136,  134,
+      133,  132,  131,  130,  129,  128,  127,  125,  123,  122,
+      121,  119,  118,  117,  115,  114,  112,  111,  110,  109,
+      108,  107,  104,  103,  102,  101,  100,   99,   98,   97,
+       95,   94,   93,   92,   91,   90,   89,   88,   87,   83,
+
+       79,   52,   45,   38,   32,   22,   18,   14,   12,   11,
+        7,    6,    5,  835,  835,  835,  835,  835,  835,  835,
+      835,  835,  835,  835,  835,  835,  835,  835,  835,  835,
+      835,  835,  835,  835,  835,  835,  835,  835,  835,  835,
+      835,  835,  835,  835,  835,  835,  835,  835,  835,  835,
+      835,  835,  835,  835,  835,  835,  835,  835,  835,  835,
+      835,  835,  835,  835,  835,  835,  835,  835,  835,  835,
+      835,  835,  835,  835,  835,  835,  835,  835,  835,  835,
+      835,  835
     } ;
 
 /* The intent behind this definition is that it'll catch
@@ -1055,7 +1076,7 @@ static yyconst flex_int16_t yy_chk[1255] =
  */
 #define ES yyextra->es_shader
 
-#line 1059 "glsl_lexer.cpp"
+#line 1080 "glsl_lexer.cpp"
 
 #define INITIAL 0
 #define PP 1
@@ -1182,7 +1203,12 @@ static int input (yyscan_t yyscanner );
 
 /* Amount of stuff to slurp up with each read. */
 #ifndef YY_READ_BUF_SIZE
+#ifdef __ia64__
+/* On IA-64, the buffer size is 16k, not 8k */
+#define YY_READ_BUF_SIZE 16384
+#else
 #define YY_READ_BUF_SIZE 8192
+#endif /* __ia64__ */
 #endif
 
 /* Copy whatever the last rule matched to the standard output. */
@@ -1201,7 +1227,7 @@ static int input (yyscan_t yyscanner );
 	if ( YY_CURRENT_BUFFER_LVALUE->yy_is_interactive ) \
 		{ \
 		int c = '*'; \
-		unsigned n; \
+		size_t n; \
 		for ( n = 0; n < max_size && \
 			     (c = getc( yyin )) != EOF && c != '\n'; ++n ) \
 			buf[n] = (char) c; \
@@ -1292,7 +1318,7 @@ YY_DECL
 #line 95 "glsl_lexer.lpp"
 
 
-#line 1296 "glsl_lexer.cpp"
+#line 1322 "glsl_lexer.cpp"
 
     yylval = yylval_param;
 
@@ -1350,13 +1376,13 @@ yy_match:
 			while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state )
 				{
 				yy_current_state = (int) yy_def[yy_current_state];
-				if ( yy_current_state >= 813 )
+				if ( yy_current_state >= 836 )
 					yy_c = yy_meta[(unsigned int) yy_c];
 				}
 			yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int) yy_c];
 			++yy_cp;
 			}
-		while ( yy_current_state != 812 );
+		while ( yy_current_state != 835 );
 		yy_cp = yyg->yy_last_accepting_cpos;
 		yy_current_state = yyg->yy_last_accepting_state;
 
@@ -1478,435 +1504,443 @@ YY_RULE_SETUP
 case 11:
 YY_RULE_SETUP
 #line 148 "glsl_lexer.lpp"
-{ BEGIN PRAGMA; }
+{
+				  BEGIN PP;
+				  return PRAGMA_INVARIANT_ALL;
+				}
 	YY_BREAK
 case 12:
-/* rule 12 can match eol */
 YY_RULE_SETUP
-#line 150 "glsl_lexer.lpp"
-{ BEGIN 0; yylineno++; yycolumn = 0; }
+#line 152 "glsl_lexer.lpp"
+{ BEGIN PRAGMA; }
 	YY_BREAK
 case 13:
+/* rule 13 can match eol */
 YY_RULE_SETUP
-#line 151 "glsl_lexer.lpp"
-{ }
+#line 154 "glsl_lexer.lpp"
+{ BEGIN 0; yylineno++; yycolumn = 0; }
 	YY_BREAK
 case 14:
 YY_RULE_SETUP
-#line 153 "glsl_lexer.lpp"
+#line 155 "glsl_lexer.lpp"
 { }
 	YY_BREAK
 case 15:
 YY_RULE_SETUP
-#line 154 "glsl_lexer.lpp"
+#line 157 "glsl_lexer.lpp"
 { }
 	YY_BREAK
 case 16:
 YY_RULE_SETUP
-#line 155 "glsl_lexer.lpp"
-return COLON;
+#line 158 "glsl_lexer.lpp"
+{ }
 	YY_BREAK
 case 17:
 YY_RULE_SETUP
-#line 156 "glsl_lexer.lpp"
+#line 159 "glsl_lexer.lpp"
+return COLON;
+	YY_BREAK
+case 18:
+YY_RULE_SETUP
+#line 160 "glsl_lexer.lpp"
 {
 				   yylval->identifier = strdup(yytext);
 				   return IDENTIFIER;
 				}
 	YY_BREAK
-case 18:
+case 19:
 YY_RULE_SETUP
-#line 160 "glsl_lexer.lpp"
+#line 164 "glsl_lexer.lpp"
 {
 				    yylval->n = strtol(yytext, NULL, 10);
 				    return INTCONSTANT;
 				}
 	YY_BREAK
-case 19:
-/* rule 19 can match eol */
+case 20:
+/* rule 20 can match eol */
 YY_RULE_SETUP
-#line 164 "glsl_lexer.lpp"
+#line 168 "glsl_lexer.lpp"
 { BEGIN 0; yylineno++; yycolumn = 0; return EOL; }
 	YY_BREAK
-case 20:
-/* rule 20 can match eol */
+case 21:
+/* rule 21 can match eol */
 YY_RULE_SETUP
-#line 166 "glsl_lexer.lpp"
+#line 170 "glsl_lexer.lpp"
 { yylineno++; yycolumn = 0; }
 	YY_BREAK
-case 21:
+case 22:
 YY_RULE_SETUP
-#line 168 "glsl_lexer.lpp"
+#line 172 "glsl_lexer.lpp"
 return ATTRIBUTE;
 	YY_BREAK
-case 22:
+case 23:
 YY_RULE_SETUP
-#line 169 "glsl_lexer.lpp"
+#line 173 "glsl_lexer.lpp"
 return CONST_TOK;
 	YY_BREAK
-case 23:
+case 24:
 YY_RULE_SETUP
-#line 170 "glsl_lexer.lpp"
+#line 174 "glsl_lexer.lpp"
 return BOOL_TOK;
 	YY_BREAK
-case 24:
+case 25:
 YY_RULE_SETUP
-#line 171 "glsl_lexer.lpp"
+#line 175 "glsl_lexer.lpp"
 return FLOAT_TOK;
 	YY_BREAK
-case 25:
+case 26:
 YY_RULE_SETUP
-#line 172 "glsl_lexer.lpp"
+#line 176 "glsl_lexer.lpp"
 return INT_TOK;
 	YY_BREAK
-case 26:
+case 27:
 YY_RULE_SETUP
-#line 173 "glsl_lexer.lpp"
+#line 177 "glsl_lexer.lpp"
 KEYWORD(130, 130, UINT_TOK);
 	YY_BREAK
-case 27:
+case 28:
 YY_RULE_SETUP
-#line 175 "glsl_lexer.lpp"
+#line 179 "glsl_lexer.lpp"
 return BREAK;
 	YY_BREAK
-case 28:
+case 29:
 YY_RULE_SETUP
-#line 176 "glsl_lexer.lpp"
+#line 180 "glsl_lexer.lpp"
 return CONTINUE;
 	YY_BREAK
-case 29:
+case 30:
 YY_RULE_SETUP
-#line 177 "glsl_lexer.lpp"
+#line 181 "glsl_lexer.lpp"
 return DO;
 	YY_BREAK
-case 30:
+case 31:
 YY_RULE_SETUP
-#line 178 "glsl_lexer.lpp"
+#line 182 "glsl_lexer.lpp"
 return WHILE;
 	YY_BREAK
-case 31:
+case 32:
 YY_RULE_SETUP
-#line 179 "glsl_lexer.lpp"
+#line 183 "glsl_lexer.lpp"
 return ELSE;
 	YY_BREAK
-case 32:
+case 33:
 YY_RULE_SETUP
-#line 180 "glsl_lexer.lpp"
+#line 184 "glsl_lexer.lpp"
 return FOR;
 	YY_BREAK
-case 33:
+case 34:
 YY_RULE_SETUP
-#line 181 "glsl_lexer.lpp"
+#line 185 "glsl_lexer.lpp"
 return IF;
 	YY_BREAK
-case 34:
+case 35:
 YY_RULE_SETUP
-#line 182 "glsl_lexer.lpp"
+#line 186 "glsl_lexer.lpp"
 return DISCARD;
 	YY_BREAK
-case 35:
+case 36:
 YY_RULE_SETUP
-#line 183 "glsl_lexer.lpp"
+#line 187 "glsl_lexer.lpp"
 return RETURN;
 	YY_BREAK
-case 36:
+case 37:
 YY_RULE_SETUP
-#line 185 "glsl_lexer.lpp"
+#line 189 "glsl_lexer.lpp"
 return BVEC2;
 	YY_BREAK
-case 37:
+case 38:
 YY_RULE_SETUP
-#line 186 "glsl_lexer.lpp"
+#line 190 "glsl_lexer.lpp"
 return BVEC3;
 	YY_BREAK
-case 38:
+case 39:
 YY_RULE_SETUP
-#line 187 "glsl_lexer.lpp"
+#line 191 "glsl_lexer.lpp"
 return BVEC4;
 	YY_BREAK
-case 39:
+case 40:
 YY_RULE_SETUP
-#line 188 "glsl_lexer.lpp"
+#line 192 "glsl_lexer.lpp"
 return IVEC2;
 	YY_BREAK
-case 40:
+case 41:
 YY_RULE_SETUP
-#line 189 "glsl_lexer.lpp"
+#line 193 "glsl_lexer.lpp"
 return IVEC3;
 	YY_BREAK
-case 41:
+case 42:
 YY_RULE_SETUP
-#line 190 "glsl_lexer.lpp"
+#line 194 "glsl_lexer.lpp"
 return IVEC4;
 	YY_BREAK
-case 42:
+case 43:
 YY_RULE_SETUP
-#line 191 "glsl_lexer.lpp"
+#line 195 "glsl_lexer.lpp"
 KEYWORD(130, 130, UVEC2);
 	YY_BREAK
-case 43:
+case 44:
 YY_RULE_SETUP
-#line 192 "glsl_lexer.lpp"
+#line 196 "glsl_lexer.lpp"
 KEYWORD(130, 130, UVEC3);
 	YY_BREAK
-case 44:
+case 45:
 YY_RULE_SETUP
-#line 193 "glsl_lexer.lpp"
+#line 197 "glsl_lexer.lpp"
 KEYWORD(130, 130, UVEC4);
 	YY_BREAK
-case 45:
+case 46:
 YY_RULE_SETUP
-#line 194 "glsl_lexer.lpp"
+#line 198 "glsl_lexer.lpp"
 return VEC2;
 	YY_BREAK
-case 46:
+case 47:
 YY_RULE_SETUP
-#line 195 "glsl_lexer.lpp"
+#line 199 "glsl_lexer.lpp"
 return VEC3;
 	YY_BREAK
-case 47:
+case 48:
 YY_RULE_SETUP
-#line 196 "glsl_lexer.lpp"
+#line 200 "glsl_lexer.lpp"
 return VEC4;
 	YY_BREAK
-case 48:
+case 49:
 YY_RULE_SETUP
-#line 197 "glsl_lexer.lpp"
+#line 201 "glsl_lexer.lpp"
 return MAT2X2;
 	YY_BREAK
-case 49:
+case 50:
 YY_RULE_SETUP
-#line 198 "glsl_lexer.lpp"
+#line 202 "glsl_lexer.lpp"
 return MAT3X3;
 	YY_BREAK
-case 50:
+case 51:
 YY_RULE_SETUP
-#line 199 "glsl_lexer.lpp"
+#line 203 "glsl_lexer.lpp"
 return MAT4X4;
 	YY_BREAK
-case 51:
+case 52:
 YY_RULE_SETUP
-#line 200 "glsl_lexer.lpp"
+#line 204 "glsl_lexer.lpp"
 KEYWORD(120, 120, MAT2X2);
 	YY_BREAK
-case 52:
+case 53:
 YY_RULE_SETUP
-#line 201 "glsl_lexer.lpp"
+#line 205 "glsl_lexer.lpp"
 KEYWORD(120, 120, MAT2X3);
 	YY_BREAK
-case 53:
+case 54:
 YY_RULE_SETUP
-#line 202 "glsl_lexer.lpp"
+#line 206 "glsl_lexer.lpp"
 KEYWORD(120, 120, MAT2X4);
 	YY_BREAK
-case 54:
+case 55:
 YY_RULE_SETUP
-#line 203 "glsl_lexer.lpp"
+#line 207 "glsl_lexer.lpp"
 KEYWORD(120, 120, MAT3X2);
 	YY_BREAK
-case 55:
+case 56:
 YY_RULE_SETUP
-#line 204 "glsl_lexer.lpp"
+#line 208 "glsl_lexer.lpp"
 KEYWORD(120, 120, MAT3X3);
 	YY_BREAK
-case 56:
+case 57:
 YY_RULE_SETUP
-#line 205 "glsl_lexer.lpp"
+#line 209 "glsl_lexer.lpp"
 KEYWORD(120, 120, MAT3X4);
 	YY_BREAK
-case 57:
+case 58:
 YY_RULE_SETUP
-#line 206 "glsl_lexer.lpp"
+#line 210 "glsl_lexer.lpp"
 KEYWORD(120, 120, MAT4X2);
 	YY_BREAK
-case 58:
+case 59:
 YY_RULE_SETUP
-#line 207 "glsl_lexer.lpp"
+#line 211 "glsl_lexer.lpp"
 KEYWORD(120, 120, MAT4X3);
 	YY_BREAK
-case 59:
+case 60:
 YY_RULE_SETUP
-#line 208 "glsl_lexer.lpp"
+#line 212 "glsl_lexer.lpp"
 KEYWORD(120, 120, MAT4X4);
 	YY_BREAK
-case 60:
+case 61:
 YY_RULE_SETUP
-#line 210 "glsl_lexer.lpp"
+#line 214 "glsl_lexer.lpp"
 return IN_TOK;
 	YY_BREAK
-case 61:
+case 62:
 YY_RULE_SETUP
-#line 211 "glsl_lexer.lpp"
+#line 215 "glsl_lexer.lpp"
 return OUT_TOK;
 	YY_BREAK
-case 62:
+case 63:
 YY_RULE_SETUP
-#line 212 "glsl_lexer.lpp"
+#line 216 "glsl_lexer.lpp"
 return INOUT_TOK;
 	YY_BREAK
-case 63:
+case 64:
 YY_RULE_SETUP
-#line 213 "glsl_lexer.lpp"
+#line 217 "glsl_lexer.lpp"
 return UNIFORM;
 	YY_BREAK
-case 64:
+case 65:
 YY_RULE_SETUP
-#line 214 "glsl_lexer.lpp"
+#line 218 "glsl_lexer.lpp"
 return VARYING;
 	YY_BREAK
-case 65:
+case 66:
 YY_RULE_SETUP
-#line 215 "glsl_lexer.lpp"
+#line 219 "glsl_lexer.lpp"
 KEYWORD(120, 120, CENTROID);
 	YY_BREAK
-case 66:
+case 67:
 YY_RULE_SETUP
-#line 216 "glsl_lexer.lpp"
+#line 220 "glsl_lexer.lpp"
 KEYWORD(120 || ES, 120 || ES, INVARIANT);
 	YY_BREAK
-case 67:
+case 68:
 YY_RULE_SETUP
-#line 217 "glsl_lexer.lpp"
+#line 221 "glsl_lexer.lpp"
 KEYWORD(130 || ES, 130, FLAT);
 	YY_BREAK
-case 68:
+case 69:
 YY_RULE_SETUP
-#line 218 "glsl_lexer.lpp"
+#line 222 "glsl_lexer.lpp"
 KEYWORD(130, 130, SMOOTH);
 	YY_BREAK
-case 69:
+case 70:
 YY_RULE_SETUP
-#line 219 "glsl_lexer.lpp"
+#line 223 "glsl_lexer.lpp"
 KEYWORD(130, 130, NOPERSPECTIVE);
 	YY_BREAK
-case 70:
+case 71:
 YY_RULE_SETUP
-#line 221 "glsl_lexer.lpp"
+#line 225 "glsl_lexer.lpp"
 return SAMPLER1D;
 	YY_BREAK
-case 71:
+case 72:
 YY_RULE_SETUP
-#line 222 "glsl_lexer.lpp"
+#line 226 "glsl_lexer.lpp"
 return SAMPLER2D;
 	YY_BREAK
-case 72:
+case 73:
 YY_RULE_SETUP
-#line 223 "glsl_lexer.lpp"
+#line 227 "glsl_lexer.lpp"
 return SAMPLER3D;
 	YY_BREAK
-case 73:
+case 74:
 YY_RULE_SETUP
-#line 224 "glsl_lexer.lpp"
+#line 228 "glsl_lexer.lpp"
 return SAMPLERCUBE;
 	YY_BREAK
-case 74:
+case 75:
 YY_RULE_SETUP
-#line 225 "glsl_lexer.lpp"
+#line 229 "glsl_lexer.lpp"
 KEYWORD(130, 130, SAMPLER1DARRAY);
 	YY_BREAK
-case 75:
+case 76:
 YY_RULE_SETUP
-#line 226 "glsl_lexer.lpp"
+#line 230 "glsl_lexer.lpp"
 KEYWORD(130, 130, SAMPLER2DARRAY);
 	YY_BREAK
-case 76:
+case 77:
 YY_RULE_SETUP
-#line 227 "glsl_lexer.lpp"
+#line 231 "glsl_lexer.lpp"
 return SAMPLER1DSHADOW;
 	YY_BREAK
-case 77:
+case 78:
 YY_RULE_SETUP
-#line 228 "glsl_lexer.lpp"
+#line 232 "glsl_lexer.lpp"
 return SAMPLER2DSHADOW;
 	YY_BREAK
-case 78:
+case 79:
 YY_RULE_SETUP
-#line 229 "glsl_lexer.lpp"
+#line 233 "glsl_lexer.lpp"
 KEYWORD(130, 130, SAMPLERCUBESHADOW);
 	YY_BREAK
-case 79:
+case 80:
 YY_RULE_SETUP
-#line 230 "glsl_lexer.lpp"
+#line 234 "glsl_lexer.lpp"
 KEYWORD(130, 130, SAMPLER1DARRAYSHADOW);
 	YY_BREAK
-case 80:
+case 81:
 YY_RULE_SETUP
-#line 231 "glsl_lexer.lpp"
+#line 235 "glsl_lexer.lpp"
 KEYWORD(130, 130, SAMPLER2DARRAYSHADOW);
 	YY_BREAK
-case 81:
+case 82:
 YY_RULE_SETUP
-#line 232 "glsl_lexer.lpp"
+#line 236 "glsl_lexer.lpp"
 KEYWORD(130, 130, ISAMPLER1D);
 	YY_BREAK
-case 82:
+case 83:
 YY_RULE_SETUP
-#line 233 "glsl_lexer.lpp"
+#line 237 "glsl_lexer.lpp"
 KEYWORD(130, 130, ISAMPLER2D);
 	YY_BREAK
-case 83:
+case 84:
 YY_RULE_SETUP
-#line 234 "glsl_lexer.lpp"
+#line 238 "glsl_lexer.lpp"
 KEYWORD(130, 130, ISAMPLER3D);
 	YY_BREAK
-case 84:
+case 85:
 YY_RULE_SETUP
-#line 235 "glsl_lexer.lpp"
+#line 239 "glsl_lexer.lpp"
 KEYWORD(130, 130, ISAMPLERCUBE);
 	YY_BREAK
-case 85:
+case 86:
 YY_RULE_SETUP
-#line 236 "glsl_lexer.lpp"
+#line 240 "glsl_lexer.lpp"
 KEYWORD(130, 130, ISAMPLER1DARRAY);
 	YY_BREAK
-case 86:
+case 87:
 YY_RULE_SETUP
-#line 237 "glsl_lexer.lpp"
+#line 241 "glsl_lexer.lpp"
 KEYWORD(130, 130, ISAMPLER2DARRAY);
 	YY_BREAK
-case 87:
+case 88:
 YY_RULE_SETUP
-#line 238 "glsl_lexer.lpp"
+#line 242 "glsl_lexer.lpp"
 KEYWORD(130, 130, USAMPLER1D);
 	YY_BREAK
-case 88:
+case 89:
 YY_RULE_SETUP
-#line 239 "glsl_lexer.lpp"
+#line 243 "glsl_lexer.lpp"
 KEYWORD(130, 130, USAMPLER2D);
 	YY_BREAK
-case 89:
+case 90:
 YY_RULE_SETUP
-#line 240 "glsl_lexer.lpp"
+#line 244 "glsl_lexer.lpp"
 KEYWORD(130, 130, USAMPLER3D);
 	YY_BREAK
-case 90:
+case 91:
 YY_RULE_SETUP
-#line 241 "glsl_lexer.lpp"
+#line 245 "glsl_lexer.lpp"
 KEYWORD(130, 130, USAMPLERCUBE);
 	YY_BREAK
-case 91:
+case 92:
 YY_RULE_SETUP
-#line 242 "glsl_lexer.lpp"
+#line 246 "glsl_lexer.lpp"
 KEYWORD(130, 130, USAMPLER1DARRAY);
 	YY_BREAK
-case 92:
+case 93:
 YY_RULE_SETUP
-#line 243 "glsl_lexer.lpp"
+#line 247 "glsl_lexer.lpp"
 KEYWORD(130, 130, USAMPLER2DARRAY);
 	YY_BREAK
-case 93:
+case 94:
 YY_RULE_SETUP
-#line 246 "glsl_lexer.lpp"
+#line 250 "glsl_lexer.lpp"
 return STRUCT;
 	YY_BREAK
-case 94:
+case 95:
 YY_RULE_SETUP
-#line 247 "glsl_lexer.lpp"
+#line 251 "glsl_lexer.lpp"
 return VOID_TOK;
 	YY_BREAK
-case 95:
+case 96:
 YY_RULE_SETUP
-#line 249 "glsl_lexer.lpp"
+#line 253 "glsl_lexer.lpp"
 {
 		  if ((yyextra->language_version >= 140)
 		      || yyextra->ARB_explicit_attrib_location_enable
@@ -1918,124 +1952,116 @@ YY_RULE_SETUP
 		   }
 		}
 	YY_BREAK
-case 96:
-YY_RULE_SETUP
-#line 260 "glsl_lexer.lpp"
-return INC_OP;
-	YY_BREAK
 case 97:
 YY_RULE_SETUP
-#line 261 "glsl_lexer.lpp"
-return DEC_OP;
+#line 264 "glsl_lexer.lpp"
+return INC_OP;
 	YY_BREAK
 case 98:
 YY_RULE_SETUP
-#line 262 "glsl_lexer.lpp"
-return LE_OP;
+#line 265 "glsl_lexer.lpp"
+return DEC_OP;
 	YY_BREAK
 case 99:
 YY_RULE_SETUP
-#line 263 "glsl_lexer.lpp"
-return GE_OP;
+#line 266 "glsl_lexer.lpp"
+return LE_OP;
 	YY_BREAK
 case 100:
 YY_RULE_SETUP
-#line 264 "glsl_lexer.lpp"
-return EQ_OP;
+#line 267 "glsl_lexer.lpp"
+return GE_OP;
 	YY_BREAK
 case 101:
 YY_RULE_SETUP
-#line 265 "glsl_lexer.lpp"
-return NE_OP;
+#line 268 "glsl_lexer.lpp"
+return EQ_OP;
 	YY_BREAK
 case 102:
 YY_RULE_SETUP
-#line 266 "glsl_lexer.lpp"
-return AND_OP;
+#line 269 "glsl_lexer.lpp"
+return NE_OP;
 	YY_BREAK
 case 103:
 YY_RULE_SETUP
-#line 267 "glsl_lexer.lpp"
-return OR_OP;
+#line 270 "glsl_lexer.lpp"
+return AND_OP;
 	YY_BREAK
 case 104:
 YY_RULE_SETUP
-#line 268 "glsl_lexer.lpp"
-return XOR_OP;
+#line 271 "glsl_lexer.lpp"
+return OR_OP;
 	YY_BREAK
 case 105:
 YY_RULE_SETUP
-#line 269 "glsl_lexer.lpp"
-return LEFT_OP;
+#line 272 "glsl_lexer.lpp"
+return XOR_OP;
 	YY_BREAK
 case 106:
 YY_RULE_SETUP
-#line 270 "glsl_lexer.lpp"
-return RIGHT_OP;
+#line 273 "glsl_lexer.lpp"
+return LEFT_OP;
 	YY_BREAK
 case 107:
 YY_RULE_SETUP
-#line 272 "glsl_lexer.lpp"
-return MUL_ASSIGN;
+#line 274 "glsl_lexer.lpp"
+return RIGHT_OP;
 	YY_BREAK
 case 108:
 YY_RULE_SETUP
-#line 273 "glsl_lexer.lpp"
-return DIV_ASSIGN;
+#line 276 "glsl_lexer.lpp"
+return MUL_ASSIGN;
 	YY_BREAK
 case 109:
 YY_RULE_SETUP
-#line 274 "glsl_lexer.lpp"
-return ADD_ASSIGN;
+#line 277 "glsl_lexer.lpp"
+return DIV_ASSIGN;
 	YY_BREAK
 case 110:
 YY_RULE_SETUP
-#line 275 "glsl_lexer.lpp"
-return MOD_ASSIGN;
+#line 278 "glsl_lexer.lpp"
+return ADD_ASSIGN;
 	YY_BREAK
 case 111:
 YY_RULE_SETUP
-#line 276 "glsl_lexer.lpp"
-return LEFT_ASSIGN;
+#line 279 "glsl_lexer.lpp"
+return MOD_ASSIGN;
 	YY_BREAK
 case 112:
 YY_RULE_SETUP
-#line 277 "glsl_lexer.lpp"
-return RIGHT_ASSIGN;
+#line 280 "glsl_lexer.lpp"
+return LEFT_ASSIGN;
 	YY_BREAK
 case 113:
 YY_RULE_SETUP
-#line 278 "glsl_lexer.lpp"
-return AND_ASSIGN;
+#line 281 "glsl_lexer.lpp"
+return RIGHT_ASSIGN;
 	YY_BREAK
 case 114:
 YY_RULE_SETUP
-#line 279 "glsl_lexer.lpp"
-return XOR_ASSIGN;
+#line 282 "glsl_lexer.lpp"
+return AND_ASSIGN;
 	YY_BREAK
 case 115:
 YY_RULE_SETUP
-#line 280 "glsl_lexer.lpp"
-return OR_ASSIGN;
+#line 283 "glsl_lexer.lpp"
+return XOR_ASSIGN;
 	YY_BREAK
 case 116:
 YY_RULE_SETUP
-#line 281 "glsl_lexer.lpp"
-return SUB_ASSIGN;
+#line 284 "glsl_lexer.lpp"
+return OR_ASSIGN;
 	YY_BREAK
 case 117:
 YY_RULE_SETUP
-#line 283 "glsl_lexer.lpp"
-{
-			    yylval->n = strtol(yytext, NULL, 10);
-			    return IS_UINT ? UINTCONSTANT : INTCONSTANT;
-			}
+#line 285 "glsl_lexer.lpp"
+return SUB_ASSIGN;
 	YY_BREAK
 case 118:
 YY_RULE_SETUP
 #line 287 "glsl_lexer.lpp"
 {
-			    yylval->n = strtol(yytext + 2, NULL, 16);
+			    yylval->n = strtol(yytext, NULL, 10);
 			    return IS_UINT ? UINTCONSTANT : INTCONSTANT;
 			}
 	YY_BREAK
@@ -2043,16 +2069,16 @@ case 119:
 YY_RULE_SETUP
 #line 291 "glsl_lexer.lpp"
 {
-			    yylval->n = strtol(yytext, NULL, 8);
+			    yylval->n = strtol(yytext + 2, NULL, 16);
 			    return IS_UINT ? UINTCONSTANT : INTCONSTANT;
 			}
 	YY_BREAK
 case 120:
 YY_RULE_SETUP
-#line 296 "glsl_lexer.lpp"
+#line 295 "glsl_lexer.lpp"
 {
-			    yylval->real = glsl_strtod(yytext, NULL);
-			    return FLOATCONSTANT;
+			    yylval->n = strtol(yytext, NULL, 8);
+			    return IS_UINT ? UINTCONSTANT : INTCONSTANT;
 			}
 	YY_BREAK
 case 121:
@@ -2089,426 +2115,434 @@ YY_RULE_SETUP
 	YY_BREAK
 case 125:
 YY_RULE_SETUP
-#line 317 "glsl_lexer.lpp"
+#line 316 "glsl_lexer.lpp"
 {
-			    yylval->n = 1;
-			    return BOOLCONSTANT;
+			    yylval->real = glsl_strtod(yytext, NULL);
+			    return FLOATCONSTANT;
 			}
 	YY_BREAK
 case 126:
 YY_RULE_SETUP
 #line 321 "glsl_lexer.lpp"
 {
+			    yylval->n = 1;
+			    return BOOLCONSTANT;
+			}
+	YY_BREAK
+case 127:
+YY_RULE_SETUP
+#line 325 "glsl_lexer.lpp"
+{
 			    yylval->n = 0;
 			    return BOOLCONSTANT;
 			}
 	YY_BREAK
 /* Reserved words in GLSL 1.10. */
-case 127:
+case 128:
 YY_RULE_SETUP
-#line 328 "glsl_lexer.lpp"
+#line 332 "glsl_lexer.lpp"
 KEYWORD(110 || ES, 999, ASM);
 	YY_BREAK
-case 128:
+case 129:
 YY_RULE_SETUP
-#line 329 "glsl_lexer.lpp"
+#line 333 "glsl_lexer.lpp"
 KEYWORD(110 || ES, 999, CLASS);
 	YY_BREAK
-case 129:
+case 130:
 YY_RULE_SETUP
-#line 330 "glsl_lexer.lpp"
+#line 334 "glsl_lexer.lpp"
 KEYWORD(110 || ES, 999, UNION);
 	YY_BREAK
-case 130:
+case 131:
 YY_RULE_SETUP
-#line 331 "glsl_lexer.lpp"
+#line 335 "glsl_lexer.lpp"
 KEYWORD(110 || ES, 999, ENUM);
 	YY_BREAK
-case 131:
+case 132:
 YY_RULE_SETUP
-#line 332 "glsl_lexer.lpp"
+#line 336 "glsl_lexer.lpp"
 KEYWORD(110 || ES, 999, TYPEDEF);
 	YY_BREAK
-case 132:
+case 133:
 YY_RULE_SETUP
-#line 333 "glsl_lexer.lpp"
+#line 337 "glsl_lexer.lpp"
 KEYWORD(110 || ES, 999, TEMPLATE);
 	YY_BREAK
-case 133:
+case 134:
 YY_RULE_SETUP
-#line 334 "glsl_lexer.lpp"
+#line 338 "glsl_lexer.lpp"
 KEYWORD(110 || ES, 999, THIS);
 	YY_BREAK
-case 134:
+case 135:
 YY_RULE_SETUP
-#line 335 "glsl_lexer.lpp"
+#line 339 "glsl_lexer.lpp"
 KEYWORD(110 || ES, 999, PACKED_TOK);
 	YY_BREAK
-case 135:
+case 136:
 YY_RULE_SETUP
-#line 336 "glsl_lexer.lpp"
+#line 340 "glsl_lexer.lpp"
 KEYWORD(110 || ES, 999, GOTO);
 	YY_BREAK
-case 136:
+case 137:
 YY_RULE_SETUP
-#line 337 "glsl_lexer.lpp"
+#line 341 "glsl_lexer.lpp"
 KEYWORD(110 || ES, 130, SWITCH);
 	YY_BREAK
-case 137:
+case 138:
 YY_RULE_SETUP
-#line 338 "glsl_lexer.lpp"
+#line 342 "glsl_lexer.lpp"
 KEYWORD(110 || ES, 130, DEFAULT);
 	YY_BREAK
-case 138:
+case 139:
 YY_RULE_SETUP
-#line 339 "glsl_lexer.lpp"
+#line 343 "glsl_lexer.lpp"
 KEYWORD(110 || ES, 999, INLINE_TOK);
 	YY_BREAK
-case 139:
+case 140:
 YY_RULE_SETUP
-#line 340 "glsl_lexer.lpp"
+#line 344 "glsl_lexer.lpp"
 KEYWORD(110 || ES, 999, NOINLINE);
 	YY_BREAK
-case 140:
+case 141:
 YY_RULE_SETUP
-#line 341 "glsl_lexer.lpp"
+#line 345 "glsl_lexer.lpp"
 KEYWORD(110 || ES, 999, VOLATILE);
 	YY_BREAK
-case 141:
+case 142:
 YY_RULE_SETUP
-#line 342 "glsl_lexer.lpp"
+#line 346 "glsl_lexer.lpp"
 KEYWORD(110 || ES, 999, PUBLIC_TOK);
 	YY_BREAK
-case 142:
+case 143:
 YY_RULE_SETUP
-#line 343 "glsl_lexer.lpp"
+#line 347 "glsl_lexer.lpp"
 KEYWORD(110 || ES, 999, STATIC);
 	YY_BREAK
-case 143:
+case 144:
 YY_RULE_SETUP
-#line 344 "glsl_lexer.lpp"
+#line 348 "glsl_lexer.lpp"
 KEYWORD(110 || ES, 999, EXTERN);
 	YY_BREAK
-case 144:
+case 145:
 YY_RULE_SETUP
-#line 345 "glsl_lexer.lpp"
+#line 349 "glsl_lexer.lpp"
 KEYWORD(110 || ES, 999, EXTERNAL);
 	YY_BREAK
-case 145:
+case 146:
 YY_RULE_SETUP
-#line 346 "glsl_lexer.lpp"
+#line 350 "glsl_lexer.lpp"
 KEYWORD(110 || ES, 999, INTERFACE);
 	YY_BREAK
-case 146:
+case 147:
 YY_RULE_SETUP
-#line 347 "glsl_lexer.lpp"
+#line 351 "glsl_lexer.lpp"
 KEYWORD(110 || ES, 999, LONG_TOK);
 	YY_BREAK
-case 147:
+case 148:
 YY_RULE_SETUP
-#line 348 "glsl_lexer.lpp"
+#line 352 "glsl_lexer.lpp"
 KEYWORD(110 || ES, 999, SHORT_TOK);
 	YY_BREAK
-case 148:
+case 149:
 YY_RULE_SETUP
-#line 349 "glsl_lexer.lpp"
+#line 353 "glsl_lexer.lpp"
 KEYWORD(110 || ES, 400, DOUBLE_TOK);
 	YY_BREAK
-case 149:
+case 150:
 YY_RULE_SETUP
-#line 350 "glsl_lexer.lpp"
+#line 354 "glsl_lexer.lpp"
 KEYWORD(110 || ES, 999, HALF);
 	YY_BREAK
-case 150:
+case 151:
 YY_RULE_SETUP
-#line 351 "glsl_lexer.lpp"
+#line 355 "glsl_lexer.lpp"
 KEYWORD(110 || ES, 999, FIXED_TOK);
 	YY_BREAK
-case 151:
+case 152:
 YY_RULE_SETUP
-#line 352 "glsl_lexer.lpp"
+#line 356 "glsl_lexer.lpp"
 KEYWORD(110 || ES, 999, UNSIGNED);
 	YY_BREAK
-case 152:
+case 153:
 YY_RULE_SETUP
-#line 353 "glsl_lexer.lpp"
+#line 357 "glsl_lexer.lpp"
 KEYWORD(110 || ES, 999, INPUT_TOK);
 	YY_BREAK
-case 153:
+case 154:
 YY_RULE_SETUP
-#line 354 "glsl_lexer.lpp"
+#line 358 "glsl_lexer.lpp"
 KEYWORD(110 || ES, 999, OUTPUT);
 	YY_BREAK
-case 154:
+case 155:
 YY_RULE_SETUP
-#line 355 "glsl_lexer.lpp"
+#line 359 "glsl_lexer.lpp"
 KEYWORD(110 || ES, 999, HVEC2);
 	YY_BREAK
-case 155:
+case 156:
 YY_RULE_SETUP
-#line 356 "glsl_lexer.lpp"
+#line 360 "glsl_lexer.lpp"
 KEYWORD(110 || ES, 999, HVEC3);
 	YY_BREAK
-case 156:
+case 157:
 YY_RULE_SETUP
-#line 357 "glsl_lexer.lpp"
+#line 361 "glsl_lexer.lpp"
 KEYWORD(110 || ES, 999, HVEC4);
 	YY_BREAK
-case 157:
+case 158:
 YY_RULE_SETUP
-#line 358 "glsl_lexer.lpp"
+#line 362 "glsl_lexer.lpp"
 KEYWORD(110 || ES, 400, DVEC2);
 	YY_BREAK
-case 158:
+case 159:
 YY_RULE_SETUP
-#line 359 "glsl_lexer.lpp"
+#line 363 "glsl_lexer.lpp"
 KEYWORD(110 || ES, 400, DVEC3);
 	YY_BREAK
-case 159:
+case 160:
 YY_RULE_SETUP
-#line 360 "glsl_lexer.lpp"
+#line 364 "glsl_lexer.lpp"
 KEYWORD(110 || ES, 400, DVEC4);
 	YY_BREAK
-case 160:
+case 161:
 YY_RULE_SETUP
-#line 361 "glsl_lexer.lpp"
+#line 365 "glsl_lexer.lpp"
 KEYWORD(110 || ES, 999, FVEC2);
 	YY_BREAK
-case 161:
+case 162:
 YY_RULE_SETUP
-#line 362 "glsl_lexer.lpp"
+#line 366 "glsl_lexer.lpp"
 KEYWORD(110 || ES, 999, FVEC3);
 	YY_BREAK
-case 162:
+case 163:
 YY_RULE_SETUP
-#line 363 "glsl_lexer.lpp"
+#line 367 "glsl_lexer.lpp"
 KEYWORD(110 || ES, 999, FVEC4);
 	YY_BREAK
-case 163:
+case 164:
 YY_RULE_SETUP
-#line 364 "glsl_lexer.lpp"
+#line 368 "glsl_lexer.lpp"
 return SAMPLER2DRECT;
 	YY_BREAK
-case 164:
+case 165:
 YY_RULE_SETUP
-#line 365 "glsl_lexer.lpp"
+#line 369 "glsl_lexer.lpp"
 KEYWORD(110 || ES, 999, SAMPLER3DRECT);
 	YY_BREAK
-case 165:
+case 166:
 YY_RULE_SETUP
-#line 366 "glsl_lexer.lpp"
+#line 370 "glsl_lexer.lpp"
 return SAMPLER2DRECTSHADOW;
 	YY_BREAK
-case 166:
+case 167:
 YY_RULE_SETUP
-#line 367 "glsl_lexer.lpp"
+#line 371 "glsl_lexer.lpp"
 KEYWORD(110 || ES, 999, SIZEOF);
 	YY_BREAK
-case 167:
+case 168:
 YY_RULE_SETUP
-#line 368 "glsl_lexer.lpp"
+#line 372 "glsl_lexer.lpp"
 KEYWORD(110 || ES, 999, CAST);
 	YY_BREAK
-case 168:
+case 169:
 YY_RULE_SETUP
-#line 369 "glsl_lexer.lpp"
+#line 373 "glsl_lexer.lpp"
 KEYWORD(110 || ES, 999, NAMESPACE);
 	YY_BREAK
-case 169:
+case 170:
 YY_RULE_SETUP
-#line 370 "glsl_lexer.lpp"
+#line 374 "glsl_lexer.lpp"
 KEYWORD(110 || ES, 999, USING);
 	YY_BREAK
 /* Additional reserved words in GLSL 1.20. */
-case 170:
+case 171:
 YY_RULE_SETUP
-#line 373 "glsl_lexer.lpp"
+#line 377 "glsl_lexer.lpp"
 KEYWORD(120, 130 || ES, LOWP);
 	YY_BREAK
-case 171:
+case 172:
 YY_RULE_SETUP
-#line 374 "glsl_lexer.lpp"
+#line 378 "glsl_lexer.lpp"
 KEYWORD(120, 130 || ES, MEDIUMP);
 	YY_BREAK
-case 172:
+case 173:
 YY_RULE_SETUP
-#line 375 "glsl_lexer.lpp"
+#line 379 "glsl_lexer.lpp"
 KEYWORD(120, 130 || ES, HIGHP);
 	YY_BREAK
-case 173:
+case 174:
 YY_RULE_SETUP
-#line 376 "glsl_lexer.lpp"
+#line 380 "glsl_lexer.lpp"
 KEYWORD(120, 130 || ES, PRECISION);
 	YY_BREAK
 /* Additional reserved words in GLSL 1.30. */
-case 174:
+case 175:
 YY_RULE_SETUP
-#line 379 "glsl_lexer.lpp"
+#line 383 "glsl_lexer.lpp"
 KEYWORD(130, 130, CASE);
 	YY_BREAK
-case 175:
+case 176:
 YY_RULE_SETUP
-#line 380 "glsl_lexer.lpp"
+#line 384 "glsl_lexer.lpp"
 KEYWORD(130, 999, COMMON);
 	YY_BREAK
-case 176:
+case 177:
 YY_RULE_SETUP
-#line 381 "glsl_lexer.lpp"
+#line 385 "glsl_lexer.lpp"
 KEYWORD(130, 999, PARTITION);
 	YY_BREAK
-case 177:
+case 178:
 YY_RULE_SETUP
-#line 382 "glsl_lexer.lpp"
+#line 386 "glsl_lexer.lpp"
 KEYWORD(130, 999, ACTIVE);
 	YY_BREAK
-case 178:
+case 179:
 YY_RULE_SETUP
-#line 383 "glsl_lexer.lpp"
+#line 387 "glsl_lexer.lpp"
 KEYWORD(130 || ES, 999, SUPERP);
 	YY_BREAK
-case 179:
+case 180:
 YY_RULE_SETUP
-#line 384 "glsl_lexer.lpp"
+#line 388 "glsl_lexer.lpp"
 KEYWORD(130, 140, SAMPLERBUFFER);
 	YY_BREAK
-case 180:
+case 181:
 YY_RULE_SETUP
-#line 385 "glsl_lexer.lpp"
+#line 389 "glsl_lexer.lpp"
 KEYWORD(130, 999, FILTER);
 	YY_BREAK
-case 181:
+case 182:
 YY_RULE_SETUP
-#line 386 "glsl_lexer.lpp"
+#line 390 "glsl_lexer.lpp"
 KEYWORD(130, 999, IMAGE1D);
 	YY_BREAK
-case 182:
+case 183:
 YY_RULE_SETUP
-#line 387 "glsl_lexer.lpp"
+#line 391 "glsl_lexer.lpp"
 KEYWORD(130, 999, IMAGE2D);
 	YY_BREAK
-case 183:
+case 184:
 YY_RULE_SETUP
-#line 388 "glsl_lexer.lpp"
+#line 392 "glsl_lexer.lpp"
 KEYWORD(130, 999, IMAGE3D);
 	YY_BREAK
-case 184:
+case 185:
 YY_RULE_SETUP
-#line 389 "glsl_lexer.lpp"
+#line 393 "glsl_lexer.lpp"
 KEYWORD(130, 999, IMAGECUBE);
 	YY_BREAK
-case 185:
+case 186:
 YY_RULE_SETUP
-#line 390 "glsl_lexer.lpp"
+#line 394 "glsl_lexer.lpp"
 KEYWORD(130, 999, IIMAGE1D);
 	YY_BREAK
-case 186:
+case 187:
 YY_RULE_SETUP
-#line 391 "glsl_lexer.lpp"
+#line 395 "glsl_lexer.lpp"
 KEYWORD(130, 999, IIMAGE2D);
 	YY_BREAK
-case 187:
+case 188:
 YY_RULE_SETUP
-#line 392 "glsl_lexer.lpp"
+#line 396 "glsl_lexer.lpp"
 KEYWORD(130, 999, IIMAGE3D);
 	YY_BREAK
-case 188:
+case 189:
 YY_RULE_SETUP
-#line 393 "glsl_lexer.lpp"
+#line 397 "glsl_lexer.lpp"
 KEYWORD(130, 999, IIMAGECUBE);
 	YY_BREAK
-case 189:
+case 190:
 YY_RULE_SETUP
-#line 394 "glsl_lexer.lpp"
+#line 398 "glsl_lexer.lpp"
 KEYWORD(130, 999, UIMAGE1D);
 	YY_BREAK
-case 190:
+case 191:
 YY_RULE_SETUP
-#line 395 "glsl_lexer.lpp"
+#line 399 "glsl_lexer.lpp"
 KEYWORD(130, 999, UIMAGE2D);
 	YY_BREAK
-case 191:
+case 192:
 YY_RULE_SETUP
-#line 396 "glsl_lexer.lpp"
+#line 400 "glsl_lexer.lpp"
 KEYWORD(130, 999, UIMAGE3D);
 	YY_BREAK
-case 192:
+case 193:
 YY_RULE_SETUP
-#line 397 "glsl_lexer.lpp"
+#line 401 "glsl_lexer.lpp"
 KEYWORD(130, 999, UIMAGECUBE);
 	YY_BREAK
-case 193:
+case 194:
 YY_RULE_SETUP
-#line 398 "glsl_lexer.lpp"
+#line 402 "glsl_lexer.lpp"
 KEYWORD(130, 999, IMAGE1DARRAY);
 	YY_BREAK
-case 194:
+case 195:
 YY_RULE_SETUP
-#line 399 "glsl_lexer.lpp"
+#line 403 "glsl_lexer.lpp"
 KEYWORD(130, 999, IMAGE2DARRAY);
 	YY_BREAK
-case 195:
+case 196:
 YY_RULE_SETUP
-#line 400 "glsl_lexer.lpp"
+#line 404 "glsl_lexer.lpp"
 KEYWORD(130, 999, IIMAGE1DARRAY);
 	YY_BREAK
-case 196:
+case 197:
 YY_RULE_SETUP
-#line 401 "glsl_lexer.lpp"
+#line 405 "glsl_lexer.lpp"
 KEYWORD(130, 999, IIMAGE2DARRAY);
 	YY_BREAK
-case 197:
+case 198:
 YY_RULE_SETUP
-#line 402 "glsl_lexer.lpp"
+#line 406 "glsl_lexer.lpp"
 KEYWORD(130, 999, UIMAGE1DARRAY);
 	YY_BREAK
-case 198:
+case 199:
 YY_RULE_SETUP
-#line 403 "glsl_lexer.lpp"
+#line 407 "glsl_lexer.lpp"
 KEYWORD(130, 999, UIMAGE2DARRAY);
 	YY_BREAK
-case 199:
+case 200:
 YY_RULE_SETUP
-#line 404 "glsl_lexer.lpp"
+#line 408 "glsl_lexer.lpp"
 KEYWORD(130, 999, IMAGE1DSHADOW);
 	YY_BREAK
-case 200:
+case 201:
 YY_RULE_SETUP
-#line 405 "glsl_lexer.lpp"
+#line 409 "glsl_lexer.lpp"
 KEYWORD(130, 999, IMAGE2DSHADOW);
 	YY_BREAK
-case 201:
+case 202:
 YY_RULE_SETUP
-#line 406 "glsl_lexer.lpp"
+#line 410 "glsl_lexer.lpp"
 KEYWORD(130, 999, IMAGE1DARRAYSHADOW);
 	YY_BREAK
-case 202:
+case 203:
 YY_RULE_SETUP
-#line 407 "glsl_lexer.lpp"
+#line 411 "glsl_lexer.lpp"
 KEYWORD(130, 999, IMAGE2DARRAYSHADOW);
 	YY_BREAK
-case 203:
+case 204:
 YY_RULE_SETUP
-#line 408 "glsl_lexer.lpp"
+#line 412 "glsl_lexer.lpp"
 KEYWORD(130, 999, IMAGEBUFFER);
 	YY_BREAK
-case 204:
+case 205:
 YY_RULE_SETUP
-#line 409 "glsl_lexer.lpp"
+#line 413 "glsl_lexer.lpp"
 KEYWORD(130, 999, IIMAGEBUFFER);
 	YY_BREAK
-case 205:
+case 206:
 YY_RULE_SETUP
-#line 410 "glsl_lexer.lpp"
+#line 414 "glsl_lexer.lpp"
 KEYWORD(130, 999, UIMAGEBUFFER);
 	YY_BREAK
-case 206:
+case 207:
 YY_RULE_SETUP
-#line 411 "glsl_lexer.lpp"
+#line 415 "glsl_lexer.lpp"
 KEYWORD(130, 999, ROW_MAJOR);
 	YY_BREAK
-case 207:
+case 208:
 YY_RULE_SETUP
-#line 413 "glsl_lexer.lpp"
+#line 417 "glsl_lexer.lpp"
 {
 			    struct _mesa_glsl_parse_state *state = yyextra;
 			    void *ctx = state;	
@@ -2516,17 +2550,17 @@ YY_RULE_SETUP
 			    return IDENTIFIER;
 			}
 	YY_BREAK
-case 208:
+case 209:
 YY_RULE_SETUP
-#line 420 "glsl_lexer.lpp"
+#line 424 "glsl_lexer.lpp"
 { return yytext[0]; }
 	YY_BREAK
-case 209:
+case 210:
 YY_RULE_SETUP
-#line 422 "glsl_lexer.lpp"
+#line 426 "glsl_lexer.lpp"
 ECHO;
 	YY_BREAK
-#line 2530 "glsl_lexer.cpp"
+#line 2564 "glsl_lexer.cpp"
 case YY_STATE_EOF(INITIAL):
 case YY_STATE_EOF(PP):
 case YY_STATE_EOF(PRAGMA):
@@ -2824,7 +2858,7 @@ static int yy_get_next_buffer (yyscan_t yyscanner)
 		while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state )
 			{
 			yy_current_state = (int) yy_def[yy_current_state];
-			if ( yy_current_state >= 813 )
+			if ( yy_current_state >= 836 )
 				yy_c = yy_meta[(unsigned int) yy_c];
 			}
 		yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int) yy_c];
@@ -2853,11 +2887,11 @@ static int yy_get_next_buffer (yyscan_t yyscanner)
 	while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state )
 		{
 		yy_current_state = (int) yy_def[yy_current_state];
-		if ( yy_current_state >= 813 )
+		if ( yy_current_state >= 836 )
 			yy_c = yy_meta[(unsigned int) yy_c];
 		}
 	yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int) yy_c];
-	yy_is_jam = (yy_current_state == 812);
+	yy_is_jam = (yy_current_state == 835);
 
 	return yy_is_jam ? 0 : yy_current_state;
 }
@@ -3262,8 +3296,8 @@ YY_BUFFER_STATE _mesa_glsl__scan_string (yyconst char * yystr , yyscan_t yyscann
 
 /** Setup the input buffer state to scan the given bytes. The next call to _mesa_glsl_lex() will
  * scan from a @e copy of @a bytes.
- * @param bytes the byte buffer to scan
- * @param len the number of bytes in the buffer pointed to by @a bytes.
+ * @param yybytes the byte buffer to scan
+ * @param _yybytes_len the number of bytes in the buffer pointed to by @a bytes.
  * @param yyscanner The scanner object.
  * @return the newly allocated buffer state object.
  */
@@ -3669,7 +3703,7 @@ void _mesa_glsl_free (void * ptr , yyscan_t yyscanner)
 
 #define YYTABLES_NAME "yytables"
 
-#line 422 "glsl_lexer.lpp"
+#line 426 "glsl_lexer.lpp"
 
 
 
diff --git a/src/glsl/glsl_lexer.lpp b/src/glsl/glsl_lexer.lpp
index 15742ac3636..d30759be2b8 100644
--- a/src/glsl/glsl_lexer.lpp
+++ b/src/glsl/glsl_lexer.lpp
@@ -145,6 +145,10 @@ HASH		^{SPC}#{SPC}
 				  BEGIN PP;
 				  return PRAGMA_OPTIMIZE_OFF;
 				}
+^{SPC}#{SPC}pragma{SPCP}STDGL{SPCP}invariant{SPC}\({SPC}all{SPC}\) {
+				  BEGIN PP;
+				  return PRAGMA_INVARIANT_ALL;
+				}
 ^{SPC}#{SPC}pragma{SPCP}	{ BEGIN PRAGMA; }
 
 <PRAGMA>\n			{ BEGIN 0; yylineno++; yycolumn = 0; }
diff --git a/src/glsl/glsl_parser.cpp b/src/glsl/glsl_parser.cpp
index 23e439c7c54..ab6e83bead6 100644
--- a/src/glsl/glsl_parser.cpp
+++ b/src/glsl/glsl_parser.cpp
@@ -1,9 +1,10 @@
-/* A Bison parser, made by GNU Bison 2.4.3.  */
+
+/* A Bison parser, made by GNU Bison 2.4.1.  */
 
 /* Skeleton implementation for Bison's Yacc-like parsers in C
    
-      Copyright (C) 1984, 1989, 1990, 2000, 2001, 2002, 2003, 2004, 2005, 2006,
-   2009, 2010 Free Software Foundation, Inc.
+      Copyright (C) 1984, 1989, 1990, 2000, 2001, 2002, 2003, 2004, 2005, 2006
+   Free Software Foundation, Inc.
    
    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -45,7 +46,7 @@
 #define YYBISON 1
 
 /* Bison version.  */
-#define YYBISON_VERSION "2.4.3"
+#define YYBISON_VERSION "2.4.1"
 
 /* Skeleton name.  */
 #define YYSKELETON_NAME "yacc.c"
@@ -113,7 +114,7 @@
 
 
 /* Line 189 of yacc.c  */
-#line 117 "glsl_parser.cpp"
+#line 118 "glsl_parser.cpp"
 
 /* Enabling traces.  */
 #ifndef YYDEBUG
@@ -257,79 +258,80 @@
      PRAGMA_DEBUG_OFF = 372,
      PRAGMA_OPTIMIZE_ON = 373,
      PRAGMA_OPTIMIZE_OFF = 374,
-     LAYOUT_TOK = 375,
-     ASM = 376,
-     CLASS = 377,
-     UNION = 378,
-     ENUM = 379,
-     TYPEDEF = 380,
-     TEMPLATE = 381,
-     THIS = 382,
-     PACKED_TOK = 383,
-     GOTO = 384,
-     INLINE_TOK = 385,
-     NOINLINE = 386,
-     VOLATILE = 387,
-     PUBLIC_TOK = 388,
-     STATIC = 389,
-     EXTERN = 390,
-     EXTERNAL = 391,
-     LONG_TOK = 392,
-     SHORT_TOK = 393,
-     DOUBLE_TOK = 394,
-     HALF = 395,
-     FIXED_TOK = 396,
-     UNSIGNED = 397,
-     INPUT_TOK = 398,
-     OUPTUT = 399,
-     HVEC2 = 400,
-     HVEC3 = 401,
-     HVEC4 = 402,
-     DVEC2 = 403,
-     DVEC3 = 404,
-     DVEC4 = 405,
-     FVEC2 = 406,
-     FVEC3 = 407,
-     FVEC4 = 408,
-     SAMPLER2DRECT = 409,
-     SAMPLER3DRECT = 410,
-     SAMPLER2DRECTSHADOW = 411,
-     SIZEOF = 412,
-     CAST = 413,
-     NAMESPACE = 414,
-     USING = 415,
-     ERROR_TOK = 416,
-     COMMON = 417,
-     PARTITION = 418,
-     ACTIVE = 419,
-     SAMPLERBUFFER = 420,
-     FILTER = 421,
-     IMAGE1D = 422,
-     IMAGE2D = 423,
-     IMAGE3D = 424,
-     IMAGECUBE = 425,
-     IMAGE1DARRAY = 426,
-     IMAGE2DARRAY = 427,
-     IIMAGE1D = 428,
-     IIMAGE2D = 429,
-     IIMAGE3D = 430,
-     IIMAGECUBE = 431,
-     IIMAGE1DARRAY = 432,
-     IIMAGE2DARRAY = 433,
-     UIMAGE1D = 434,
-     UIMAGE2D = 435,
-     UIMAGE3D = 436,
-     UIMAGECUBE = 437,
-     UIMAGE1DARRAY = 438,
-     UIMAGE2DARRAY = 439,
-     IMAGE1DSHADOW = 440,
-     IMAGE2DSHADOW = 441,
-     IMAGEBUFFER = 442,
-     IIMAGEBUFFER = 443,
-     UIMAGEBUFFER = 444,
-     IMAGE1DARRAYSHADOW = 445,
-     IMAGE2DARRAYSHADOW = 446,
-     ROW_MAJOR = 447
+     PRAGMA_INVARIANT_ALL = 375,
+     LAYOUT_TOK = 376,
+     ASM = 377,
+     CLASS = 378,
+     UNION = 379,
+     ENUM = 380,
+     TYPEDEF = 381,
+     TEMPLATE = 382,
+     THIS = 383,
+     PACKED_TOK = 384,
+     GOTO = 385,
+     INLINE_TOK = 386,
+     NOINLINE = 387,
+     VOLATILE = 388,
+     PUBLIC_TOK = 389,
+     STATIC = 390,
+     EXTERN = 391,
+     EXTERNAL = 392,
+     LONG_TOK = 393,
+     SHORT_TOK = 394,
+     DOUBLE_TOK = 395,
+     HALF = 396,
+     FIXED_TOK = 397,
+     UNSIGNED = 398,
+     INPUT_TOK = 399,
+     OUPTUT = 400,
+     HVEC2 = 401,
+     HVEC3 = 402,
+     HVEC4 = 403,
+     DVEC2 = 404,
+     DVEC3 = 405,
+     DVEC4 = 406,
+     FVEC2 = 407,
+     FVEC3 = 408,
+     FVEC4 = 409,
+     SAMPLER2DRECT = 410,
+     SAMPLER3DRECT = 411,
+     SAMPLER2DRECTSHADOW = 412,
+     SIZEOF = 413,
+     CAST = 414,
+     NAMESPACE = 415,
+     USING = 416,
+     ERROR_TOK = 417,
+     COMMON = 418,
+     PARTITION = 419,
+     ACTIVE = 420,
+     SAMPLERBUFFER = 421,
+     FILTER = 422,
+     IMAGE1D = 423,
+     IMAGE2D = 424,
+     IMAGE3D = 425,
+     IMAGECUBE = 426,
+     IMAGE1DARRAY = 427,
+     IMAGE2DARRAY = 428,
+     IIMAGE1D = 429,
+     IIMAGE2D = 430,
+     IIMAGE3D = 431,
+     IIMAGECUBE = 432,
+     IIMAGE1DARRAY = 433,
+     IIMAGE2DARRAY = 434,
+     UIMAGE1D = 435,
+     UIMAGE2D = 436,
+     UIMAGE3D = 437,
+     UIMAGECUBE = 438,
+     UIMAGE1DARRAY = 439,
+     UIMAGE2DARRAY = 440,
+     IMAGE1DSHADOW = 441,
+     IMAGE2DSHADOW = 442,
+     IMAGEBUFFER = 443,
+     IIMAGEBUFFER = 444,
+     UIMAGEBUFFER = 445,
+     IMAGE1DARRAYSHADOW = 446,
+     IMAGE2DARRAYSHADOW = 447,
+     ROW_MAJOR = 448
    };
 #endif
 
@@ -373,7 +375,7 @@ typedef union YYSTYPE
 
 
 /* Line 214 of yacc.c  */
-#line 377 "glsl_parser.cpp"
+#line 379 "glsl_parser.cpp"
 } YYSTYPE;
 # define YYSTYPE_IS_TRIVIAL 1
 # define yystype YYSTYPE /* obsolescent; will be withdrawn */
@@ -398,7 +400,7 @@ typedef struct YYLTYPE
 
 
 /* Line 264 of yacc.c  */
-#line 402 "glsl_parser.cpp"
+#line 404 "glsl_parser.cpp"
 
 #ifdef short
 # undef short
@@ -448,7 +450,7 @@ typedef short int yytype_int16;
 #define YYSIZE_MAXIMUM ((YYSIZE_T) -1)
 
 #ifndef YY_
-# if defined YYENABLE_NLS && YYENABLE_NLS
+# if YYENABLE_NLS
 #  if ENABLE_NLS
 #   include <libintl.h> /* INFRINGES ON USER NAME SPACE */
 #   define YY_(msgid) dgettext ("bison-runtime", msgid)
@@ -615,20 +617,20 @@ union yyalloc
 /* YYFINAL -- State number of the termination state.  */
 #define YYFINAL  5
 /* YYLAST -- Last index in YYTABLE.  */
-#define YYLAST   3718
+#define YYLAST   3738
 
 /* YYNTOKENS -- Number of terminals.  */
-#define YYNTOKENS  217
+#define YYNTOKENS  218
 /* YYNNTS -- Number of nonterminals.  */
 #define YYNNTS  87
 /* YYNRULES -- Number of rules.  */
-#define YYNRULES  278
+#define YYNRULES  279
 /* YYNRULES -- Number of states.  */
-#define YYNSTATES  413
+#define YYNSTATES  415
 
 /* YYTRANSLATE(YYLEX) -- Bison symbol number corresponding to YYLEX.  */
 #define YYUNDEFTOK  2
-#define YYMAXUTOK   447
+#define YYMAXUTOK   448
 
 #define YYTRANSLATE(YYX)						\
   ((unsigned int) (YYX) <= YYMAXUTOK ? yytranslate[YYX] : YYUNDEFTOK)
@@ -639,16 +641,16 @@ static const yytype_uint8 yytranslate[] =
        0,     2,     2,     2,     2,     2,     2,     2,     2,     2,
        2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
        2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
-       2,     2,     2,   201,     2,     2,     2,   205,   208,     2,
-     193,   194,   203,   199,   198,   200,   197,   204,     2,     2,
-       2,     2,     2,     2,     2,     2,     2,     2,   212,   214,
-     206,   213,   207,   211,     2,     2,     2,     2,     2,     2,
+       2,     2,     2,   202,     2,     2,     2,   206,   209,     2,
+     194,   195,   204,   200,   199,   201,   198,   205,     2,     2,
+       2,     2,     2,     2,     2,     2,     2,     2,   213,   215,
+     207,   214,   208,   212,     2,     2,     2,     2,     2,     2,
        2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
        2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
-       2,   195,     2,   196,   209,     2,     2,     2,     2,     2,
+       2,   196,     2,   197,   210,     2,     2,     2,     2,     2,
        2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
        2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
-       2,     2,     2,   215,   210,   216,   202,     2,     2,     2,
+       2,     2,     2,   216,   211,   217,   203,     2,     2,     2,
        2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
        2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
        2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
@@ -680,7 +682,7 @@ static const yytype_uint8 yytranslate[] =
      155,   156,   157,   158,   159,   160,   161,   162,   163,   164,
      165,   166,   167,   168,   169,   170,   171,   172,   173,   174,
      175,   176,   177,   178,   179,   180,   181,   182,   183,   184,
-     185,   186,   187,   188,   189,   190,   191,   192
+     185,   186,   187,   188,   189,   190,   191,   192,   193
 };
 
 #if YYDEBUG
@@ -689,151 +691,151 @@ static const yytype_uint8 yytranslate[] =
 static const yytype_uint16 yyprhs[] =
 {
        0,     0,     3,     4,     9,    10,    14,    17,    20,    23,
-      26,    27,    30,    36,    38,    41,    43,    45,    47,    49,
-      51,    53,    57,    59,    64,    66,    70,    73,    76,    78,
-      80,    82,    86,    89,    92,    95,    97,   100,   104,   107,
-     109,   111,   113,   115,   118,   121,   124,   126,   128,   130,
-     132,   134,   138,   142,   146,   148,   152,   156,   158,   162,
-     166,   168,   172,   176,   180,   184,   186,   190,   194,   196,
-     200,   202,   206,   208,   212,   214,   218,   220,   224,   226,
-     230,   232,   238,   240,   244,   246,   248,   250,   252,   254,
-     256,   258,   260,   262,   264,   266,   268,   272,   274,   277,
-     280,   285,   288,   290,   292,   295,   299,   303,   306,   312,
-     316,   319,   323,   326,   327,   329,   331,   333,   335,   337,
-     341,   347,   354,   362,   371,   377,   379,   382,   387,   393,
-     400,   408,   413,   416,   418,   421,   426,   428,   432,   434,
-     438,   440,   442,   444,   446,   448,   450,   453,   455,   458,
-     461,   465,   467,   469,   471,   473,   476,   478,   480,   483,
-     486,   488,   490,   493,   495,   499,   504,   506,   508,   510,
-     512,   514,   516,   518,   520,   522,   524,   526,   528,   530,
-     532,   534,   536,   538,   540,   542,   544,   546,   548,   550,
-     552,   554,   556,   558,   560,   562,   564,   566,   568,   570,
-     572,   574,   576,   578,   580,   582,   584,   586,   588,   590,
-     592,   594,   596,   598,   600,   602,   604,   606,   608,   610,
-     612,   614,   616,   618,   624,   629,   631,   634,   638,   640,
-     644,   646,   651,   653,   655,   657,   659,   661,   663,   665,
-     667,   669,   671,   673,   676,   680,   682,   684,   687,   691,
-     693,   696,   698,   701,   707,   711,   713,   715,   720,   726,
-     730,   733,   739,   747,   754,   756,   758,   760,   761,   764,
-     768,   771,   774,   777,   781,   784,   786,   788,   790
+      26,    29,    30,    33,    39,    41,    44,    46,    48,    50,
+      52,    54,    56,    60,    62,    67,    69,    73,    76,    79,
+      81,    83,    85,    89,    92,    95,    98,   100,   103,   107,
+     110,   112,   114,   116,   118,   121,   124,   127,   129,   131,
+     133,   135,   137,   141,   145,   149,   151,   155,   159,   161,
+     165,   169,   171,   175,   179,   183,   187,   189,   193,   197,
+     199,   203,   205,   209,   211,   215,   217,   221,   223,   227,
+     229,   233,   235,   241,   243,   247,   249,   251,   253,   255,
+     257,   259,   261,   263,   265,   267,   269,   271,   275,   277,
+     280,   283,   288,   291,   293,   295,   298,   302,   306,   309,
+     315,   319,   322,   326,   329,   330,   332,   334,   336,   338,
+     340,   344,   350,   357,   365,   374,   380,   382,   385,   390,
+     396,   403,   411,   416,   419,   421,   424,   429,   431,   435,
+     437,   441,   443,   445,   447,   449,   451,   453,   456,   458,
+     461,   464,   468,   470,   472,   474,   476,   479,   481,   483,
+     486,   489,   491,   493,   496,   498,   502,   507,   509,   511,
+     513,   515,   517,   519,   521,   523,   525,   527,   529,   531,
+     533,   535,   537,   539,   541,   543,   545,   547,   549,   551,
+     553,   555,   557,   559,   561,   563,   565,   567,   569,   571,
+     573,   575,   577,   579,   581,   583,   585,   587,   589,   591,
+     593,   595,   597,   599,   601,   603,   605,   607,   609,   611,
+     613,   615,   617,   619,   621,   627,   632,   634,   637,   641,
+     643,   647,   649,   654,   656,   658,   660,   662,   664,   666,
+     668,   670,   672,   674,   676,   679,   683,   685,   687,   690,
+     694,   696,   699,   701,   704,   710,   714,   716,   718,   723,
+     729,   733,   736,   742,   750,   757,   759,   761,   763,   764,
+     767,   771,   774,   777,   780,   784,   787,   789,   791,   793
 };
 
 /* YYRHS -- A `-1'-separated list of the rules' RHS.  */
 static const yytype_int16 yyrhs[] =
 {
-     218,     0,    -1,    -1,   220,   222,   219,   224,    -1,    -1,
+     219,     0,    -1,    -1,   221,   223,   220,   225,    -1,    -1,
      109,    78,   113,    -1,   116,   113,    -1,   117,   113,    -1,
-     118,   113,    -1,   119,   113,    -1,    -1,   222,   223,    -1,
-     110,    76,   112,    76,   113,    -1,   302,    -1,   224,   302,
-      -1,    76,    -1,   225,    -1,    78,    -1,    79,    -1,    77,
-      -1,    80,    -1,   193,   252,   194,    -1,   226,    -1,   227,
-     195,   228,   196,    -1,   229,    -1,   227,   197,    76,    -1,
-     227,    84,    -1,   227,    85,    -1,   252,    -1,   230,    -1,
-     231,    -1,   227,   197,   231,    -1,   233,   194,    -1,   232,
-     194,    -1,   234,    74,    -1,   234,    -1,   234,   250,    -1,
-     233,   198,   250,    -1,   235,   193,    -1,   273,    -1,    76,
-      -1,    81,    -1,   227,    -1,    84,   236,    -1,    85,   236,
-      -1,   237,   236,    -1,   199,    -1,   200,    -1,   201,    -1,
-     202,    -1,   236,    -1,   238,   203,   236,    -1,   238,   204,
-     236,    -1,   238,   205,   236,    -1,   238,    -1,   239,   199,
-     238,    -1,   239,   200,   238,    -1,   239,    -1,   240,    82,
-     239,    -1,   240,    83,   239,    -1,   240,    -1,   241,   206,
-     240,    -1,   241,   207,   240,    -1,   241,    86,   240,    -1,
-     241,    87,   240,    -1,   241,    -1,   242,    88,   241,    -1,
-     242,    89,   241,    -1,   242,    -1,   243,   208,   242,    -1,
-     243,    -1,   244,   209,   243,    -1,   244,    -1,   245,   210,
-     244,    -1,   245,    -1,   246,    90,   245,    -1,   246,    -1,
-     247,    92,   246,    -1,   247,    -1,   248,    91,   247,    -1,
-     248,    -1,   248,   211,   252,   212,   250,    -1,   249,    -1,
-     236,   251,   250,    -1,   213,    -1,    93,    -1,    94,    -1,
-      96,    -1,    95,    -1,   102,    -1,    97,    -1,    98,    -1,
-      99,    -1,   100,    -1,   101,    -1,   250,    -1,   252,   198,
-     250,    -1,   249,    -1,   255,   214,    -1,   263,   214,    -1,
-     108,   277,   274,   214,    -1,   256,   194,    -1,   258,    -1,
-     257,    -1,   258,   260,    -1,   257,   198,   260,    -1,   265,
-      76,   193,    -1,   273,    76,    -1,   273,    76,   195,   253,
-     196,    -1,   270,   261,   259,    -1,   261,   259,    -1,   270,
-     261,   262,    -1,   261,   262,    -1,    -1,    33,    -1,    34,
-      -1,    35,    -1,   273,    -1,   264,    -1,   263,   198,    76,
-      -1,   263,   198,    76,   195,   196,    -1,   263,   198,    76,
-     195,   253,   196,    -1,   263,   198,    76,   195,   196,   213,
-     283,    -1,   263,   198,    76,   195,   253,   196,   213,   283,
-      -1,   263,   198,    76,   213,   283,    -1,   265,    -1,   265,
-      76,    -1,   265,    76,   195,   196,    -1,   265,    76,   195,
-     253,   196,    -1,   265,    76,   195,   196,   213,   283,    -1,
-     265,    76,   195,   253,   196,   213,   283,    -1,   265,    76,
-     213,   283,    -1,   103,    76,    -1,   273,    -1,   271,   273,
-      -1,   120,   193,   267,   194,    -1,   268,    -1,   267,   198,
-     268,    -1,    76,    -1,    76,   213,    78,    -1,    40,    -1,
-      39,    -1,    38,    -1,     4,    -1,   272,    -1,   266,    -1,
-     266,   272,    -1,   269,    -1,   269,   272,    -1,   103,   272,
-      -1,   103,   269,   272,    -1,   103,    -1,     4,    -1,     3,
-      -1,    37,    -1,    32,    37,    -1,    33,    -1,    34,    -1,
-      32,    33,    -1,    32,    34,    -1,    36,    -1,   274,    -1,
-     277,   274,    -1,   275,    -1,   275,   195,   196,    -1,   275,
-     195,   253,   196,    -1,   276,    -1,   278,    -1,    76,    -1,
-      74,    -1,     6,    -1,     7,    -1,     8,    -1,     5,    -1,
-      29,    -1,    30,    -1,    31,    -1,    20,    -1,    21,    -1,
-      22,    -1,    23,    -1,    24,    -1,    25,    -1,    26,    -1,
-      27,    -1,    28,    -1,    41,    -1,    42,    -1,    43,    -1,
-      44,    -1,    45,    -1,    46,    -1,    47,    -1,    48,    -1,
-      49,    -1,    50,    -1,    51,    -1,   154,    -1,    52,    -1,
-      53,    -1,    54,    -1,    55,    -1,   156,    -1,    56,    -1,
-      57,    -1,    58,    -1,    59,    -1,    60,    -1,    61,    -1,
-      62,    -1,    63,    -1,    64,    -1,    65,    -1,    66,    -1,
-      67,    -1,    68,    -1,    69,    -1,    70,    -1,    71,    -1,
-      72,    -1,   106,    -1,   105,    -1,   104,    -1,    73,    76,
-     215,   279,   216,    -1,    73,   215,   279,   216,    -1,   280,
-      -1,   279,   280,    -1,   273,   281,   214,    -1,   282,    -1,
-     281,   198,   282,    -1,    76,    -1,    76,   195,   253,   196,
-      -1,   250,    -1,   254,    -1,   287,    -1,   286,    -1,   284,
-      -1,   291,    -1,   292,    -1,   295,    -1,   296,    -1,   297,
-      -1,   301,    -1,   215,   216,    -1,   215,   290,   216,    -1,
-     289,    -1,   286,    -1,   215,   216,    -1,   215,   290,   216,
-      -1,   285,    -1,   290,   285,    -1,   214,    -1,   252,   214,
-      -1,    14,   193,   252,   194,   293,    -1,   285,    12,   285,
-      -1,   285,    -1,   252,    -1,   265,    76,   213,   283,    -1,
-      17,   193,   252,   194,   287,    -1,    18,   252,   212,    -1,
-      19,   212,    -1,    75,   193,   294,   194,   288,    -1,    11,
-     285,    75,   193,   252,   194,   214,    -1,    13,   193,   298,
-     300,   194,   288,    -1,   291,    -1,   284,    -1,   294,    -1,
-      -1,   299,   214,    -1,   299,   214,   252,    -1,    10,   214,
-      -1,     9,   214,    -1,    16,   214,    -1,    16,   252,   214,
-      -1,    15,   214,    -1,   303,    -1,   254,    -1,   221,    -1,
-     255,   289,    -1
+     118,   113,    -1,   119,   113,    -1,   120,   113,    -1,    -1,
+     223,   224,    -1,   110,    76,   112,    76,   113,    -1,   303,
+      -1,   225,   303,    -1,    76,    -1,   226,    -1,    78,    -1,
+      79,    -1,    77,    -1,    80,    -1,   194,   253,   195,    -1,
+     227,    -1,   228,   196,   229,   197,    -1,   230,    -1,   228,
+     198,    76,    -1,   228,    84,    -1,   228,    85,    -1,   253,
+      -1,   231,    -1,   232,    -1,   228,   198,   232,    -1,   234,
+     195,    -1,   233,   195,    -1,   235,    74,    -1,   235,    -1,
+     235,   251,    -1,   234,   199,   251,    -1,   236,   194,    -1,
+     274,    -1,    76,    -1,    81,    -1,   228,    -1,    84,   237,
+      -1,    85,   237,    -1,   238,   237,    -1,   200,    -1,   201,
+      -1,   202,    -1,   203,    -1,   237,    -1,   239,   204,   237,
+      -1,   239,   205,   237,    -1,   239,   206,   237,    -1,   239,
+      -1,   240,   200,   239,    -1,   240,   201,   239,    -1,   240,
+      -1,   241,    82,   240,    -1,   241,    83,   240,    -1,   241,
+      -1,   242,   207,   241,    -1,   242,   208,   241,    -1,   242,
+      86,   241,    -1,   242,    87,   241,    -1,   242,    -1,   243,
+      88,   242,    -1,   243,    89,   242,    -1,   243,    -1,   244,
+     209,   243,    -1,   244,    -1,   245,   210,   244,    -1,   245,
+      -1,   246,   211,   245,    -1,   246,    -1,   247,    90,   246,
+      -1,   247,    -1,   248,    92,   247,    -1,   248,    -1,   249,
+      91,   248,    -1,   249,    -1,   249,   212,   253,   213,   251,
+      -1,   250,    -1,   237,   252,   251,    -1,   214,    -1,    93,
+      -1,    94,    -1,    96,    -1,    95,    -1,   102,    -1,    97,
+      -1,    98,    -1,    99,    -1,   100,    -1,   101,    -1,   251,
+      -1,   253,   199,   251,    -1,   250,    -1,   256,   215,    -1,
+     264,   215,    -1,   108,   278,   275,   215,    -1,   257,   195,
+      -1,   259,    -1,   258,    -1,   259,   261,    -1,   258,   199,
+     261,    -1,   266,    76,   194,    -1,   274,    76,    -1,   274,
+      76,   196,   254,   197,    -1,   271,   262,   260,    -1,   262,
+     260,    -1,   271,   262,   263,    -1,   262,   263,    -1,    -1,
+      33,    -1,    34,    -1,    35,    -1,   274,    -1,   265,    -1,
+     264,   199,    76,    -1,   264,   199,    76,   196,   197,    -1,
+     264,   199,    76,   196,   254,   197,    -1,   264,   199,    76,
+     196,   197,   214,   284,    -1,   264,   199,    76,   196,   254,
+     197,   214,   284,    -1,   264,   199,    76,   214,   284,    -1,
+     266,    -1,   266,    76,    -1,   266,    76,   196,   197,    -1,
+     266,    76,   196,   254,   197,    -1,   266,    76,   196,   197,
+     214,   284,    -1,   266,    76,   196,   254,   197,   214,   284,
+      -1,   266,    76,   214,   284,    -1,   103,    76,    -1,   274,
+      -1,   272,   274,    -1,   121,   194,   268,   195,    -1,   269,
+      -1,   268,   199,   269,    -1,    76,    -1,    76,   214,    78,
+      -1,    40,    -1,    39,    -1,    38,    -1,     4,    -1,   273,
+      -1,   267,    -1,   267,   273,    -1,   270,    -1,   270,   273,
+      -1,   103,   273,    -1,   103,   270,   273,    -1,   103,    -1,
+       4,    -1,     3,    -1,    37,    -1,    32,    37,    -1,    33,
+      -1,    34,    -1,    32,    33,    -1,    32,    34,    -1,    36,
+      -1,   275,    -1,   278,   275,    -1,   276,    -1,   276,   196,
+     197,    -1,   276,   196,   254,   197,    -1,   277,    -1,   279,
+      -1,    76,    -1,    74,    -1,     6,    -1,     7,    -1,     8,
+      -1,     5,    -1,    29,    -1,    30,    -1,    31,    -1,    20,
+      -1,    21,    -1,    22,    -1,    23,    -1,    24,    -1,    25,
+      -1,    26,    -1,    27,    -1,    28,    -1,    41,    -1,    42,
+      -1,    43,    -1,    44,    -1,    45,    -1,    46,    -1,    47,
+      -1,    48,    -1,    49,    -1,    50,    -1,    51,    -1,   155,
+      -1,    52,    -1,    53,    -1,    54,    -1,    55,    -1,   157,
+      -1,    56,    -1,    57,    -1,    58,    -1,    59,    -1,    60,
+      -1,    61,    -1,    62,    -1,    63,    -1,    64,    -1,    65,
+      -1,    66,    -1,    67,    -1,    68,    -1,    69,    -1,    70,
+      -1,    71,    -1,    72,    -1,   106,    -1,   105,    -1,   104,
+      -1,    73,    76,   216,   280,   217,    -1,    73,   216,   280,
+     217,    -1,   281,    -1,   280,   281,    -1,   274,   282,   215,
+      -1,   283,    -1,   282,   199,   283,    -1,    76,    -1,    76,
+     196,   254,   197,    -1,   251,    -1,   255,    -1,   288,    -1,
+     287,    -1,   285,    -1,   292,    -1,   293,    -1,   296,    -1,
+     297,    -1,   298,    -1,   302,    -1,   216,   217,    -1,   216,
+     291,   217,    -1,   290,    -1,   287,    -1,   216,   217,    -1,
+     216,   291,   217,    -1,   286,    -1,   291,   286,    -1,   215,
+      -1,   253,   215,    -1,    14,   194,   253,   195,   294,    -1,
+     286,    12,   286,    -1,   286,    -1,   253,    -1,   266,    76,
+     214,   284,    -1,    17,   194,   253,   195,   288,    -1,    18,
+     253,   213,    -1,    19,   213,    -1,    75,   194,   295,   195,
+     289,    -1,    11,   286,    75,   194,   253,   195,   215,    -1,
+      13,   194,   299,   301,   195,   289,    -1,   292,    -1,   285,
+      -1,   295,    -1,    -1,   300,   215,    -1,   300,   215,   253,
+      -1,    10,   215,    -1,     9,   215,    -1,    16,   215,    -1,
+      16,   253,   215,    -1,    15,   215,    -1,   304,    -1,   255,
+      -1,   222,    -1,   256,   290,    -1
 };
 
 /* YYRLINE[YYN] -- source line where rule number YYN was defined.  */
 static const yytype_uint16 yyrline[] =
 {
-       0,   212,   212,   211,   218,   220,   240,   241,   242,   243,
-     246,   248,   252,   261,   269,   280,   284,   291,   298,   305,
-     312,   319,   326,   327,   333,   337,   344,   350,   359,   363,
-     367,   368,   377,   378,   382,   383,   387,   393,   405,   409,
-     415,   422,   433,   434,   440,   446,   456,   457,   458,   459,
-     463,   464,   470,   476,   485,   486,   492,   501,   502,   508,
-     517,   518,   524,   530,   536,   545,   546,   552,   561,   562,
-     571,   572,   581,   582,   591,   592,   601,   602,   611,   612,
-     621,   622,   631,   632,   641,   642,   643,   644,   645,   646,
-     647,   648,   649,   650,   651,   655,   659,   675,   679,   683,
-     687,   701,   705,   706,   710,   715,   723,   734,   744,   759,
-     766,   771,   782,   795,   798,   803,   808,   817,   821,   822,
-     831,   840,   849,   858,   867,   880,   891,   900,   909,   918,
-     927,   936,   945,   959,   966,   977,   984,   985,  1004,  1033,
-    1074,  1079,  1084,  1092,  1100,  1101,  1102,  1107,  1108,  1113,
-    1118,  1124,  1132,  1137,  1142,  1147,  1153,  1158,  1163,  1168,
-    1173,  1181,  1182,  1190,  1191,  1197,  1206,  1212,  1218,  1227,
-    1228,  1229,  1230,  1231,  1232,  1233,  1234,  1235,  1236,  1237,
-    1238,  1239,  1240,  1241,  1242,  1243,  1244,  1245,  1246,  1247,
-    1248,  1249,  1250,  1251,  1252,  1253,  1254,  1255,  1256,  1257,
-    1258,  1259,  1260,  1261,  1262,  1263,  1264,  1265,  1266,  1267,
-    1268,  1269,  1270,  1271,  1272,  1273,  1274,  1275,  1276,  1277,
-    1281,  1292,  1303,  1317,  1323,  1332,  1337,  1345,  1360,  1365,
-    1373,  1379,  1388,  1392,  1398,  1399,  1403,  1404,  1405,  1406,
-    1407,  1408,  1409,  1413,  1419,  1428,  1429,  1433,  1439,  1448,
-    1458,  1470,  1476,  1485,  1494,  1499,  1507,  1511,  1525,  1529,
-    1530,  1534,  1541,  1548,  1558,  1559,  1563,  1565,  1571,  1576,
-    1585,  1591,  1597,  1603,  1609,  1618,  1619,  1620,  1624
+       0,   213,   213,   212,   219,   221,   241,   242,   243,   244,
+     245,   260,   262,   266,   275,   283,   294,   298,   305,   312,
+     319,   326,   333,   340,   341,   347,   351,   358,   364,   373,
+     377,   381,   382,   391,   392,   396,   397,   401,   407,   419,
+     423,   429,   436,   447,   448,   454,   460,   470,   471,   472,
+     473,   477,   478,   484,   490,   499,   500,   506,   515,   516,
+     522,   531,   532,   538,   544,   550,   559,   560,   566,   575,
+     576,   585,   586,   595,   596,   605,   606,   615,   616,   625,
+     626,   635,   636,   645,   646,   655,   656,   657,   658,   659,
+     660,   661,   662,   663,   664,   665,   669,   673,   689,   693,
+     697,   701,   715,   719,   720,   724,   729,   737,   748,   758,
+     773,   780,   785,   796,   809,   812,   817,   822,   831,   835,
+     836,   845,   854,   863,   872,   881,   894,   905,   914,   923,
+     932,   941,   950,   959,   973,   980,   991,   998,   999,  1018,
+    1047,  1088,  1093,  1098,  1106,  1114,  1115,  1116,  1121,  1122,
+    1127,  1132,  1138,  1146,  1151,  1156,  1161,  1167,  1172,  1177,
+    1182,  1187,  1195,  1196,  1204,  1205,  1211,  1220,  1226,  1232,
+    1241,  1242,  1243,  1244,  1245,  1246,  1247,  1248,  1249,  1250,
+    1251,  1252,  1253,  1254,  1255,  1256,  1257,  1258,  1259,  1260,
+    1261,  1262,  1263,  1264,  1265,  1266,  1267,  1268,  1269,  1270,
+    1271,  1272,  1273,  1274,  1275,  1276,  1277,  1278,  1279,  1280,
+    1281,  1282,  1283,  1284,  1285,  1286,  1287,  1288,  1289,  1290,
+    1291,  1295,  1306,  1317,  1331,  1337,  1346,  1351,  1359,  1374,
+    1379,  1387,  1393,  1402,  1406,  1412,  1413,  1417,  1418,  1419,
+    1420,  1421,  1422,  1423,  1427,  1433,  1442,  1443,  1447,  1453,
+    1462,  1472,  1484,  1490,  1499,  1508,  1513,  1521,  1525,  1539,
+    1543,  1544,  1548,  1555,  1562,  1572,  1573,  1577,  1579,  1585,
+    1590,  1599,  1605,  1611,  1617,  1623,  1632,  1633,  1634,  1638
 };
 #endif
 
@@ -864,28 +866,28 @@ static const char *const yytname[] =
   "LOWP", "MEDIUMP", "HIGHP", "SUPERP", "PRECISION", "VERSION",
   "EXTENSION", "LINE", "COLON", "EOL", "INTERFACE", "OUTPUT",
   "PRAGMA_DEBUG_ON", "PRAGMA_DEBUG_OFF", "PRAGMA_OPTIMIZE_ON",
-  "PRAGMA_OPTIMIZE_OFF", "LAYOUT_TOK", "ASM", "CLASS", "UNION", "ENUM",
-  "TYPEDEF", "TEMPLATE", "THIS", "PACKED_TOK", "GOTO", "INLINE_TOK",
-  "NOINLINE", "VOLATILE", "PUBLIC_TOK", "STATIC", "EXTERN", "EXTERNAL",
-  "LONG_TOK", "SHORT_TOK", "DOUBLE_TOK", "HALF", "FIXED_TOK", "UNSIGNED",
-  "INPUT_TOK", "OUPTUT", "HVEC2", "HVEC3", "HVEC4", "DVEC2", "DVEC3",
-  "DVEC4", "FVEC2", "FVEC3", "FVEC4", "SAMPLER2DRECT", "SAMPLER3DRECT",
-  "SAMPLER2DRECTSHADOW", "SIZEOF", "CAST", "NAMESPACE", "USING",
-  "ERROR_TOK", "COMMON", "PARTITION", "ACTIVE", "SAMPLERBUFFER", "FILTER",
-  "IMAGE1D", "IMAGE2D", "IMAGE3D", "IMAGECUBE", "IMAGE1DARRAY",
-  "IMAGE2DARRAY", "IIMAGE1D", "IIMAGE2D", "IIMAGE3D", "IIMAGECUBE",
-  "IIMAGE1DARRAY", "IIMAGE2DARRAY", "UIMAGE1D", "UIMAGE2D", "UIMAGE3D",
-  "UIMAGECUBE", "UIMAGE1DARRAY", "UIMAGE2DARRAY", "IMAGE1DSHADOW",
-  "IMAGE2DSHADOW", "IMAGEBUFFER", "IIMAGEBUFFER", "UIMAGEBUFFER",
-  "IMAGE1DARRAYSHADOW", "IMAGE2DARRAYSHADOW", "ROW_MAJOR", "'('", "')'",
-  "'['", "']'", "'.'", "','", "'+'", "'-'", "'!'", "'~'", "'*'", "'/'",
-  "'%'", "'<'", "'>'", "'&'", "'^'", "'|'", "'?'", "':'", "'='", "';'",
-  "'{'", "'}'", "$accept", "translation_unit", "$@1", "version_statement",
-  "pragma_statement", "extension_statement_list", "extension_statement",
-  "external_declaration_list", "variable_identifier", "primary_expression",
-  "postfix_expression", "integer_expression", "function_call",
-  "function_call_or_method", "function_call_generic",
-  "function_call_header_no_parameters",
+  "PRAGMA_OPTIMIZE_OFF", "PRAGMA_INVARIANT_ALL", "LAYOUT_TOK", "ASM",
+  "CLASS", "UNION", "ENUM", "TYPEDEF", "TEMPLATE", "THIS", "PACKED_TOK",
+  "GOTO", "INLINE_TOK", "NOINLINE", "VOLATILE", "PUBLIC_TOK", "STATIC",
+  "EXTERN", "EXTERNAL", "LONG_TOK", "SHORT_TOK", "DOUBLE_TOK", "HALF",
+  "FIXED_TOK", "UNSIGNED", "INPUT_TOK", "OUPTUT", "HVEC2", "HVEC3",
+  "HVEC4", "DVEC2", "DVEC3", "DVEC4", "FVEC2", "FVEC3", "FVEC4",
+  "SAMPLER2DRECT", "SAMPLER3DRECT", "SAMPLER2DRECTSHADOW", "SIZEOF",
+  "CAST", "NAMESPACE", "USING", "ERROR_TOK", "COMMON", "PARTITION",
+  "ACTIVE", "SAMPLERBUFFER", "FILTER", "IMAGE1D", "IMAGE2D", "IMAGE3D",
+  "IMAGECUBE", "IMAGE1DARRAY", "IMAGE2DARRAY", "IIMAGE1D", "IIMAGE2D",
+  "IIMAGE3D", "IIMAGECUBE", "IIMAGE1DARRAY", "IIMAGE2DARRAY", "UIMAGE1D",
+  "UIMAGE2D", "UIMAGE3D", "UIMAGECUBE", "UIMAGE1DARRAY", "UIMAGE2DARRAY",
+  "IMAGE1DSHADOW", "IMAGE2DSHADOW", "IMAGEBUFFER", "IIMAGEBUFFER",
+  "UIMAGEBUFFER", "IMAGE1DARRAYSHADOW", "IMAGE2DARRAYSHADOW", "ROW_MAJOR",
+  "'('", "')'", "'['", "']'", "'.'", "','", "'+'", "'-'", "'!'", "'~'",
+  "'*'", "'/'", "'%'", "'<'", "'>'", "'&'", "'^'", "'|'", "'?'", "':'",
+  "'='", "';'", "'{'", "'}'", "$accept", "translation_unit", "$@1",
+  "version_statement", "pragma_statement", "extension_statement_list",
+  "extension_statement", "external_declaration_list",
+  "variable_identifier", "primary_expression", "postfix_expression",
+  "integer_expression", "function_call", "function_call_or_method",
+  "function_call_generic", "function_call_header_no_parameters",
   "function_call_header_with_parameters", "function_call_header",
   "function_identifier", "unary_expression", "unary_operator",
   "multiplicative_expression", "additive_expression", "shift_expression",
@@ -940,76 +942,76 @@ static const yytype_uint16 yytoknum[] =
      415,   416,   417,   418,   419,   420,   421,   422,   423,   424,
      425,   426,   427,   428,   429,   430,   431,   432,   433,   434,
      435,   436,   437,   438,   439,   440,   441,   442,   443,   444,
-     445,   446,   447,    40,    41,    91,    93,    46,    44,    43,
-      45,    33,   126,    42,    47,    37,    60,    62,    38,    94,
-     124,    63,    58,    61,    59,   123,   125
+     445,   446,   447,   448,    40,    41,    91,    93,    46,    44,
+      43,    45,    33,   126,    42,    47,    37,    60,    62,    38,
+      94,   124,    63,    58,    61,    59,   123,   125
 };
 # endif
 
 /* YYR1[YYN] -- Symbol number of symbol that rule YYN derives.  */
 static const yytype_uint16 yyr1[] =
 {
-       0,   217,   219,   218,   220,   220,   221,   221,   221,   221,
-     222,   222,   223,   224,   224,   225,   226,   226,   226,   226,
-     226,   226,   227,   227,   227,   227,   227,   227,   228,   229,
-     230,   230,   231,   231,   232,   232,   233,   233,   234,   235,
-     235,   235,   236,   236,   236,   236,   237,   237,   237,   237,
-     238,   238,   238,   238,   239,   239,   239,   240,   240,   240,
-     241,   241,   241,   241,   241,   242,   242,   242,   243,   243,
-     244,   244,   245,   245,   246,   246,   247,   247,   248,   248,
-     249,   249,   250,   250,   251,   251,   251,   251,   251,   251,
-     251,   251,   251,   251,   251,   252,   252,   253,   254,   254,
-     254,   255,   256,   256,   257,   257,   258,   259,   259,   260,
-     260,   260,   260,   261,   261,   261,   261,   262,   263,   263,
-     263,   263,   263,   263,   263,   264,   264,   264,   264,   264,
-     264,   264,   264,   265,   265,   266,   267,   267,   268,   268,
-     269,   269,   269,   270,   271,   271,   271,   271,   271,   271,
-     271,   271,   272,   272,   272,   272,   272,   272,   272,   272,
-     272,   273,   273,   274,   274,   274,   275,   275,   275,   276,
-     276,   276,   276,   276,   276,   276,   276,   276,   276,   276,
-     276,   276,   276,   276,   276,   276,   276,   276,   276,   276,
-     276,   276,   276,   276,   276,   276,   276,   276,   276,   276,
-     276,   276,   276,   276,   276,   276,   276,   276,   276,   276,
-     276,   276,   276,   276,   276,   276,   276,   276,   276,   276,
-     277,   277,   277,   278,   278,   279,   279,   280,   281,   281,
-     282,   282,   283,   284,   285,   285,   286,   286,   286,   286,
-     286,   286,   286,   287,   287,   288,   288,   289,   289,   290,
-     290,   291,   291,   292,   293,   293,   294,   294,   295,   296,
-     296,   297,   297,   297,   298,   298,   299,   299,   300,   300,
-     301,   301,   301,   301,   301,   302,   302,   302,   303
+       0,   218,   220,   219,   221,   221,   222,   222,   222,   222,
+     222,   223,   223,   224,   225,   225,   226,   227,   227,   227,
+     227,   227,   227,   228,   228,   228,   228,   228,   228,   229,
+     230,   231,   231,   232,   232,   233,   233,   234,   234,   235,
+     236,   236,   236,   237,   237,   237,   237,   238,   238,   238,
+     238,   239,   239,   239,   239,   240,   240,   240,   241,   241,
+     241,   242,   242,   242,   242,   242,   243,   243,   243,   244,
+     244,   245,   245,   246,   246,   247,   247,   248,   248,   249,
+     249,   250,   250,   251,   251,   252,   252,   252,   252,   252,
+     252,   252,   252,   252,   252,   252,   253,   253,   254,   255,
+     255,   255,   256,   257,   257,   258,   258,   259,   260,   260,
+     261,   261,   261,   261,   262,   262,   262,   262,   263,   264,
+     264,   264,   264,   264,   264,   264,   265,   265,   265,   265,
+     265,   265,   265,   265,   266,   266,   267,   268,   268,   269,
+     269,   270,   270,   270,   271,   272,   272,   272,   272,   272,
+     272,   272,   272,   273,   273,   273,   273,   273,   273,   273,
+     273,   273,   274,   274,   275,   275,   275,   276,   276,   276,
+     277,   277,   277,   277,   277,   277,   277,   277,   277,   277,
+     277,   277,   277,   277,   277,   277,   277,   277,   277,   277,
+     277,   277,   277,   277,   277,   277,   277,   277,   277,   277,
+     277,   277,   277,   277,   277,   277,   277,   277,   277,   277,
+     277,   277,   277,   277,   277,   277,   277,   277,   277,   277,
+     277,   278,   278,   278,   279,   279,   280,   280,   281,   282,
+     282,   283,   283,   284,   285,   286,   286,   287,   287,   287,
+     287,   287,   287,   287,   288,   288,   289,   289,   290,   290,
+     291,   291,   292,   292,   293,   294,   294,   295,   295,   296,
+     297,   297,   298,   298,   298,   299,   299,   300,   300,   301,
+     301,   302,   302,   302,   302,   302,   303,   303,   303,   304
 };
 
 /* YYR2[YYN] -- Number of symbols composing right hand side of rule YYN.  */
 static const yytype_uint8 yyr2[] =
 {
        0,     2,     0,     4,     0,     3,     2,     2,     2,     2,
-       0,     2,     5,     1,     2,     1,     1,     1,     1,     1,
-       1,     3,     1,     4,     1,     3,     2,     2,     1,     1,
-       1,     3,     2,     2,     2,     1,     2,     3,     2,     1,
-       1,     1,     1,     2,     2,     2,     1,     1,     1,     1,
-       1,     3,     3,     3,     1,     3,     3,     1,     3,     3,
-       1,     3,     3,     3,     3,     1,     3,     3,     1,     3,
-       1,     3,     1,     3,     1,     3,     1,     3,     1,     3,
-       1,     5,     1,     3,     1,     1,     1,     1,     1,     1,
-       1,     1,     1,     1,     1,     1,     3,     1,     2,     2,
-       4,     2,     1,     1,     2,     3,     3,     2,     5,     3,
-       2,     3,     2,     0,     1,     1,     1,     1,     1,     3,
-       5,     6,     7,     8,     5,     1,     2,     4,     5,     6,
-       7,     4,     2,     1,     2,     4,     1,     3,     1,     3,
-       1,     1,     1,     1,     1,     1,     2,     1,     2,     2,
-       3,     1,     1,     1,     1,     2,     1,     1,     2,     2,
-       1,     1,     2,     1,     3,     4,     1,     1,     1,     1,
+       2,     0,     2,     5,     1,     2,     1,     1,     1,     1,
+       1,     1,     3,     1,     4,     1,     3,     2,     2,     1,
+       1,     1,     3,     2,     2,     2,     1,     2,     3,     2,
+       1,     1,     1,     1,     2,     2,     2,     1,     1,     1,
+       1,     1,     3,     3,     3,     1,     3,     3,     1,     3,
+       3,     1,     3,     3,     3,     3,     1,     3,     3,     1,
+       3,     1,     3,     1,     3,     1,     3,     1,     3,     1,
+       3,     1,     5,     1,     3,     1,     1,     1,     1,     1,
+       1,     1,     1,     1,     1,     1,     1,     3,     1,     2,
+       2,     4,     2,     1,     1,     2,     3,     3,     2,     5,
+       3,     2,     3,     2,     0,     1,     1,     1,     1,     1,
+       3,     5,     6,     7,     8,     5,     1,     2,     4,     5,
+       6,     7,     4,     2,     1,     2,     4,     1,     3,     1,
+       3,     1,     1,     1,     1,     1,     1,     2,     1,     2,
+       2,     3,     1,     1,     1,     1,     2,     1,     1,     2,
+       2,     1,     1,     2,     1,     3,     4,     1,     1,     1,
        1,     1,     1,     1,     1,     1,     1,     1,     1,     1,
        1,     1,     1,     1,     1,     1,     1,     1,     1,     1,
        1,     1,     1,     1,     1,     1,     1,     1,     1,     1,
        1,     1,     1,     1,     1,     1,     1,     1,     1,     1,
        1,     1,     1,     1,     1,     1,     1,     1,     1,     1,
-       1,     1,     1,     5,     4,     1,     2,     3,     1,     3,
-       1,     4,     1,     1,     1,     1,     1,     1,     1,     1,
-       1,     1,     1,     2,     3,     1,     1,     2,     3,     1,
-       2,     1,     2,     5,     3,     1,     1,     4,     5,     3,
-       2,     5,     7,     6,     1,     1,     1,     0,     2,     3,
-       2,     2,     2,     3,     2,     1,     1,     1,     2
+       1,     1,     1,     1,     5,     4,     1,     2,     3,     1,
+       3,     1,     4,     1,     1,     1,     1,     1,     1,     1,
+       1,     1,     1,     1,     2,     3,     1,     1,     2,     3,
+       1,     2,     1,     2,     5,     3,     1,     1,     4,     5,
+       3,     2,     5,     7,     6,     1,     1,     1,     0,     2,
+       3,     2,     2,     2,     3,     2,     1,     1,     1,     2
 };
 
 /* YYDEFACT[STATE-NAME] -- Default rule to reduce with in state
@@ -1017,614 +1019,638 @@ static const yytype_uint8 yyr2[] =
    means the default is an error.  */
 static const yytype_uint16 yydefact[] =
 {
-       4,     0,     0,    10,     0,     1,     2,     5,     0,     0,
-      11,     0,   153,   152,   173,   170,   171,   172,   177,   178,
-     179,   180,   181,   182,   183,   184,   185,   174,   175,   176,
-       0,   156,   157,   160,   154,   142,   141,   140,   186,   187,
-     188,   189,   190,   191,   192,   193,   194,   195,   196,   198,
-     199,   200,   201,   203,   204,   205,   206,   207,   208,   209,
-     210,   211,   212,   213,   214,   215,   216,   217,   218,   219,
-       0,   169,   168,   151,   222,   221,   220,     0,     0,     0,
-       0,     0,     0,   197,   202,   277,     3,   276,     0,     0,
-     103,   113,     0,   118,   125,   145,   147,     0,   144,   133,
-     161,   163,   166,     0,   167,    13,   275,     0,   158,   159,
-     155,     0,     0,   132,     0,   149,     0,     6,     7,     8,
-       9,     0,    14,    98,     0,   278,   101,   113,   143,   114,
-     115,   116,   104,     0,   113,     0,    99,   126,   146,   148,
-     134,     0,   162,     0,     0,     0,     0,   225,   150,     0,
-     138,     0,   136,     0,     0,     0,     0,     0,     0,     0,
-       0,     0,     0,     0,    15,    19,    17,    18,    20,    41,
-       0,     0,     0,    46,    47,    48,    49,   251,     0,   247,
-      16,    22,    42,    24,    29,    30,     0,     0,    35,     0,
-      50,     0,    54,    57,    60,    65,    68,    70,    72,    74,
-      76,    78,    80,    82,    95,     0,   233,     0,   133,   236,
-     249,   235,   234,     0,   237,   238,   239,   240,   241,   242,
-     105,   110,   112,   117,     0,   119,   106,     0,     0,   164,
-      50,    97,     0,    39,    12,     0,   230,     0,   228,   224,
-     226,   100,     0,   135,     0,   271,   270,     0,     0,     0,
-     274,   272,     0,     0,     0,   260,     0,    43,    44,     0,
-     243,     0,    26,    27,     0,     0,    33,    32,     0,   169,
-      36,    38,    85,    86,    88,    87,    90,    91,    92,    93,
-      94,    89,    84,     0,    45,     0,     0,     0,     0,     0,
+       4,     0,     0,    11,     0,     1,     2,     5,     0,     0,
+      12,     0,   154,   153,   174,   171,   172,   173,   178,   179,
+     180,   181,   182,   183,   184,   185,   186,   175,   176,   177,
+       0,   157,   158,   161,   155,   143,   142,   141,   187,   188,
+     189,   190,   191,   192,   193,   194,   195,   196,   197,   199,
+     200,   201,   202,   204,   205,   206,   207,   208,   209,   210,
+     211,   212,   213,   214,   215,   216,   217,   218,   219,   220,
+       0,   170,   169,   152,   223,   222,   221,     0,     0,     0,
+       0,     0,     0,     0,   198,   203,   278,     3,   277,     0,
+       0,   104,   114,     0,   119,   126,   146,   148,     0,   145,
+     134,   162,   164,   167,     0,   168,    14,   276,     0,   159,
+     160,   156,     0,     0,   133,     0,   150,     0,     6,     7,
+       8,     9,    10,     0,    15,    99,     0,   279,   102,   114,
+     144,   115,   116,   117,   105,     0,   114,     0,   100,   127,
+     147,   149,   135,     0,   163,     0,     0,     0,     0,   226,
+     151,     0,   139,     0,   137,     0,     0,     0,     0,     0,
+       0,     0,     0,     0,     0,     0,    16,    20,    18,    19,
+      21,    42,     0,     0,     0,    47,    48,    49,    50,   252,
+       0,   248,    17,    23,    43,    25,    30,    31,     0,     0,
+      36,     0,    51,     0,    55,    58,    61,    66,    69,    71,
+      73,    75,    77,    79,    81,    83,    96,     0,   234,     0,
+     134,   237,   250,   236,   235,     0,   238,   239,   240,   241,
+     242,   243,   106,   111,   113,   118,     0,   120,   107,     0,
+       0,   165,    51,    98,     0,    40,    13,     0,   231,     0,
+     229,   225,   227,   101,     0,   136,     0,   272,   271,     0,
+       0,     0,   275,   273,     0,     0,     0,   261,     0,    44,
+      45,     0,   244,     0,    27,    28,     0,     0,    34,    33,
+       0,   170,    37,    39,    86,    87,    89,    88,    91,    92,
+      93,    94,    95,    90,    85,     0,    46,     0,     0,     0,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
-       0,     0,     0,     0,     0,     0,   252,   248,   250,   107,
-     109,   111,     0,     0,   127,     0,   232,   131,   165,   223,
-       0,     0,   227,   139,   137,     0,   265,   264,   267,     0,
-     273,     0,   259,   151,   256,     0,     0,    21,   244,     0,
-      28,    25,    31,    37,    83,    51,    52,    53,    55,    56,
-      58,    59,    63,    64,    61,    62,    66,    67,    69,    71,
-      73,    75,    77,    79,     0,    96,     0,   120,     0,   124,
-       0,   128,     0,   229,     0,   266,     0,     0,     0,     0,
-       0,     0,    23,     0,     0,     0,   121,   129,     0,   231,
-       0,   268,     0,   255,   253,   258,     0,   246,   261,   245,
-      81,   108,   122,     0,   130,     0,   269,   263,     0,   257,
-     123,   262,   254
+       0,     0,     0,     0,     0,     0,     0,     0,   253,   249,
+     251,   108,   110,   112,     0,     0,   128,     0,   233,   132,
+     166,   224,     0,     0,   228,   140,   138,     0,   266,   265,
+     268,     0,   274,     0,   260,   152,   257,     0,     0,    22,
+     245,     0,    29,    26,    32,    38,    84,    52,    53,    54,
+      56,    57,    59,    60,    64,    65,    62,    63,    67,    68,
+      70,    72,    74,    76,    78,    80,     0,    97,     0,   121,
+       0,   125,     0,   129,     0,   230,     0,   267,     0,     0,
+       0,     0,     0,     0,    24,     0,     0,     0,   122,   130,
+       0,   232,     0,   269,     0,   256,   254,   259,     0,   247,
+     262,   246,    82,   109,   123,     0,   131,     0,   270,   264,
+       0,   258,   124,   263,   255
 };
 
 /* YYDEFGOTO[NTERM-NUM].  */
 static const yytype_int16 yydefgoto[] =
 {
-      -1,     2,     9,     3,    85,     6,    10,    86,   180,   181,
-     182,   339,   183,   184,   185,   186,   187,   188,   189,   190,
-     191,   192,   193,   194,   195,   196,   197,   198,   199,   200,
-     201,   202,   203,   204,   283,   205,   232,   206,   207,    89,
-      90,    91,   221,   132,   133,   222,    92,    93,    94,    95,
-     151,   152,    96,   134,    97,    98,   233,   100,   101,   102,
-     103,   104,   146,   147,   237,   238,   317,   209,   210,   211,
-     212,   398,   399,   213,   214,   215,   394,   336,   216,   217,
-     218,   328,   376,   377,   219,   105,   106
+      -1,     2,     9,     3,    86,     6,    10,    87,   182,   183,
+     184,   341,   185,   186,   187,   188,   189,   190,   191,   192,
+     193,   194,   195,   196,   197,   198,   199,   200,   201,   202,
+     203,   204,   205,   206,   285,   207,   234,   208,   209,    90,
+      91,    92,   223,   134,   135,   224,    93,    94,    95,    96,
+     153,   154,    97,   136,    98,    99,   235,   101,   102,   103,
+     104,   105,   148,   149,   239,   240,   319,   211,   212,   213,
+     214,   400,   401,   215,   216,   217,   396,   338,   218,   219,
+     220,   330,   378,   379,   221,   106,   107
 };
 
 /* YYPACT[STATE-NUM] -- Index in YYTABLE of the portion describing
    STATE-NUM.  */
-#define YYPACT_NINF -350
+#define YYPACT_NINF -321
 static const yytype_int16 yypact[] =
 {
-     -78,   -56,    54,  -350,   -52,  -350,   -37,  -350,     8,  3302,
-    -350,   -26,  -350,  -350,  -350,  -350,  -350,  -350,  -350,  -350,
-    -350,  -350,  -350,  -350,  -350,  -350,  -350,  -350,  -350,  -350,
-      88,  -350,  -350,  -350,  -350,  -350,  -350,  -350,  -350,  -350,
-    -350,  -350,  -350,  -350,  -350,  -350,  -350,  -350,  -350,  -350,
-    -350,  -350,  -350,  -350,  -350,  -350,  -350,  -350,  -350,  -350,
-    -350,  -350,  -350,  -350,  -350,  -350,  -350,  -350,  -350,  -350,
-     -66,  -350,  -350,    42,  -350,  -350,  -350,    70,    -4,    10,
-      29,    34,   -79,  -350,  -350,  -350,  3302,  -350,   -19,   -24,
-     -69,     5,  -154,  -350,   102,    16,    16,  3490,  -350,  -350,
-    -350,    18,  -350,  3562,  -350,  -350,  -350,   108,  -350,  -350,
-    -350,    -8,  3490,  -350,    16,  -350,  3562,  -350,  -350,  -350,
-    -350,   138,  -350,  -350,   387,  -350,  -350,    24,  -350,  -350,
-    -350,  -350,  -350,  3490,   147,   141,  -350,  -166,  -350,  -350,
-    -350,  2387,  -350,   106,  3490,   144,  1772,  -350,  -350,     7,
-      11,   -87,  -350,    14,    15,  1243,    30,    32,    20,  2004,
-      37,  2936,    25,    39,   -65,  -350,  -350,  -350,  -350,  -350,
-    2936,  2936,  2936,  -350,  -350,  -350,  -350,  -350,   601,  -350,
-    -350,  -350,   -59,  -350,  -350,  -350,    28,   -82,  3119,    43,
-     -30,  2936,   -11,    -2,   118,   -74,   114,    35,    31,    36,
-     148,   152,   -77,  -350,  -350,  -115,  -350,    40,    52,  -350,
-    -350,  -350,  -350,   815,  -350,  -350,  -350,  -350,  -350,  -350,
-    -350,  -350,  -350,   172,  3490,  -180,  -350,  2570,  2936,  -350,
-    -350,  -350,    53,  -350,  -350,  1888,    55,  -113,  -350,  -350,
-    -350,  -350,   173,  -350,   138,  -350,  -350,   178,  1656,  2936,
-    -350,  -350,  -108,  2936,  -161,  -350,  2204,  -350,  -350,   -68,
-    -350,  1029,  -350,  -350,  2936,  3418,  -350,  -350,  2936,    61,
-    -350,  -350,  -350,  -350,  -350,  -350,  -350,  -350,  -350,  -350,
-    -350,  -350,  -350,  2936,  -350,  2936,  2936,  2936,  2936,  2936,
-    2936,  2936,  2936,  2936,  2936,  2936,  2936,  2936,  2936,  2936,
-    2936,  2936,  2936,  2936,  2936,  2936,  -350,  -350,  -350,    63,
-    -350,  -350,  2753,  2936,    46,    60,  -350,  -350,  -350,  -350,
-    2936,   144,  -350,  -350,  -350,    67,  -350,  -350,  2204,   -55,
-    -350,   -54,  -350,   238,    65,   188,    71,  -350,  -350,    72,
-      65,    73,  -350,  -350,  -350,  -350,  -350,  -350,   -11,   -11,
-      -2,    -2,   118,   118,   118,   118,   -74,   -74,   114,    35,
-      31,    36,   148,   152,  -157,  -350,  2936,    56,    83,  -350,
-    2936,    68,    84,  -350,  2936,  -350,    69,    90,  1243,    74,
-      77,  1456,  -350,  2936,    86,  2936,    79,  -350,  2936,  -350,
-     -53,  2936,  1456,   255,  -350,  -350,  2936,  -350,  -350,  -350,
-    -350,  -350,  -350,  2936,  -350,    80,    65,  -350,  1243,  -350,
-    -350,  -350,  -350
+     -86,   -57,    45,  -321,   -56,  -321,   -50,  -321,   -10,  3320,
+    -321,   -26,  -321,  -321,  -321,  -321,  -321,  -321,  -321,  -321,
+    -321,  -321,  -321,  -321,  -321,  -321,  -321,  -321,  -321,  -321,
+      79,  -321,  -321,  -321,  -321,  -321,  -321,  -321,  -321,  -321,
+    -321,  -321,  -321,  -321,  -321,  -321,  -321,  -321,  -321,  -321,
+    -321,  -321,  -321,  -321,  -321,  -321,  -321,  -321,  -321,  -321,
+    -321,  -321,  -321,  -321,  -321,  -321,  -321,  -321,  -321,  -321,
+     -70,  -321,  -321,    43,  -321,  -321,  -321,    18,   -22,   -12,
+      -5,     2,    25,  -101,  -321,  -321,  -321,  3320,  -321,   -43,
+     -55,   -54,     6,  -148,  -321,    52,   211,   211,  3509,  -321,
+    -321,  -321,   -47,  -321,  3581,  -321,  -321,  -321,   110,  -321,
+    -321,  -321,   -14,  3509,  -321,   211,  -321,  3581,  -321,  -321,
+    -321,  -321,  -321,   131,  -321,  -321,   389,  -321,  -321,    15,
+    -321,  -321,  -321,  -321,  -321,  3509,   109,   136,  -321,  -152,
+    -321,  -321,  -321,  2400,  -321,   105,  3509,   143,  1781,  -321,
+    -321,     7,     9,  -107,  -321,    10,    12,  1249,    27,    36,
+      17,  2015,    40,  2952,    22,    42,   -65,  -321,  -321,  -321,
+    -321,  -321,  2952,  2952,  2952,  -321,  -321,  -321,  -321,  -321,
+     604,  -321,  -321,  -321,   -20,  -321,  -321,  -321,    47,   -92,
+    3136,    46,   -67,  2952,   -24,   -16,   111,   -73,   108,    37,
+      41,    39,   162,   161,   -82,  -321,  -321,  -147,  -321,    44,
+      61,  -321,  -321,  -321,  -321,   819,  -321,  -321,  -321,  -321,
+    -321,  -321,  -321,  -321,  -321,   181,  3509,  -160,  -321,  2584,
+    2952,  -321,  -321,  -321,    63,  -321,  -321,  1898,    62,  -146,
+    -321,  -321,  -321,  -321,   183,  -321,   131,  -321,  -321,   187,
+    1664,  2952,  -321,  -321,  -144,  2952,  -140,  -321,  2216,  -321,
+    -321,   -81,  -321,  1034,  -321,  -321,  2952,  3437,  -321,  -321,
+    2952,    70,  -321,  -321,  -321,  -321,  -321,  -321,  -321,  -321,
+    -321,  -321,  -321,  -321,  -321,  2952,  -321,  2952,  2952,  2952,
+    2952,  2952,  2952,  2952,  2952,  2952,  2952,  2952,  2952,  2952,
+    2952,  2952,  2952,  2952,  2952,  2952,  2952,  2952,  -321,  -321,
+    -321,    74,  -321,  -321,  2768,  2952,    64,    69,  -321,  -321,
+    -321,  -321,  2952,   143,  -321,  -321,  -321,    82,  -321,  -321,
+    2216,   -74,  -321,   -68,  -321,   235,    78,   203,    85,  -321,
+    -321,    84,    78,    88,  -321,  -321,  -321,  -321,  -321,  -321,
+     -24,   -24,   -16,   -16,   111,   111,   111,   111,   -73,   -73,
+     108,    37,    41,    39,   162,   161,  -129,  -321,  2952,    71,
+      86,  -321,  2952,    72,    87,  -321,  2952,  -321,    73,    92,
+    1249,    75,    76,  1463,  -321,  2952,    95,  2952,    80,  -321,
+    2952,  -321,   -63,  2952,  1463,   277,  -321,  -321,  2952,  -321,
+    -321,  -321,  -321,  -321,  -321,  2952,  -321,    81,    78,  -321,
+    1249,  -321,  -321,  -321,  -321
 };
 
 /* YYPGOTO[NTERM-NUM].  */
 static const yytype_int16 yypgoto[] =
 {
-    -350,  -350,  -350,  -350,  -350,  -350,  -350,  -350,  -350,  -350,
-    -350,  -350,  -350,  -350,    22,  -350,  -350,  -350,  -350,  -135,
-    -350,   -83,   -81,  -104,   -85,   -13,    -6,    -5,    -3,    -1,
-      -7,  -350,  -133,   -97,  -350,  -156,  -193,     9,    12,  -350,
-    -350,  -350,    76,   170,   168,    81,  -350,  -350,  -239,  -350,
-    -350,    59,   -71,  -350,  -350,   -72,    -9,     1,  -350,  -350,
-     227,  -350,   163,  -139,  -350,   -12,  -283,    62,  -151,  -349,
-     -67,   -84,   223,   135,    66,  -350,  -350,   -10,  -350,  -350,
-    -350,  -350,  -350,  -350,  -350,   229,  -350
+    -321,  -321,  -321,  -321,  -321,  -321,  -321,  -321,  -321,  -321,
+    -321,  -321,  -321,  -321,    26,  -321,  -321,  -321,  -321,  -135,
+    -321,   -91,   -88,  -106,   -90,    -3,    -6,    -4,    -2,    -1,
+       0,  -321,  -139,  -174,  -321,  -156,  -217,    11,    13,  -321,
+    -321,  -321,    83,   170,   164,    89,  -321,  -321,  -243,  -321,
+    -321,    56,   -71,  -321,  -321,   -72,    -9,   -32,  -321,  -321,
+     227,  -321,   160,  -131,  -321,   -15,  -195,    57,  -154,  -320,
+     -69,   -84,   222,   133,    66,  -321,  -321,   -13,  -321,  -321,
+    -321,  -321,  -321,  -321,  -321,   231,  -321
 };
 
 /* YYTABLE[YYPACT[STATE-NUM]].  What to do in state STATE-NUM.  If
    positive, shift that token.  If negative, reduce the rule which
    number is the opposite.  If zero, do what YYDEFACT says.
    If YYTABLE_NINF, syntax error.  */
-#define YYTABLE_NINF -169
+#define YYTABLE_NINF -170
 static const yytype_int16 yytable[] =
 {
-      99,   115,   114,   252,   247,   254,   230,   240,   231,   128,
-     111,  -168,   292,   293,   303,   312,   259,   335,    87,    12,
-      13,    88,     4,   138,   139,   262,   263,   226,   128,   227,
-     369,     1,   397,   313,   315,   257,   258,   305,   129,   130,
-     131,   305,   148,   397,   135,    12,    13,   228,    30,    31,
-      32,   332,    33,    34,     5,   383,   284,   129,   130,   131,
-     136,     7,   308,   272,   273,   274,   275,   276,   277,   278,
-     279,   280,   281,     8,    30,    31,    32,    99,    33,    34,
-      35,    36,    37,   305,    11,   321,   107,   387,   140,   335,
-     305,   270,   230,   329,   231,    87,   240,   331,    88,   306,
-     334,   322,   402,   145,   142,   404,   330,   243,   340,   117,
-     308,   244,   267,   409,   121,   208,   268,   149,   113,   368,
-     410,   108,   109,   118,   223,   110,   337,   372,   -40,   127,
-     305,   316,   294,   295,   304,   145,   264,   145,   265,   378,
-     379,   405,   119,   305,   305,   305,   208,   120,   364,   112,
-     345,   346,   347,   230,   230,   230,   230,   230,   230,   230,
-     230,   230,   230,   230,   230,   230,   230,   230,   230,   208,
-     126,   343,   334,   384,    74,    75,    76,   230,   137,   231,
-     129,   130,   131,   282,   143,   230,   344,   231,   352,   353,
-     354,   355,   285,   286,   287,   123,   124,   288,   289,  -102,
-     290,   291,   296,   297,   208,   348,   349,   144,   365,   350,
-     351,   356,   357,   141,   150,   223,   316,   225,   390,   234,
-     236,   241,   266,   248,   242,   249,   145,   393,   245,   246,
-     253,   230,   256,   231,   250,   406,   271,   255,   301,   208,
-     299,    12,    13,   298,   302,   -39,   300,   208,   309,   318,
-     320,   323,   208,   325,   123,   -34,   371,   412,   366,   370,
-     374,   115,   114,   305,   380,   381,   -40,   408,   382,   385,
-      30,    31,    32,   316,    33,    34,    35,    36,    37,   386,
-     389,   388,   401,   391,   392,   358,   400,   342,   316,   178,
-     396,   316,   403,   359,   411,   360,   363,   220,   361,   316,
-     310,   362,   224,   324,   116,   311,   316,   235,   407,   373,
-     326,   125,   395,   261,   327,   122,     0,     0,   375,   208,
+     100,   116,   115,   249,   233,   254,   112,   256,   232,   305,
+     130,  -169,   317,   294,   295,   337,   272,   242,   261,   130,
+      88,     4,    89,     1,   140,   141,   274,   275,   276,   277,
+     278,   279,   280,   281,   282,   283,   314,   259,   260,   131,
+     132,   133,   228,   150,   229,     5,    12,    13,   131,   132,
+     133,   137,   307,   323,   315,   307,   318,     7,   286,   307,
+       8,   310,   230,   399,   264,   265,    11,   138,   308,   324,
+     307,   332,   144,   334,   399,    30,    31,    32,   100,    33,
+      34,    35,    36,    37,   385,   151,   108,   337,   245,   142,
+     233,   118,   246,   123,   232,   331,   345,   370,    88,   333,
+      89,   119,   336,   269,   147,   374,   242,   270,   120,   310,
+     342,   346,   109,   110,   339,   121,   111,   210,   307,   114,
+     371,   380,    74,    75,    76,   307,   225,   381,   139,   -41,
+     306,   307,   407,   367,   296,   297,   307,   147,   122,   147,
+     128,   318,   131,   132,   133,   129,   113,   284,   210,   143,
+     366,   386,   347,   348,   349,   232,   232,   232,   232,   232,
+     232,   232,   232,   232,   232,   232,   232,   232,   232,   232,
+     232,   210,   125,   126,   336,   233,   266,   389,   267,   232,
+     287,   288,   289,   233,   290,   291,   145,   232,   354,   355,
+     356,   357,   404,   292,   293,   406,   298,   299,   318,   350,
+     351,  -103,   146,   411,   352,   353,   210,   152,   358,   359,
+     412,   402,   227,   318,    12,    13,   318,   225,   236,   238,
+     392,   250,   243,   244,   318,   247,   395,   248,   147,   233,
+     251,   318,   252,   232,   255,   257,   258,   408,    12,    13,
+     273,   210,   268,    30,    31,    32,   300,    33,    34,   210,
+     302,   301,   303,   304,   210,   -40,   414,   311,   322,   125,
+     320,   325,   327,   116,   115,   -35,   373,    30,    31,    32,
+     368,    33,    34,    35,    36,    37,   376,   307,   372,   382,
+     383,   384,   -41,   388,   391,   387,   390,   394,   393,   410,
+     398,   180,   403,   344,   405,   361,   413,   360,   362,   222,
+     226,   363,   326,   364,   117,   365,   237,   328,   375,   312,
+     409,   127,   397,   263,     0,   313,   329,   377,   124,     0,
+       0,   210,     0,     0,     0,     0,     0,     0,     0,     0,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
-       0,     0,     0,     0,     0,     0,     0,     0,     0,   208,
-       0,     0,   208,     0,     0,     0,     0,     0,     0,     0,
-       0,     0,     0,   208,     0,     0,     0,     0,     0,     0,
-      12,    13,    14,    15,    16,    17,   153,   154,   155,   208,
-     156,   157,   158,   159,   160,   161,   162,    18,    19,    20,
-      21,    22,    23,    24,    25,    26,    27,    28,    29,    30,
-      31,    32,     0,    33,    34,    35,    36,    37,    38,    39,
-      40,    41,    42,    43,    44,    45,    46,    47,    48,    49,
-      50,    51,    52,    53,    54,    55,    56,    57,    58,    59,
-      60,    61,    62,    63,    64,    65,    66,    67,    68,    69,
-      70,    71,   163,   164,   165,   166,   167,   168,   169,     0,
-       0,   170,   171,     0,     0,     0,     0,     0,     0,     0,
+       0,   210,     0,     0,   210,     0,     0,     0,     0,     0,
+       0,     0,     0,     0,     0,   210,     0,     0,     0,     0,
+       0,     0,    12,    13,    14,    15,    16,    17,   155,   156,
+     157,   210,   158,   159,   160,   161,   162,   163,   164,    18,
+      19,    20,    21,    22,    23,    24,    25,    26,    27,    28,
+      29,    30,    31,    32,     0,    33,    34,    35,    36,    37,
+      38,    39,    40,    41,    42,    43,    44,    45,    46,    47,
+      48,    49,    50,    51,    52,    53,    54,    55,    56,    57,
+      58,    59,    60,    61,    62,    63,    64,    65,    66,    67,
+      68,    69,    70,    71,   165,   166,   167,   168,   169,   170,
+     171,     0,     0,   172,   173,     0,     0,     0,     0,     0,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
-      73,    74,    75,    76,     0,    77,     0,     0,     0,     0,
-       0,     0,     0,     0,     0,     0,     0,    82,     0,     0,
+       0,     0,    73,    74,    75,    76,     0,    77,     0,     0,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
+      83,     0,     0,     0,     0,     0,     0,     0,     0,     0,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
-       0,    83,     0,    84,     0,     0,     0,     0,     0,     0,
+       0,     0,     0,     0,    84,     0,    85,     0,     0,     0,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
-     172,     0,     0,     0,     0,     0,   173,   174,   175,   176,
+       0,     0,     0,   174,     0,     0,     0,     0,     0,   175,
+     176,   177,   178,     0,     0,     0,     0,     0,     0,     0,
+       0,     0,     0,     0,   179,   180,   181,    12,    13,    14,
+      15,    16,    17,   155,   156,   157,     0,   158,   159,   160,
+     161,   162,   163,   164,    18,    19,    20,    21,    22,    23,
+      24,    25,    26,    27,    28,    29,    30,    31,    32,     0,
+      33,    34,    35,    36,    37,    38,    39,    40,    41,    42,
+      43,    44,    45,    46,    47,    48,    49,    50,    51,    52,
+      53,    54,    55,    56,    57,    58,    59,    60,    61,    62,
+      63,    64,    65,    66,    67,    68,    69,    70,    71,   165,
+     166,   167,   168,   169,   170,   171,     0,     0,   172,   173,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
-       0,   177,   178,   179,    12,    13,    14,    15,    16,    17,
-     153,   154,   155,     0,   156,   157,   158,   159,   160,   161,
-     162,    18,    19,    20,    21,    22,    23,    24,    25,    26,
-      27,    28,    29,    30,    31,    32,     0,    33,    34,    35,
-      36,    37,    38,    39,    40,    41,    42,    43,    44,    45,
-      46,    47,    48,    49,    50,    51,    52,    53,    54,    55,
-      56,    57,    58,    59,    60,    61,    62,    63,    64,    65,
-      66,    67,    68,    69,    70,    71,   163,   164,   165,   166,
-     167,   168,   169,     0,     0,   170,   171,     0,     0,     0,
+       0,     0,     0,     0,     0,     0,     0,    73,    74,    75,
+      76,     0,    77,     0,     0,     0,     0,     0,     0,     0,
+       0,     0,     0,     0,     0,    83,     0,     0,     0,     0,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
-       0,     0,     0,     0,    73,    74,    75,    76,     0,    77,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
-       0,    82,     0,     0,     0,     0,     0,     0,     0,     0,
+       0,     0,     0,     0,     0,     0,     0,     0,     0,    84,
+       0,    85,     0,     0,     0,     0,     0,     0,     0,     0,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
-       0,     0,     0,     0,     0,    83,     0,    84,     0,     0,
+       0,     0,     0,     0,     0,     0,     0,     0,   174,     0,
+       0,     0,     0,     0,   175,   176,   177,   178,     0,     0,
+       0,     0,     0,     0,     0,     0,     0,     0,     0,   179,
+     180,   262,    12,    13,    14,    15,    16,    17,   155,   156,
+     157,     0,   158,   159,   160,   161,   162,   163,   164,    18,
+      19,    20,    21,    22,    23,    24,    25,    26,    27,    28,
+      29,    30,    31,    32,     0,    33,    34,    35,    36,    37,
+      38,    39,    40,    41,    42,    43,    44,    45,    46,    47,
+      48,    49,    50,    51,    52,    53,    54,    55,    56,    57,
+      58,    59,    60,    61,    62,    63,    64,    65,    66,    67,
+      68,    69,    70,    71,   165,   166,   167,   168,   169,   170,
+     171,     0,     0,   172,   173,     0,     0,     0,     0,     0,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
+       0,     0,    73,    74,    75,    76,     0,    77,     0,     0,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
+      83,     0,     0,     0,     0,     0,     0,     0,     0,     0,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
-       0,     0,     0,     0,   172,     0,     0,     0,     0,     0,
-     173,   174,   175,   176,     0,     0,     0,     0,     0,     0,
-       0,     0,     0,     0,     0,   177,   178,   260,    12,    13,
-      14,    15,    16,    17,   153,   154,   155,     0,   156,   157,
-     158,   159,   160,   161,   162,    18,    19,    20,    21,    22,
-      23,    24,    25,    26,    27,    28,    29,    30,    31,    32,
-       0,    33,    34,    35,    36,    37,    38,    39,    40,    41,
-      42,    43,    44,    45,    46,    47,    48,    49,    50,    51,
-      52,    53,    54,    55,    56,    57,    58,    59,    60,    61,
-      62,    63,    64,    65,    66,    67,    68,    69,    70,    71,
-     163,   164,   165,   166,   167,   168,   169,     0,     0,   170,
-     171,     0,     0,     0,     0,     0,     0,     0,     0,     0,
-       0,     0,     0,     0,     0,     0,     0,     0,    73,    74,
-      75,    76,     0,    77,     0,     0,     0,     0,     0,     0,
-       0,     0,     0,     0,     0,    82,     0,     0,     0,     0,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
+       0,     0,     0,     0,    84,     0,    85,     0,     0,     0,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
-       0,     0,     0,     0,     0,     0,     0,     0,     0,    83,
-       0,    84,     0,     0,     0,     0,     0,     0,     0,     0,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
-       0,     0,     0,     0,     0,     0,     0,     0,   172,     0,
-       0,     0,     0,     0,   173,   174,   175,   176,     0,     0,
-       0,     0,     0,     0,     0,     0,     0,     0,     0,   177,
-     178,   307,    12,    13,    14,    15,    16,    17,   153,   154,
-     155,     0,   156,   157,   158,   159,   160,   161,   162,    18,
+       0,     0,     0,   174,     0,     0,     0,     0,     0,   175,
+     176,   177,   178,     0,     0,     0,     0,     0,     0,     0,
+       0,     0,     0,     0,   179,   180,   309,    12,    13,    14,
+      15,    16,    17,   155,   156,   157,     0,   158,   159,   160,
+     161,   162,   163,   164,    18,    19,    20,    21,    22,    23,
+      24,    25,    26,    27,    28,    29,    30,    31,    32,     0,
+      33,    34,    35,    36,    37,    38,    39,    40,    41,    42,
+      43,    44,    45,    46,    47,    48,    49,    50,    51,    52,
+      53,    54,    55,    56,    57,    58,    59,    60,    61,    62,
+      63,    64,    65,    66,    67,    68,    69,    70,    71,   165,
+     166,   167,   168,   169,   170,   171,     0,     0,   172,   173,
+       0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
+       0,     0,     0,     0,     0,     0,     0,    73,    74,    75,
+      76,     0,    77,     0,     0,     0,     0,     0,     0,     0,
+       0,     0,     0,     0,     0,    83,     0,     0,     0,     0,
+       0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
+       0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
+       0,     0,     0,     0,     0,     0,     0,     0,     0,    84,
+       0,    85,     0,     0,     0,     0,     0,     0,     0,     0,
+       0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
+       0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
+       0,     0,     0,     0,     0,     0,     0,     0,   174,     0,
+       0,     0,     0,     0,   175,   176,   177,   178,     0,     0,
+       0,     0,     0,     0,     0,     0,     0,     0,     0,   179,
+     180,   340,    12,    13,    14,    15,    16,    17,   155,   156,
+     157,     0,   158,   159,   160,   161,   162,   163,   164,    18,
       19,    20,    21,    22,    23,    24,    25,    26,    27,    28,
       29,    30,    31,    32,     0,    33,    34,    35,    36,    37,
       38,    39,    40,    41,    42,    43,    44,    45,    46,    47,
       48,    49,    50,    51,    52,    53,    54,    55,    56,    57,
       58,    59,    60,    61,    62,    63,    64,    65,    66,    67,
-      68,    69,    70,    71,   163,   164,   165,   166,   167,   168,
-     169,     0,     0,   170,   171,     0,     0,     0,     0,     0,
+      68,    69,    70,    71,   165,   166,   167,   168,   169,   170,
+     171,     0,     0,   172,   173,     0,     0,     0,     0,     0,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
        0,     0,    73,    74,    75,    76,     0,    77,     0,     0,
-       0,     0,     0,     0,     0,     0,     0,     0,     0,    82,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
+      83,     0,     0,     0,     0,     0,     0,     0,     0,     0,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
-       0,     0,     0,    83,     0,    84,     0,     0,     0,     0,
+       0,     0,     0,     0,    84,     0,    85,     0,     0,     0,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
-       0,     0,   172,     0,     0,     0,     0,     0,   173,   174,
-     175,   176,     0,     0,     0,     0,     0,     0,     0,     0,
-       0,     0,     0,   177,   178,   338,    12,    13,    14,    15,
-      16,    17,   153,   154,   155,     0,   156,   157,   158,   159,
-     160,   161,   162,    18,    19,    20,    21,    22,    23,    24,
+       0,     0,     0,   174,     0,     0,     0,     0,     0,   175,
+     176,   177,   178,     0,     0,     0,     0,     0,     0,     0,
+       0,     0,     0,     0,   179,   180,    12,    13,    14,    15,
+      16,    17,   155,   156,   157,     0,   158,   159,   160,   161,
+     162,   163,   164,    18,    19,    20,    21,    22,    23,    24,
       25,    26,    27,    28,    29,    30,    31,    32,     0,    33,
       34,    35,    36,    37,    38,    39,    40,    41,    42,    43,
       44,    45,    46,    47,    48,    49,    50,    51,    52,    53,
       54,    55,    56,    57,    58,    59,    60,    61,    62,    63,
-      64,    65,    66,    67,    68,    69,    70,    71,   163,   164,
-     165,   166,   167,   168,   169,     0,     0,   170,   171,     0,
+      64,    65,    66,    67,    68,    69,    70,    71,   165,   166,
+     167,   168,   169,   170,   171,     0,     0,   172,   173,     0,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
        0,     0,     0,     0,     0,     0,    73,    74,    75,    76,
        0,    77,     0,     0,     0,     0,     0,     0,     0,     0,
-       0,     0,     0,    82,     0,     0,     0,     0,     0,     0,
-       0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
-       0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
-       0,     0,     0,     0,     0,     0,     0,    83,     0,    84,
-       0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
-       0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
-       0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
-       0,     0,     0,     0,     0,     0,   172,     0,     0,     0,
-       0,     0,   173,   174,   175,   176,     0,     0,     0,     0,
-       0,     0,     0,     0,     0,     0,     0,   177,   178,    12,
-      13,    14,    15,    16,    17,   153,   154,   155,     0,   156,
-     157,   158,   159,   160,   161,   162,    18,    19,    20,    21,
-      22,    23,    24,    25,    26,    27,    28,    29,    30,    31,
-      32,     0,    33,    34,    35,    36,    37,    38,    39,    40,
-      41,    42,    43,    44,    45,    46,    47,    48,    49,    50,
-      51,    52,    53,    54,    55,    56,    57,    58,    59,    60,
-      61,    62,    63,    64,    65,    66,    67,    68,    69,    70,
-      71,   163,   164,   165,   166,   167,   168,   169,     0,     0,
-     170,   171,     0,     0,     0,     0,     0,     0,     0,     0,
-       0,     0,     0,     0,     0,     0,     0,     0,     0,    73,
-      74,    75,    76,     0,    77,     0,     0,     0,     0,     0,
-       0,     0,     0,     0,     0,     0,    82,     0,     0,     0,
+       0,     0,     0,     0,    83,     0,     0,     0,     0,     0,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
+       0,     0,     0,     0,     0,     0,     0,     0,    84,     0,
+      85,     0,     0,     0,     0,     0,     0,     0,     0,     0,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
-      83,     0,    84,     0,     0,     0,     0,     0,     0,     0,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
+       0,     0,     0,     0,     0,     0,     0,   174,     0,     0,
+       0,     0,     0,   175,   176,   177,   178,    12,    13,    14,
+      15,    16,    17,     0,     0,     0,     0,     0,   179,   126,
+       0,     0,     0,     0,    18,    19,    20,    21,    22,    23,
+      24,    25,    26,    27,    28,    29,    30,    31,    32,     0,
+      33,    34,    35,    36,    37,    38,    39,    40,    41,    42,
+      43,    44,    45,    46,    47,    48,    49,    50,    51,    52,
+      53,    54,    55,    56,    57,    58,    59,    60,    61,    62,
+      63,    64,    65,    66,    67,    68,    69,    70,    71,     0,
+     166,   167,   168,   169,   170,   171,     0,     0,   172,   173,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
-       0,     0,     0,     0,     0,     0,     0,     0,     0,   172,
-       0,     0,     0,     0,     0,   173,   174,   175,   176,    12,
-      13,    14,    15,    16,    17,     0,     0,     0,     0,     0,
-     177,   124,     0,     0,     0,     0,    18,    19,    20,    21,
-      22,    23,    24,    25,    26,    27,    28,    29,    30,    31,
-      32,     0,    33,    34,    35,    36,    37,    38,    39,    40,
-      41,    42,    43,    44,    45,    46,    47,    48,    49,    50,
-      51,    52,    53,    54,    55,    56,    57,    58,    59,    60,
-      61,    62,    63,    64,    65,    66,    67,    68,    69,    70,
-      71,     0,   164,   165,   166,   167,   168,   169,     0,     0,
-     170,   171,     0,     0,     0,     0,     0,     0,     0,     0,
-       0,     0,     0,     0,     0,     0,     0,     0,     0,    73,
-      74,    75,    76,     0,    77,     0,     0,     0,     0,     0,
-       0,     0,     0,     0,     0,     0,    82,    14,    15,    16,
-      17,     0,     0,     0,     0,     0,     0,     0,     0,     0,
-       0,     0,    18,    19,    20,    21,    22,    23,    24,    25,
-      26,    27,    28,    29,     0,     0,     0,     0,     0,     0,
-      83,     0,    84,    38,    39,    40,    41,    42,    43,    44,
-      45,    46,    47,    48,    49,    50,    51,    52,    53,    54,
-      55,    56,    57,    58,    59,    60,    61,    62,    63,    64,
-      65,    66,    67,    68,    69,    70,    71,     0,    72,   172,
-       0,     0,     0,     0,     0,   173,   174,   175,   176,     0,
+       0,     0,     0,     0,     0,     0,     0,    73,    74,    75,
+      76,     0,    77,     0,     0,     0,     0,     0,     0,     0,
+       0,     0,     0,     0,     0,    83,    14,    15,    16,    17,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
-     177,     0,     0,     0,     0,     0,    74,    75,    76,     0,
+       0,    18,    19,    20,    21,    22,    23,    24,    25,    26,
+      27,    28,    29,     0,     0,     0,     0,     0,     0,    84,
+       0,    85,    38,    39,    40,    41,    42,    43,    44,    45,
+      46,    47,    48,    49,    50,    51,    52,    53,    54,    55,
+      56,    57,    58,    59,    60,    61,    62,    63,    64,    65,
+      66,    67,    68,    69,    70,    71,     0,    72,   174,     0,
+       0,     0,     0,     0,   175,   176,   177,   178,     0,     0,
+       0,     0,     0,     0,     0,     0,     0,     0,     0,   179,
+       0,     0,     0,     0,     0,    74,    75,    76,     0,     0,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
        0,     0,     0,    14,    15,    16,    17,     0,     0,     0,
        0,     0,     0,     0,     0,     0,     0,     0,    18,    19,
       20,    21,    22,    23,    24,    25,    26,    27,    28,    29,
-       0,     0,     0,     0,     0,     0,    83,     0,    84,    38,
+       0,     0,     0,     0,     0,     0,    84,     0,    85,    38,
       39,    40,    41,    42,    43,    44,    45,    46,    47,    48,
       49,    50,    51,    52,    53,    54,    55,    56,    57,    58,
       59,    60,    61,    62,    63,    64,    65,    66,    67,    68,
       69,    70,    71,     0,    72,     0,     0,     0,     0,     0,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
-       0,     0,     0,     0,     0,     0,     0,     0,   239,     0,
+       0,     0,     0,     0,     0,     0,     0,     0,   241,     0,
        0,     0,    74,    75,    76,     0,     0,     0,     0,     0,
-       0,     0,     0,     0,     0,     0,     0,     0,     0,    14,
-      15,    16,    17,     0,     0,     0,     0,     0,     0,     0,
-       0,     0,     0,     0,    18,    19,    20,    21,    22,    23,
-      24,    25,    26,    27,    28,    29,     0,     0,     0,     0,
-       0,     0,    83,     0,    84,    38,    39,    40,    41,    42,
-      43,    44,    45,    46,    47,    48,    49,    50,    51,    52,
-      53,    54,    55,    56,    57,    58,    59,    60,    61,    62,
-      63,    64,    65,    66,    67,    68,    69,    70,    71,     0,
-     164,   165,   166,   167,   168,   169,     0,     0,   170,   171,
-       0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
-       0,     0,     0,     0,   319,     0,     0,     0,    74,    75,
-      76,     0,     0,     0,     0,     0,     0,     0,     0,     0,
-       0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
-       0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
-       0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
-       0,     0,     0,     0,     0,     0,     0,     0,    83,     0,
-      84,     0,     0,     0,     0,     0,     0,     0,     0,     0,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
+      14,    15,    16,    17,     0,     0,     0,     0,     0,     0,
+       0,     0,     0,     0,     0,    18,    19,    20,    21,    22,
+      23,    24,    25,    26,    27,    28,    29,     0,     0,     0,
+       0,     0,     0,    84,     0,    85,    38,    39,    40,    41,
+      42,    43,    44,    45,    46,    47,    48,    49,    50,    51,
+      52,    53,    54,    55,    56,    57,    58,    59,    60,    61,
+      62,    63,    64,    65,    66,    67,    68,    69,    70,    71,
+       0,   166,   167,   168,   169,   170,   171,     0,     0,   172,
+     173,     0,     0,     0,     0,     0,     0,     0,     0,     0,
+       0,     0,     0,     0,     0,   321,     0,     0,     0,    74,
+      75,    76,     0,     0,     0,     0,     0,     0,     0,     0,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
-       0,     0,     0,     0,     0,     0,     0,   172,     0,     0,
-       0,     0,     0,   173,   174,   175,   176,    12,    13,    14,
-      15,    16,    17,     0,     0,     0,     0,     0,   251,     0,
-       0,     0,     0,     0,    18,    19,    20,    21,    22,    23,
-      24,    25,    26,    27,    28,    29,    30,    31,    32,     0,
-      33,    34,    35,    36,    37,    38,    39,    40,    41,    42,
-      43,    44,    45,    46,    47,    48,    49,    50,    51,    52,
-      53,    54,    55,    56,    57,    58,    59,    60,    61,    62,
-      63,    64,    65,    66,    67,    68,    69,    70,    71,     0,
-     164,   165,   166,   167,   168,   169,     0,     0,   170,   171,
-       0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
-       0,     0,     0,     0,     0,     0,     0,   333,    74,    75,
-      76,     0,     0,     0,     0,     0,     0,     0,     0,     0,
-       0,     0,     0,     0,    82,     0,     0,     0,     0,     0,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
-       0,     0,     0,     0,     0,     0,     0,     0,    83,     0,
-      84,     0,     0,     0,     0,     0,     0,     0,     0,     0,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
+      84,     0,    85,     0,     0,     0,     0,     0,     0,     0,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
-       0,     0,    14,    15,    16,    17,     0,   172,     0,     0,
-       0,     0,     0,   173,   174,   175,   176,    18,    19,    20,
-      21,    22,    23,    24,    25,    26,    27,    28,    29,     0,
-       0,     0,     0,     0,     0,     0,     0,     0,    38,    39,
-      40,    41,    42,    43,    44,    45,    46,    47,    48,    49,
-      50,    51,    52,    53,    54,    55,    56,    57,    58,    59,
-      60,    61,    62,    63,    64,    65,    66,    67,    68,    69,
-      70,    71,     0,   164,   165,   166,   167,   168,   169,     0,
-       0,   170,   171,     0,     0,     0,     0,     0,     0,     0,
-       0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
-       0,    74,    75,    76,     0,     0,     0,     0,     0,     0,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
+       0,     0,     0,     0,     0,     0,     0,     0,     0,   174,
+       0,     0,     0,     0,     0,   175,   176,   177,   178,    12,
+      13,    14,    15,    16,    17,     0,     0,     0,     0,     0,
+     253,     0,     0,     0,     0,     0,    18,    19,    20,    21,
+      22,    23,    24,    25,    26,    27,    28,    29,    30,    31,
+      32,     0,    33,    34,    35,    36,    37,    38,    39,    40,
+      41,    42,    43,    44,    45,    46,    47,    48,    49,    50,
+      51,    52,    53,    54,    55,    56,    57,    58,    59,    60,
+      61,    62,    63,    64,    65,    66,    67,    68,    69,    70,
+      71,     0,   166,   167,   168,   169,   170,   171,     0,     0,
+     172,   173,     0,     0,     0,     0,     0,     0,     0,     0,
+       0,     0,     0,     0,     0,     0,     0,     0,     0,   335,
+      74,    75,    76,     0,     0,     0,     0,     0,     0,     0,
+       0,     0,     0,     0,     0,     0,     0,    83,     0,     0,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
-       0,    83,     0,    84,     0,     0,     0,     0,     0,     0,
+       0,    84,     0,    85,     0,     0,     0,     0,     0,     0,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
        0,     0,     0,     0,     0,    14,    15,    16,    17,     0,
-     172,     0,     0,   229,     0,     0,   173,   174,   175,   176,
+     174,     0,     0,     0,     0,     0,   175,   176,   177,   178,
       18,    19,    20,    21,    22,    23,    24,    25,    26,    27,
       28,    29,     0,     0,     0,     0,     0,     0,     0,     0,
        0,    38,    39,    40,    41,    42,    43,    44,    45,    46,
       47,    48,    49,    50,    51,    52,    53,    54,    55,    56,
       57,    58,    59,    60,    61,    62,    63,    64,    65,    66,
-      67,    68,    69,    70,    71,     0,   164,   165,   166,   167,
-     168,   169,     0,     0,   170,   171,     0,     0,     0,     0,
+      67,    68,    69,    70,    71,     0,   166,   167,   168,   169,
+     170,   171,     0,     0,   172,   173,     0,     0,     0,     0,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
        0,     0,     0,     0,    74,    75,    76,     0,     0,     0,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
-       0,     0,     0,     0,    83,     0,    84,     0,     0,     0,
+       0,     0,     0,     0,     0,    84,     0,    85,     0,     0,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
-       0,     0,     0,     0,     0,     0,     0,     0,    14,    15,
-      16,    17,     0,   172,     0,     0,   314,     0,     0,   173,
-     174,   175,   176,    18,    19,    20,    21,    22,    23,    24,
-      25,    26,    27,    28,    29,     0,     0,     0,     0,     0,
-       0,     0,     0,     0,    38,    39,    40,    41,    42,    43,
-      44,    45,    46,    47,    48,    49,    50,    51,    52,    53,
-      54,    55,    56,    57,    58,    59,    60,    61,    62,    63,
-      64,    65,    66,    67,    68,    69,    70,    71,     0,   164,
-     165,   166,   167,   168,   169,     0,     0,   170,   171,     0,
+       0,     0,     0,     0,     0,     0,     0,     0,     0,    14,
+      15,    16,    17,     0,   174,     0,     0,   231,     0,     0,
+     175,   176,   177,   178,    18,    19,    20,    21,    22,    23,
+      24,    25,    26,    27,    28,    29,     0,     0,     0,     0,
+       0,     0,     0,     0,     0,    38,    39,    40,    41,    42,
+      43,    44,    45,    46,    47,    48,    49,    50,    51,    52,
+      53,    54,    55,    56,    57,    58,    59,    60,    61,    62,
+      63,    64,    65,    66,    67,    68,    69,    70,    71,     0,
+     166,   167,   168,   169,   170,   171,     0,     0,   172,   173,
+       0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
+       0,     0,     0,     0,     0,     0,     0,     0,    74,    75,
+      76,     0,     0,     0,     0,     0,     0,     0,     0,     0,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
-       0,     0,     0,     0,     0,     0,     0,    74,    75,    76,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
+       0,     0,     0,     0,     0,     0,     0,     0,     0,    84,
+       0,    85,     0,     0,     0,     0,     0,     0,     0,     0,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
-       0,     0,     0,     0,     0,     0,     0,    83,     0,    84,
+       0,     0,     0,    14,    15,    16,    17,     0,   174,     0,
+       0,   316,     0,     0,   175,   176,   177,   178,    18,    19,
+      20,    21,    22,    23,    24,    25,    26,    27,    28,    29,
+       0,     0,     0,     0,     0,     0,     0,     0,     0,    38,
+      39,    40,    41,    42,    43,    44,    45,    46,    47,    48,
+      49,    50,    51,    52,    53,    54,    55,    56,    57,    58,
+      59,    60,    61,    62,    63,    64,    65,    66,    67,    68,
+      69,    70,    71,     0,   166,   167,   168,   169,   170,   171,
+       0,     0,   172,   173,     0,     0,     0,     0,     0,     0,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
+       0,     0,    74,    75,    76,     0,     0,     0,     0,     0,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
-       0,    14,    15,    16,    17,     0,   172,     0,     0,   367,
-       0,     0,   173,   174,   175,   176,    18,    19,    20,    21,
-      22,    23,    24,    25,    26,    27,    28,    29,     0,     0,
-       0,     0,     0,     0,     0,     0,     0,    38,    39,    40,
-      41,    42,    43,    44,    45,    46,    47,    48,    49,    50,
-      51,    52,    53,    54,    55,    56,    57,    58,    59,    60,
-      61,    62,    63,    64,    65,    66,    67,    68,    69,    70,
-      71,     0,   164,   165,   166,   167,   168,   169,     0,     0,
-     170,   171,     0,     0,     0,     0,     0,     0,     0,     0,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
-      74,    75,    76,     0,     0,     0,     0,     0,     0,     0,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
+       0,     0,     0,    84,     0,    85,     0,     0,     0,     0,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
+       0,     0,     0,     0,     0,     0,     0,    14,    15,    16,
+      17,     0,   174,     0,     0,   369,     0,     0,   175,   176,
+     177,   178,    18,    19,    20,    21,    22,    23,    24,    25,
+      26,    27,    28,    29,     0,     0,     0,     0,     0,     0,
+       0,     0,     0,    38,    39,    40,    41,    42,    43,    44,
+      45,    46,    47,    48,    49,    50,    51,    52,    53,    54,
+      55,    56,    57,    58,    59,    60,    61,    62,    63,    64,
+      65,    66,    67,    68,    69,    70,    71,     0,   166,   167,
+     168,   169,   170,   171,     0,     0,   172,   173,     0,     0,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
-      83,     0,    84,     0,     0,     0,     0,     0,     0,     0,
+       0,     0,     0,     0,     0,     0,    74,    75,    76,     0,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
-       0,     0,     0,     0,    14,    15,    16,    17,     0,   172,
-       0,     0,     0,     0,     0,   173,   174,   175,   176,    18,
-      19,    20,    21,    22,    23,    24,    25,    26,    27,    28,
-      29,     0,     0,     0,     0,     0,     0,     0,     0,     0,
-      38,    39,    40,    41,    42,    43,    44,    45,    46,    47,
-      48,    49,    50,    51,    52,    53,    54,    55,    56,    57,
-      58,    59,    60,    61,    62,    63,    64,    65,    66,    67,
-      68,    69,    70,   269,     0,   164,   165,   166,   167,   168,
-     169,     0,     0,   170,   171,     0,     0,     0,     0,     0,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
-       0,     0,     0,    74,    75,    76,     0,     0,     0,     0,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
+       0,     0,     0,     0,     0,     0,     0,    84,     0,    85,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
-       0,     0,     0,    83,     0,    84,     0,     0,     0,     0,
+       0,    14,    15,    16,    17,     0,   174,     0,     0,     0,
+       0,     0,   175,   176,   177,   178,    18,    19,    20,    21,
+      22,    23,    24,    25,    26,    27,    28,    29,     0,     0,
+       0,     0,     0,     0,     0,     0,     0,    38,    39,    40,
+      41,    42,    43,    44,    45,    46,    47,    48,    49,    50,
+      51,    52,    53,    54,    55,    56,    57,    58,    59,    60,
+      61,    62,    63,    64,    65,    66,    67,    68,    69,    70,
+     271,     0,   166,   167,   168,   169,   170,   171,     0,     0,
+     172,   173,     0,     0,     0,     0,     0,     0,     0,     0,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
+      74,    75,    76,     0,     0,     0,     0,     0,     0,     0,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
-       0,     0,     0,     0,     0,    12,    13,    14,    15,    16,
-      17,     0,   172,     0,     0,     0,     0,     0,   173,   174,
-     175,   176,    18,    19,    20,    21,    22,    23,    24,    25,
-      26,    27,    28,    29,    30,    31,    32,     0,    33,    34,
-      35,    36,    37,    38,    39,    40,    41,    42,    43,    44,
-      45,    46,    47,    48,    49,    50,    51,    52,    53,    54,
-      55,    56,    57,    58,    59,    60,    61,    62,    63,    64,
-      65,    66,    67,    68,    69,    70,    71,     0,    72,     0,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
-       0,     0,     0,     0,     0,    73,    74,    75,    76,     0,
-      77,     0,     0,     0,     0,     0,     0,     0,    78,    79,
-      80,    81,    82,    14,    15,    16,    17,     0,     0,     0,
-       0,     0,     0,     0,     0,     0,     0,     0,    18,    19,
-      20,    21,    22,    23,    24,    25,    26,    27,    28,    29,
-       0,     0,     0,     0,     0,     0,    83,     0,    84,    38,
-      39,    40,    41,    42,    43,    44,    45,    46,    47,    48,
-      49,    50,    51,    52,    53,    54,    55,    56,    57,    58,
-      59,    60,    61,    62,    63,    64,    65,    66,    67,    68,
-      69,    70,    71,     0,   341,    14,    15,    16,    17,   169,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
+       0,    84,     0,    85,     0,     0,     0,     0,     0,     0,
+       0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
+       0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
+       0,     0,     0,    12,    13,    14,    15,    16,    17,     0,
+     174,     0,     0,     0,     0,     0,   175,   176,   177,   178,
       18,    19,    20,    21,    22,    23,    24,    25,    26,    27,
-      28,    29,    74,    75,    76,     0,     0,     0,     0,     0,
-       0,    38,    39,    40,    41,    42,    43,    44,    45,    46,
+      28,    29,    30,    31,    32,     0,    33,    34,    35,    36,
+      37,    38,    39,    40,    41,    42,    43,    44,    45,    46,
       47,    48,    49,    50,    51,    52,    53,    54,    55,    56,
       57,    58,    59,    60,    61,    62,    63,    64,    65,    66,
-      67,    68,    69,    70,    71,     0,    72,    14,    15,    16,
-      17,     0,    83,     0,    84,     0,     0,     0,     0,     0,
-       0,     0,    18,    19,    20,    21,    22,    23,    24,    25,
-      26,    27,    28,    29,    74,    75,    76,     0,     0,     0,
-       0,     0,     0,    38,    39,    40,    41,    42,    43,    44,
-      45,    46,    47,    48,    49,    50,    51,    52,    53,    54,
-      55,    56,    57,    58,    59,    60,    61,    62,    63,    64,
-      65,    66,    67,    68,    69,    70,    71,     0,    72,     0,
-       0,     0,     0,     0,    83,     0,    84,     0,     0,     0,
+      67,    68,    69,    70,    71,     0,    72,     0,     0,     0,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
+       0,     0,     0,    73,    74,    75,    76,     0,    77,     0,
+       0,     0,     0,     0,     0,     0,    78,    79,    80,    81,
+      82,    83,    14,    15,    16,    17,     0,     0,     0,     0,
+       0,     0,     0,     0,     0,     0,     0,    18,    19,    20,
+      21,    22,    23,    24,    25,    26,    27,    28,    29,     0,
+       0,     0,     0,     0,     0,    84,     0,    85,    38,    39,
+      40,    41,    42,    43,    44,    45,    46,    47,    48,    49,
+      50,    51,    52,    53,    54,    55,    56,    57,    58,    59,
+      60,    61,    62,    63,    64,    65,    66,    67,    68,    69,
+      70,    71,     0,   343,    14,    15,    16,    17,   171,     0,
+       0,     0,     0,     0,     0,     0,     0,     0,     0,    18,
+      19,    20,    21,    22,    23,    24,    25,    26,    27,    28,
+      29,    74,    75,    76,     0,     0,     0,     0,     0,     0,
+      38,    39,    40,    41,    42,    43,    44,    45,    46,    47,
+      48,    49,    50,    51,    52,    53,    54,    55,    56,    57,
+      58,    59,    60,    61,    62,    63,    64,    65,    66,    67,
+      68,    69,    70,    71,     0,    72,    14,    15,    16,    17,
+       0,     0,    84,     0,    85,     0,     0,     0,     0,     0,
+       0,    18,    19,    20,    21,    22,    23,    24,    25,    26,
+      27,    28,    29,    74,    75,    76,     0,     0,     0,     0,
+       0,     0,    38,    39,    40,    41,    42,    43,    44,    45,
+      46,    47,    48,    49,    50,    51,    52,    53,    54,    55,
+      56,    57,    58,    59,    60,    61,    62,    63,    64,    65,
+      66,    67,    68,    69,    70,    71,     0,    72,     0,     0,
+       0,     0,     0,     0,    84,     0,    85,     0,     0,     0,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
-       0,     0,     0,     0,     0,     0,    83,     0,    84
+       0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
+       0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
+       0,     0,     0,     0,     0,     0,    84,     0,    85
 };
 
 static const yytype_int16 yycheck[] =
 {
-       9,    73,    73,   159,   155,   161,   141,   146,   141,     4,
-      76,    76,    86,    87,    91,   195,   172,   256,     9,     3,
-       4,     9,    78,    95,    96,    84,    85,   193,     4,   195,
-     313,   109,   381,   213,   227,   170,   171,   198,    33,    34,
-      35,   198,   114,   392,   198,     3,     4,   213,    32,    33,
-      34,   212,    36,    37,     0,   212,   191,    33,    34,    35,
-     214,   113,   213,    93,    94,    95,    96,    97,    98,    99,
-     100,   101,   102,   110,    32,    33,    34,    86,    36,    37,
-      38,    39,    40,   198,    76,   198,   112,   370,    97,   328,
-     198,   188,   227,   249,   227,    86,   235,   253,    86,   214,
-     256,   214,   385,   112,   103,   388,   214,   194,   264,   113,
-     261,   198,   194,   396,   193,   124,   198,   116,    76,   312,
-     403,    33,    34,   113,   133,    37,   194,   320,   193,   198,
-     198,   228,   206,   207,   211,   144,   195,   146,   197,   194,
-     194,   194,   113,   198,   198,   198,   155,   113,   304,   215,
-     285,   286,   287,   288,   289,   290,   291,   292,   293,   294,
-     295,   296,   297,   298,   299,   300,   301,   302,   303,   178,
-     194,   268,   328,   366,   104,   105,   106,   312,    76,   312,
-      33,    34,    35,   213,    76,   320,   283,   320,   292,   293,
-     294,   295,   203,   204,   205,   214,   215,   199,   200,   194,
-      82,    83,    88,    89,   213,   288,   289,   215,   305,   290,
-     291,   296,   297,   195,    76,   224,   313,    76,   374,   113,
-      76,   214,   194,   193,   213,   193,   235,   378,   214,   214,
-     193,   366,   193,   366,   214,   391,   193,   212,    90,   248,
-     209,     3,     4,   208,    92,   193,   210,   256,    76,   196,
-     195,    78,   261,    75,   214,   194,   196,   408,   195,   213,
-     193,   333,   333,   198,    76,   194,   193,    12,   196,   213,
-      32,    33,    34,   370,    36,    37,    38,    39,    40,   196,
-     196,   213,   196,   214,   194,   298,   383,   265,   385,   215,
-     213,   388,   213,   299,   214,   300,   303,   127,   301,   396,
-     224,   302,   134,   244,    77,   224,   403,   144,   392,   321,
-     248,    88,   379,   178,   248,    86,    -1,    -1,   328,   328,
+       9,    73,    73,   157,   143,   161,    76,   163,   143,    91,
+       4,    76,   229,    86,    87,   258,   190,   148,   174,     4,
+       9,    78,     9,   109,    96,    97,    93,    94,    95,    96,
+      97,    98,    99,   100,   101,   102,   196,   172,   173,    33,
+      34,    35,   194,   115,   196,     0,     3,     4,    33,    34,
+      35,   199,   199,   199,   214,   199,   230,   113,   193,   199,
+     110,   215,   214,   383,    84,    85,    76,   215,   215,   215,
+     199,   215,   104,   213,   394,    32,    33,    34,    87,    36,
+      37,    38,    39,    40,   213,   117,   112,   330,   195,    98,
+     229,   113,   199,   194,   229,   251,   270,   314,    87,   255,
+      87,   113,   258,   195,   113,   322,   237,   199,   113,   263,
+     266,   285,    33,    34,   195,   113,    37,   126,   199,    76,
+     315,   195,   104,   105,   106,   199,   135,   195,    76,   194,
+     212,   199,   195,   307,   207,   208,   199,   146,   113,   148,
+     195,   315,    33,    34,    35,   199,   216,   214,   157,   196,
+     306,   368,   287,   288,   289,   290,   291,   292,   293,   294,
+     295,   296,   297,   298,   299,   300,   301,   302,   303,   304,
+     305,   180,   215,   216,   330,   314,   196,   372,   198,   314,
+     204,   205,   206,   322,   200,   201,    76,   322,   294,   295,
+     296,   297,   387,    82,    83,   390,    88,    89,   372,   290,
+     291,   195,   216,   398,   292,   293,   215,    76,   298,   299,
+     405,   385,    76,   387,     3,     4,   390,   226,   113,    76,
+     376,   194,   215,   214,   398,   215,   380,   215,   237,   368,
+     194,   405,   215,   368,   194,   213,   194,   393,     3,     4,
+     194,   250,   195,    32,    33,    34,   209,    36,    37,   258,
+     211,   210,    90,    92,   263,   194,   410,    76,   196,   215,
+     197,    78,    75,   335,   335,   195,   197,    32,    33,    34,
+     196,    36,    37,    38,    39,    40,   194,   199,   214,    76,
+     195,   197,   194,   197,   197,   214,   214,   195,   215,    12,
+     214,   216,   197,   267,   214,   301,   215,   300,   302,   129,
+     136,   303,   246,   304,    77,   305,   146,   250,   323,   226,
+     394,    89,   381,   180,    -1,   226,   250,   330,    87,    -1,
+      -1,   330,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
       -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
       -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
       -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
       -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
-      -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,   378,
-      -1,    -1,   381,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
-      -1,    -1,    -1,   392,    -1,    -1,    -1,    -1,    -1,    -1,
-       3,     4,     5,     6,     7,     8,     9,    10,    11,   408,
-      13,    14,    15,    16,    17,    18,    19,    20,    21,    22,
-      23,    24,    25,    26,    27,    28,    29,    30,    31,    32,
-      33,    34,    -1,    36,    37,    38,    39,    40,    41,    42,
-      43,    44,    45,    46,    47,    48,    49,    50,    51,    52,
-      53,    54,    55,    56,    57,    58,    59,    60,    61,    62,
-      63,    64,    65,    66,    67,    68,    69,    70,    71,    72,
-      73,    74,    75,    76,    77,    78,    79,    80,    81,    -1,
-      -1,    84,    85,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
+      -1,   380,    -1,    -1,   383,    -1,    -1,    -1,    -1,    -1,
+      -1,    -1,    -1,    -1,    -1,   394,    -1,    -1,    -1,    -1,
+      -1,    -1,     3,     4,     5,     6,     7,     8,     9,    10,
+      11,   410,    13,    14,    15,    16,    17,    18,    19,    20,
+      21,    22,    23,    24,    25,    26,    27,    28,    29,    30,
+      31,    32,    33,    34,    -1,    36,    37,    38,    39,    40,
+      41,    42,    43,    44,    45,    46,    47,    48,    49,    50,
+      51,    52,    53,    54,    55,    56,    57,    58,    59,    60,
+      61,    62,    63,    64,    65,    66,    67,    68,    69,    70,
+      71,    72,    73,    74,    75,    76,    77,    78,    79,    80,
+      81,    -1,    -1,    84,    85,    -1,    -1,    -1,    -1,    -1,
       -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
-     103,   104,   105,   106,    -1,   108,    -1,    -1,    -1,    -1,
-      -1,    -1,    -1,    -1,    -1,    -1,    -1,   120,    -1,    -1,
+      -1,    -1,   103,   104,   105,   106,    -1,   108,    -1,    -1,
       -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
+     121,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
       -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
       -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
-      -1,   154,    -1,   156,    -1,    -1,    -1,    -1,    -1,    -1,
+      -1,    -1,    -1,    -1,   155,    -1,   157,    -1,    -1,    -1,
       -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
       -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
       -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
-     193,    -1,    -1,    -1,    -1,    -1,   199,   200,   201,   202,
+      -1,    -1,    -1,   194,    -1,    -1,    -1,    -1,    -1,   200,
+     201,   202,   203,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
+      -1,    -1,    -1,    -1,   215,   216,   217,     3,     4,     5,
+       6,     7,     8,     9,    10,    11,    -1,    13,    14,    15,
+      16,    17,    18,    19,    20,    21,    22,    23,    24,    25,
+      26,    27,    28,    29,    30,    31,    32,    33,    34,    -1,
+      36,    37,    38,    39,    40,    41,    42,    43,    44,    45,
+      46,    47,    48,    49,    50,    51,    52,    53,    54,    55,
+      56,    57,    58,    59,    60,    61,    62,    63,    64,    65,
+      66,    67,    68,    69,    70,    71,    72,    73,    74,    75,
+      76,    77,    78,    79,    80,    81,    -1,    -1,    84,    85,
       -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
-      -1,   214,   215,   216,     3,     4,     5,     6,     7,     8,
-       9,    10,    11,    -1,    13,    14,    15,    16,    17,    18,
-      19,    20,    21,    22,    23,    24,    25,    26,    27,    28,
-      29,    30,    31,    32,    33,    34,    -1,    36,    37,    38,
-      39,    40,    41,    42,    43,    44,    45,    46,    47,    48,
-      49,    50,    51,    52,    53,    54,    55,    56,    57,    58,
-      59,    60,    61,    62,    63,    64,    65,    66,    67,    68,
-      69,    70,    71,    72,    73,    74,    75,    76,    77,    78,
-      79,    80,    81,    -1,    -1,    84,    85,    -1,    -1,    -1,
+      -1,    -1,    -1,    -1,    -1,    -1,    -1,   103,   104,   105,
+     106,    -1,   108,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
+      -1,    -1,    -1,    -1,    -1,   121,    -1,    -1,    -1,    -1,
       -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
-      -1,    -1,    -1,    -1,   103,   104,   105,   106,    -1,   108,
       -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
-      -1,   120,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
+      -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,   155,
+      -1,   157,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
       -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
       -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
-      -1,    -1,    -1,    -1,    -1,   154,    -1,   156,    -1,    -1,
+      -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,   194,    -1,
+      -1,    -1,    -1,    -1,   200,   201,   202,   203,    -1,    -1,
+      -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,   215,
+     216,   217,     3,     4,     5,     6,     7,     8,     9,    10,
+      11,    -1,    13,    14,    15,    16,    17,    18,    19,    20,
+      21,    22,    23,    24,    25,    26,    27,    28,    29,    30,
+      31,    32,    33,    34,    -1,    36,    37,    38,    39,    40,
+      41,    42,    43,    44,    45,    46,    47,    48,    49,    50,
+      51,    52,    53,    54,    55,    56,    57,    58,    59,    60,
+      61,    62,    63,    64,    65,    66,    67,    68,    69,    70,
+      71,    72,    73,    74,    75,    76,    77,    78,    79,    80,
+      81,    -1,    -1,    84,    85,    -1,    -1,    -1,    -1,    -1,
       -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
+      -1,    -1,   103,   104,   105,   106,    -1,   108,    -1,    -1,
+      -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
+     121,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
+      -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
+      -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
+      -1,    -1,    -1,    -1,   155,    -1,   157,    -1,    -1,    -1,
       -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
       -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
-      -1,    -1,    -1,    -1,   193,    -1,    -1,    -1,    -1,    -1,
-     199,   200,   201,   202,    -1,    -1,    -1,    -1,    -1,    -1,
-      -1,    -1,    -1,    -1,    -1,   214,   215,   216,     3,     4,
-       5,     6,     7,     8,     9,    10,    11,    -1,    13,    14,
-      15,    16,    17,    18,    19,    20,    21,    22,    23,    24,
-      25,    26,    27,    28,    29,    30,    31,    32,    33,    34,
-      -1,    36,    37,    38,    39,    40,    41,    42,    43,    44,
-      45,    46,    47,    48,    49,    50,    51,    52,    53,    54,
-      55,    56,    57,    58,    59,    60,    61,    62,    63,    64,
-      65,    66,    67,    68,    69,    70,    71,    72,    73,    74,
-      75,    76,    77,    78,    79,    80,    81,    -1,    -1,    84,
-      85,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
-      -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,   103,   104,
-     105,   106,    -1,   108,    -1,    -1,    -1,    -1,    -1,    -1,
-      -1,    -1,    -1,    -1,    -1,   120,    -1,    -1,    -1,    -1,
       -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
+      -1,    -1,    -1,   194,    -1,    -1,    -1,    -1,    -1,   200,
+     201,   202,   203,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
+      -1,    -1,    -1,    -1,   215,   216,   217,     3,     4,     5,
+       6,     7,     8,     9,    10,    11,    -1,    13,    14,    15,
+      16,    17,    18,    19,    20,    21,    22,    23,    24,    25,
+      26,    27,    28,    29,    30,    31,    32,    33,    34,    -1,
+      36,    37,    38,    39,    40,    41,    42,    43,    44,    45,
+      46,    47,    48,    49,    50,    51,    52,    53,    54,    55,
+      56,    57,    58,    59,    60,    61,    62,    63,    64,    65,
+      66,    67,    68,    69,    70,    71,    72,    73,    74,    75,
+      76,    77,    78,    79,    80,    81,    -1,    -1,    84,    85,
+      -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
+      -1,    -1,    -1,    -1,    -1,    -1,    -1,   103,   104,   105,
+     106,    -1,   108,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
+      -1,    -1,    -1,    -1,    -1,   121,    -1,    -1,    -1,    -1,
       -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
-      -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,   154,
-      -1,   156,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
       -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
+      -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,   155,
+      -1,   157,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
       -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
-      -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,   193,    -1,
-      -1,    -1,    -1,    -1,   199,   200,   201,   202,    -1,    -1,
-      -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,   214,
-     215,   216,     3,     4,     5,     6,     7,     8,     9,    10,
+      -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
+      -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,   194,    -1,
+      -1,    -1,    -1,    -1,   200,   201,   202,   203,    -1,    -1,
+      -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,   215,
+     216,   217,     3,     4,     5,     6,     7,     8,     9,    10,
       11,    -1,    13,    14,    15,    16,    17,    18,    19,    20,
       21,    22,    23,    24,    25,    26,    27,    28,    29,    30,
       31,    32,    33,    34,    -1,    36,    37,    38,    39,    40,
@@ -1635,17 +1661,17 @@ static const yytype_int16 yycheck[] =
       81,    -1,    -1,    84,    85,    -1,    -1,    -1,    -1,    -1,
       -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
       -1,    -1,   103,   104,   105,   106,    -1,   108,    -1,    -1,
-      -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,   120,
       -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
+     121,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
       -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
       -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
-      -1,    -1,    -1,   154,    -1,   156,    -1,    -1,    -1,    -1,
+      -1,    -1,    -1,    -1,   155,    -1,   157,    -1,    -1,    -1,
       -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
       -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
       -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
-      -1,    -1,   193,    -1,    -1,    -1,    -1,    -1,   199,   200,
-     201,   202,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
-      -1,    -1,    -1,   214,   215,   216,     3,     4,     5,     6,
+      -1,    -1,    -1,   194,    -1,    -1,    -1,    -1,    -1,   200,
+     201,   202,   203,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
+      -1,    -1,    -1,    -1,   215,   216,     3,     4,     5,     6,
        7,     8,     9,    10,    11,    -1,    13,    14,    15,    16,
       17,    18,    19,    20,    21,    22,    23,    24,    25,    26,
       27,    28,    29,    30,    31,    32,    33,    34,    -1,    36,
@@ -1657,38 +1683,72 @@ static const yytype_int16 yycheck[] =
       -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
       -1,    -1,    -1,    -1,    -1,    -1,   103,   104,   105,   106,
       -1,   108,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
-      -1,    -1,    -1,   120,    -1,    -1,    -1,    -1,    -1,    -1,
+      -1,    -1,    -1,    -1,   121,    -1,    -1,    -1,    -1,    -1,
+      -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
+      -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
+      -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,   155,    -1,
+     157,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
+      -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
+      -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
+      -1,    -1,    -1,    -1,    -1,    -1,    -1,   194,    -1,    -1,
+      -1,    -1,    -1,   200,   201,   202,   203,     3,     4,     5,
+       6,     7,     8,    -1,    -1,    -1,    -1,    -1,   215,   216,
+      -1,    -1,    -1,    -1,    20,    21,    22,    23,    24,    25,
+      26,    27,    28,    29,    30,    31,    32,    33,    34,    -1,
+      36,    37,    38,    39,    40,    41,    42,    43,    44,    45,
+      46,    47,    48,    49,    50,    51,    52,    53,    54,    55,
+      56,    57,    58,    59,    60,    61,    62,    63,    64,    65,
+      66,    67,    68,    69,    70,    71,    72,    73,    74,    -1,
+      76,    77,    78,    79,    80,    81,    -1,    -1,    84,    85,
       -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
+      -1,    -1,    -1,    -1,    -1,    -1,    -1,   103,   104,   105,
+     106,    -1,   108,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
+      -1,    -1,    -1,    -1,    -1,   121,     5,     6,     7,     8,
       -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
-      -1,    -1,    -1,    -1,    -1,    -1,    -1,   154,    -1,   156,
+      -1,    20,    21,    22,    23,    24,    25,    26,    27,    28,
+      29,    30,    31,    -1,    -1,    -1,    -1,    -1,    -1,   155,
+      -1,   157,    41,    42,    43,    44,    45,    46,    47,    48,
+      49,    50,    51,    52,    53,    54,    55,    56,    57,    58,
+      59,    60,    61,    62,    63,    64,    65,    66,    67,    68,
+      69,    70,    71,    72,    73,    74,    -1,    76,   194,    -1,
+      -1,    -1,    -1,    -1,   200,   201,   202,   203,    -1,    -1,
+      -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,   215,
+      -1,    -1,    -1,    -1,    -1,   104,   105,   106,    -1,    -1,
       -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
+      -1,    -1,    -1,     5,     6,     7,     8,    -1,    -1,    -1,
+      -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    20,    21,
+      22,    23,    24,    25,    26,    27,    28,    29,    30,    31,
+      -1,    -1,    -1,    -1,    -1,    -1,   155,    -1,   157,    41,
+      42,    43,    44,    45,    46,    47,    48,    49,    50,    51,
+      52,    53,    54,    55,    56,    57,    58,    59,    60,    61,
+      62,    63,    64,    65,    66,    67,    68,    69,    70,    71,
+      72,    73,    74,    -1,    76,    -1,    -1,    -1,    -1,    -1,
       -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
+      -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,   217,    -1,
+      -1,    -1,   104,   105,   106,    -1,    -1,    -1,    -1,    -1,
+      -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
+       5,     6,     7,     8,    -1,    -1,    -1,    -1,    -1,    -1,
+      -1,    -1,    -1,    -1,    -1,    20,    21,    22,    23,    24,
+      25,    26,    27,    28,    29,    30,    31,    -1,    -1,    -1,
+      -1,    -1,    -1,   155,    -1,   157,    41,    42,    43,    44,
+      45,    46,    47,    48,    49,    50,    51,    52,    53,    54,
+      55,    56,    57,    58,    59,    60,    61,    62,    63,    64,
+      65,    66,    67,    68,    69,    70,    71,    72,    73,    74,
+      -1,    76,    77,    78,    79,    80,    81,    -1,    -1,    84,
+      85,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
+      -1,    -1,    -1,    -1,    -1,   217,    -1,    -1,    -1,   104,
+     105,   106,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
       -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
-      -1,    -1,    -1,    -1,    -1,    -1,   193,    -1,    -1,    -1,
-      -1,    -1,   199,   200,   201,   202,    -1,    -1,    -1,    -1,
-      -1,    -1,    -1,    -1,    -1,    -1,    -1,   214,   215,     3,
-       4,     5,     6,     7,     8,     9,    10,    11,    -1,    13,
-      14,    15,    16,    17,    18,    19,    20,    21,    22,    23,
-      24,    25,    26,    27,    28,    29,    30,    31,    32,    33,
-      34,    -1,    36,    37,    38,    39,    40,    41,    42,    43,
-      44,    45,    46,    47,    48,    49,    50,    51,    52,    53,
-      54,    55,    56,    57,    58,    59,    60,    61,    62,    63,
-      64,    65,    66,    67,    68,    69,    70,    71,    72,    73,
-      74,    75,    76,    77,    78,    79,    80,    81,    -1,    -1,
-      84,    85,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
-      -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,   103,
-     104,   105,   106,    -1,   108,    -1,    -1,    -1,    -1,    -1,
-      -1,    -1,    -1,    -1,    -1,    -1,   120,    -1,    -1,    -1,
       -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
       -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
       -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
-     154,    -1,   156,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
+     155,    -1,   157,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
       -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
       -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
-      -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,   193,
-      -1,    -1,    -1,    -1,    -1,   199,   200,   201,   202,     3,
+      -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,   194,
+      -1,    -1,    -1,    -1,    -1,   200,   201,   202,   203,     3,
        4,     5,     6,     7,     8,    -1,    -1,    -1,    -1,    -1,
-     214,   215,    -1,    -1,    -1,    -1,    20,    21,    22,    23,
+     215,    -1,    -1,    -1,    -1,    -1,    20,    21,    22,    23,
       24,    25,    26,    27,    28,    29,    30,    31,    32,    33,
       34,    -1,    36,    37,    38,    39,    40,    41,    42,    43,
       44,    45,    46,    47,    48,    49,    50,    51,    52,    53,
@@ -1697,126 +1757,90 @@ static const yytype_int16 yycheck[] =
       74,    -1,    76,    77,    78,    79,    80,    81,    -1,    -1,
       84,    85,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
       -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,   103,
-     104,   105,   106,    -1,   108,    -1,    -1,    -1,    -1,    -1,
-      -1,    -1,    -1,    -1,    -1,    -1,   120,     5,     6,     7,
-       8,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
-      -1,    -1,    20,    21,    22,    23,    24,    25,    26,    27,
-      28,    29,    30,    31,    -1,    -1,    -1,    -1,    -1,    -1,
-     154,    -1,   156,    41,    42,    43,    44,    45,    46,    47,
-      48,    49,    50,    51,    52,    53,    54,    55,    56,    57,
-      58,    59,    60,    61,    62,    63,    64,    65,    66,    67,
-      68,    69,    70,    71,    72,    73,    74,    -1,    76,   193,
-      -1,    -1,    -1,    -1,    -1,   199,   200,   201,   202,    -1,
+     104,   105,   106,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
+      -1,    -1,    -1,    -1,    -1,    -1,    -1,   121,    -1,    -1,
       -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
-     214,    -1,    -1,    -1,    -1,    -1,   104,   105,   106,    -1,
       -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
-      -1,    -1,    -1,     5,     6,     7,     8,    -1,    -1,    -1,
-      -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    20,    21,
-      22,    23,    24,    25,    26,    27,    28,    29,    30,    31,
-      -1,    -1,    -1,    -1,    -1,    -1,   154,    -1,   156,    41,
-      42,    43,    44,    45,    46,    47,    48,    49,    50,    51,
-      52,    53,    54,    55,    56,    57,    58,    59,    60,    61,
-      62,    63,    64,    65,    66,    67,    68,    69,    70,    71,
-      72,    73,    74,    -1,    76,    -1,    -1,    -1,    -1,    -1,
       -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
-      -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,   216,    -1,
-      -1,    -1,   104,   105,   106,    -1,    -1,    -1,    -1,    -1,
-      -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,     5,
-       6,     7,     8,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
-      -1,    -1,    -1,    -1,    20,    21,    22,    23,    24,    25,
-      26,    27,    28,    29,    30,    31,    -1,    -1,    -1,    -1,
-      -1,    -1,   154,    -1,   156,    41,    42,    43,    44,    45,
-      46,    47,    48,    49,    50,    51,    52,    53,    54,    55,
-      56,    57,    58,    59,    60,    61,    62,    63,    64,    65,
-      66,    67,    68,    69,    70,    71,    72,    73,    74,    -1,
-      76,    77,    78,    79,    80,    81,    -1,    -1,    84,    85,
+      -1,   155,    -1,   157,    -1,    -1,    -1,    -1,    -1,    -1,
       -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
-      -1,    -1,    -1,    -1,   216,    -1,    -1,    -1,   104,   105,
-     106,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
       -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
+      -1,    -1,    -1,    -1,    -1,     5,     6,     7,     8,    -1,
+     194,    -1,    -1,    -1,    -1,    -1,   200,   201,   202,   203,
+      20,    21,    22,    23,    24,    25,    26,    27,    28,    29,
+      30,    31,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
+      -1,    41,    42,    43,    44,    45,    46,    47,    48,    49,
+      50,    51,    52,    53,    54,    55,    56,    57,    58,    59,
+      60,    61,    62,    63,    64,    65,    66,    67,    68,    69,
+      70,    71,    72,    73,    74,    -1,    76,    77,    78,    79,
+      80,    81,    -1,    -1,    84,    85,    -1,    -1,    -1,    -1,
+      -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
+      -1,    -1,    -1,    -1,   104,   105,   106,    -1,    -1,    -1,
       -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
       -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
-      -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,   154,    -1,
-     156,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
       -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
       -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
-      -1,    -1,    -1,    -1,    -1,    -1,    -1,   193,    -1,    -1,
-      -1,    -1,    -1,   199,   200,   201,   202,     3,     4,     5,
-       6,     7,     8,    -1,    -1,    -1,    -1,    -1,   214,    -1,
-      -1,    -1,    -1,    -1,    20,    21,    22,    23,    24,    25,
-      26,    27,    28,    29,    30,    31,    32,    33,    34,    -1,
-      36,    37,    38,    39,    40,    41,    42,    43,    44,    45,
+      -1,    -1,    -1,    -1,    -1,   155,    -1,   157,    -1,    -1,
+      -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
+      -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
+      -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,     5,
+       6,     7,     8,    -1,   194,    -1,    -1,   197,    -1,    -1,
+     200,   201,   202,   203,    20,    21,    22,    23,    24,    25,
+      26,    27,    28,    29,    30,    31,    -1,    -1,    -1,    -1,
+      -1,    -1,    -1,    -1,    -1,    41,    42,    43,    44,    45,
       46,    47,    48,    49,    50,    51,    52,    53,    54,    55,
       56,    57,    58,    59,    60,    61,    62,    63,    64,    65,
       66,    67,    68,    69,    70,    71,    72,    73,    74,    -1,
       76,    77,    78,    79,    80,    81,    -1,    -1,    84,    85,
       -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
-      -1,    -1,    -1,    -1,    -1,    -1,    -1,   103,   104,   105,
+      -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,   104,   105,
      106,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
-      -1,    -1,    -1,    -1,   120,    -1,    -1,    -1,    -1,    -1,
       -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
       -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
-      -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,   154,    -1,
-     156,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
-      -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
-      -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
-      -1,    -1,     5,     6,     7,     8,    -1,   193,    -1,    -1,
-      -1,    -1,    -1,   199,   200,   201,   202,    20,    21,    22,
-      23,    24,    25,    26,    27,    28,    29,    30,    31,    -1,
-      -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    41,    42,
-      43,    44,    45,    46,    47,    48,    49,    50,    51,    52,
-      53,    54,    55,    56,    57,    58,    59,    60,    61,    62,
-      63,    64,    65,    66,    67,    68,    69,    70,    71,    72,
-      73,    74,    -1,    76,    77,    78,    79,    80,    81,    -1,
-      -1,    84,    85,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
-      -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
-      -1,   104,   105,   106,    -1,    -1,    -1,    -1,    -1,    -1,
       -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
+      -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,   155,
+      -1,   157,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
       -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
       -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
+      -1,    -1,    -1,     5,     6,     7,     8,    -1,   194,    -1,
+      -1,   197,    -1,    -1,   200,   201,   202,   203,    20,    21,
+      22,    23,    24,    25,    26,    27,    28,    29,    30,    31,
+      -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    41,
+      42,    43,    44,    45,    46,    47,    48,    49,    50,    51,
+      52,    53,    54,    55,    56,    57,    58,    59,    60,    61,
+      62,    63,    64,    65,    66,    67,    68,    69,    70,    71,
+      72,    73,    74,    -1,    76,    77,    78,    79,    80,    81,
+      -1,    -1,    84,    85,    -1,    -1,    -1,    -1,    -1,    -1,
       -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
-      -1,   154,    -1,   156,    -1,    -1,    -1,    -1,    -1,    -1,
-      -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
-      -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
-      -1,    -1,    -1,    -1,    -1,     5,     6,     7,     8,    -1,
-     193,    -1,    -1,   196,    -1,    -1,   199,   200,   201,   202,
-      20,    21,    22,    23,    24,    25,    26,    27,    28,    29,
-      30,    31,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
-      -1,    41,    42,    43,    44,    45,    46,    47,    48,    49,
-      50,    51,    52,    53,    54,    55,    56,    57,    58,    59,
-      60,    61,    62,    63,    64,    65,    66,    67,    68,    69,
-      70,    71,    72,    73,    74,    -1,    76,    77,    78,    79,
-      80,    81,    -1,    -1,    84,    85,    -1,    -1,    -1,    -1,
-      -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
-      -1,    -1,    -1,    -1,   104,   105,   106,    -1,    -1,    -1,
+      -1,    -1,   104,   105,   106,    -1,    -1,    -1,    -1,    -1,
       -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
       -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
       -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
       -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
-      -1,    -1,    -1,    -1,   154,    -1,   156,    -1,    -1,    -1,
+      -1,    -1,    -1,   155,    -1,   157,    -1,    -1,    -1,    -1,
       -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
       -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
-      -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,     5,     6,
-       7,     8,    -1,   193,    -1,    -1,   196,    -1,    -1,   199,
-     200,   201,   202,    20,    21,    22,    23,    24,    25,    26,
-      27,    28,    29,    30,    31,    -1,    -1,    -1,    -1,    -1,
-      -1,    -1,    -1,    -1,    41,    42,    43,    44,    45,    46,
-      47,    48,    49,    50,    51,    52,    53,    54,    55,    56,
-      57,    58,    59,    60,    61,    62,    63,    64,    65,    66,
-      67,    68,    69,    70,    71,    72,    73,    74,    -1,    76,
-      77,    78,    79,    80,    81,    -1,    -1,    84,    85,    -1,
+      -1,    -1,    -1,    -1,    -1,    -1,    -1,     5,     6,     7,
+       8,    -1,   194,    -1,    -1,   197,    -1,    -1,   200,   201,
+     202,   203,    20,    21,    22,    23,    24,    25,    26,    27,
+      28,    29,    30,    31,    -1,    -1,    -1,    -1,    -1,    -1,
+      -1,    -1,    -1,    41,    42,    43,    44,    45,    46,    47,
+      48,    49,    50,    51,    52,    53,    54,    55,    56,    57,
+      58,    59,    60,    61,    62,    63,    64,    65,    66,    67,
+      68,    69,    70,    71,    72,    73,    74,    -1,    76,    77,
+      78,    79,    80,    81,    -1,    -1,    84,    85,    -1,    -1,
       -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
-      -1,    -1,    -1,    -1,    -1,    -1,    -1,   104,   105,   106,
+      -1,    -1,    -1,    -1,    -1,    -1,   104,   105,   106,    -1,
       -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
       -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
       -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
       -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
-      -1,    -1,    -1,    -1,    -1,    -1,    -1,   154,    -1,   156,
+      -1,    -1,    -1,    -1,    -1,    -1,    -1,   155,    -1,   157,
       -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
       -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
       -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
-      -1,     5,     6,     7,     8,    -1,   193,    -1,    -1,   196,
-      -1,    -1,   199,   200,   201,   202,    20,    21,    22,    23,
+      -1,     5,     6,     7,     8,    -1,   194,    -1,    -1,    -1,
+      -1,    -1,   200,   201,   202,   203,    20,    21,    22,    23,
       24,    25,    26,    27,    28,    29,    30,    31,    -1,    -1,
       -1,    -1,    -1,    -1,    -1,    -1,    -1,    41,    42,    43,
       44,    45,    46,    47,    48,    49,    50,    51,    52,    53,
@@ -1830,117 +1854,99 @@ static const yytype_int16 yycheck[] =
       -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
       -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
       -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
-     154,    -1,   156,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
+      -1,   155,    -1,   157,    -1,    -1,    -1,    -1,    -1,    -1,
       -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
       -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
-      -1,    -1,    -1,    -1,     5,     6,     7,     8,    -1,   193,
-      -1,    -1,    -1,    -1,    -1,   199,   200,   201,   202,    20,
+      -1,    -1,    -1,     3,     4,     5,     6,     7,     8,    -1,
+     194,    -1,    -1,    -1,    -1,    -1,   200,   201,   202,   203,
+      20,    21,    22,    23,    24,    25,    26,    27,    28,    29,
+      30,    31,    32,    33,    34,    -1,    36,    37,    38,    39,
+      40,    41,    42,    43,    44,    45,    46,    47,    48,    49,
+      50,    51,    52,    53,    54,    55,    56,    57,    58,    59,
+      60,    61,    62,    63,    64,    65,    66,    67,    68,    69,
+      70,    71,    72,    73,    74,    -1,    76,    -1,    -1,    -1,
+      -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
+      -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
+      -1,    -1,    -1,   103,   104,   105,   106,    -1,   108,    -1,
+      -1,    -1,    -1,    -1,    -1,    -1,   116,   117,   118,   119,
+     120,   121,     5,     6,     7,     8,    -1,    -1,    -1,    -1,
+      -1,    -1,    -1,    -1,    -1,    -1,    -1,    20,    21,    22,
+      23,    24,    25,    26,    27,    28,    29,    30,    31,    -1,
+      -1,    -1,    -1,    -1,    -1,   155,    -1,   157,    41,    42,
+      43,    44,    45,    46,    47,    48,    49,    50,    51,    52,
+      53,    54,    55,    56,    57,    58,    59,    60,    61,    62,
+      63,    64,    65,    66,    67,    68,    69,    70,    71,    72,
+      73,    74,    -1,    76,     5,     6,     7,     8,    81,    -1,
+      -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    20,
       21,    22,    23,    24,    25,    26,    27,    28,    29,    30,
-      31,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
+      31,   104,   105,   106,    -1,    -1,    -1,    -1,    -1,    -1,
       41,    42,    43,    44,    45,    46,    47,    48,    49,    50,
       51,    52,    53,    54,    55,    56,    57,    58,    59,    60,
       61,    62,    63,    64,    65,    66,    67,    68,    69,    70,
-      71,    72,    73,    74,    -1,    76,    77,    78,    79,    80,
-      81,    -1,    -1,    84,    85,    -1,    -1,    -1,    -1,    -1,
-      -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
-      -1,    -1,    -1,   104,   105,   106,    -1,    -1,    -1,    -1,
-      -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
-      -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
-      -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
-      -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
-      -1,    -1,    -1,   154,    -1,   156,    -1,    -1,    -1,    -1,
-      -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
-      -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
-      -1,    -1,    -1,    -1,    -1,     3,     4,     5,     6,     7,
-       8,    -1,   193,    -1,    -1,    -1,    -1,    -1,   199,   200,
-     201,   202,    20,    21,    22,    23,    24,    25,    26,    27,
-      28,    29,    30,    31,    32,    33,    34,    -1,    36,    37,
-      38,    39,    40,    41,    42,    43,    44,    45,    46,    47,
-      48,    49,    50,    51,    52,    53,    54,    55,    56,    57,
-      58,    59,    60,    61,    62,    63,    64,    65,    66,    67,
-      68,    69,    70,    71,    72,    73,    74,    -1,    76,    -1,
-      -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
-      -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
-      -1,    -1,    -1,    -1,    -1,   103,   104,   105,   106,    -1,
-     108,    -1,    -1,    -1,    -1,    -1,    -1,    -1,   116,   117,
-     118,   119,   120,     5,     6,     7,     8,    -1,    -1,    -1,
-      -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    20,    21,
-      22,    23,    24,    25,    26,    27,    28,    29,    30,    31,
-      -1,    -1,    -1,    -1,    -1,    -1,   154,    -1,   156,    41,
-      42,    43,    44,    45,    46,    47,    48,    49,    50,    51,
-      52,    53,    54,    55,    56,    57,    58,    59,    60,    61,
-      62,    63,    64,    65,    66,    67,    68,    69,    70,    71,
-      72,    73,    74,    -1,    76,     5,     6,     7,     8,    81,
-      -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
-      20,    21,    22,    23,    24,    25,    26,    27,    28,    29,
-      30,    31,   104,   105,   106,    -1,    -1,    -1,    -1,    -1,
-      -1,    41,    42,    43,    44,    45,    46,    47,    48,    49,
-      50,    51,    52,    53,    54,    55,    56,    57,    58,    59,
-      60,    61,    62,    63,    64,    65,    66,    67,    68,    69,
-      70,    71,    72,    73,    74,    -1,    76,     5,     6,     7,
-       8,    -1,   154,    -1,   156,    -1,    -1,    -1,    -1,    -1,
-      -1,    -1,    20,    21,    22,    23,    24,    25,    26,    27,
-      28,    29,    30,    31,   104,   105,   106,    -1,    -1,    -1,
-      -1,    -1,    -1,    41,    42,    43,    44,    45,    46,    47,
-      48,    49,    50,    51,    52,    53,    54,    55,    56,    57,
-      58,    59,    60,    61,    62,    63,    64,    65,    66,    67,
-      68,    69,    70,    71,    72,    73,    74,    -1,    76,    -1,
-      -1,    -1,    -1,    -1,   154,    -1,   156,    -1,    -1,    -1,
+      71,    72,    73,    74,    -1,    76,     5,     6,     7,     8,
+      -1,    -1,   155,    -1,   157,    -1,    -1,    -1,    -1,    -1,
+      -1,    20,    21,    22,    23,    24,    25,    26,    27,    28,
+      29,    30,    31,   104,   105,   106,    -1,    -1,    -1,    -1,
+      -1,    -1,    41,    42,    43,    44,    45,    46,    47,    48,
+      49,    50,    51,    52,    53,    54,    55,    56,    57,    58,
+      59,    60,    61,    62,    63,    64,    65,    66,    67,    68,
+      69,    70,    71,    72,    73,    74,    -1,    76,    -1,    -1,
+      -1,    -1,    -1,    -1,   155,    -1,   157,    -1,    -1,    -1,
       -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
       -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
       -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
       -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
       -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
       -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,    -1,
-      -1,    -1,    -1,    -1,    -1,    -1,   154,    -1,   156
+      -1,    -1,    -1,    -1,    -1,    -1,   155,    -1,   157
 };
 
 /* YYSTOS[STATE-NUM] -- The (internal number of the) accessing
    symbol of state STATE-NUM.  */
 static const yytype_uint16 yystos[] =
 {
-       0,   109,   218,   220,    78,     0,   222,   113,   110,   219,
-     223,    76,     3,     4,     5,     6,     7,     8,    20,    21,
+       0,   109,   219,   221,    78,     0,   223,   113,   110,   220,
+     224,    76,     3,     4,     5,     6,     7,     8,    20,    21,
       22,    23,    24,    25,    26,    27,    28,    29,    30,    31,
       32,    33,    34,    36,    37,    38,    39,    40,    41,    42,
       43,    44,    45,    46,    47,    48,    49,    50,    51,    52,
       53,    54,    55,    56,    57,    58,    59,    60,    61,    62,
       63,    64,    65,    66,    67,    68,    69,    70,    71,    72,
       73,    74,    76,   103,   104,   105,   106,   108,   116,   117,
-     118,   119,   120,   154,   156,   221,   224,   254,   255,   256,
-     257,   258,   263,   264,   265,   266,   269,   271,   272,   273,
-     274,   275,   276,   277,   278,   302,   303,   112,    33,    34,
-      37,    76,   215,    76,   269,   272,   277,   113,   113,   113,
-     113,   193,   302,   214,   215,   289,   194,   198,     4,    33,
-      34,    35,   260,   261,   270,   198,   214,    76,   272,   272,
-     273,   195,   274,    76,   215,   273,   279,   280,   272,   274,
-      76,   267,   268,     9,    10,    11,    13,    14,    15,    16,
-      17,    18,    19,    75,    76,    77,    78,    79,    80,    81,
-      84,    85,   193,   199,   200,   201,   202,   214,   215,   216,
-     225,   226,   227,   229,   230,   231,   232,   233,   234,   235,
-     236,   237,   238,   239,   240,   241,   242,   243,   244,   245,
-     246,   247,   248,   249,   250,   252,   254,   255,   273,   284,
-     285,   286,   287,   290,   291,   292,   295,   296,   297,   301,
-     260,   259,   262,   273,   261,    76,   193,   195,   213,   196,
-     236,   249,   253,   273,   113,   279,    76,   281,   282,   216,
-     280,   214,   213,   194,   198,   214,   214,   285,   193,   193,
-     214,   214,   252,   193,   252,   212,   193,   236,   236,   252,
-     216,   290,    84,    85,   195,   197,   194,   194,   198,    74,
-     250,   193,    93,    94,    95,    96,    97,    98,    99,   100,
-     101,   102,   213,   251,   236,   203,   204,   205,   199,   200,
-      82,    83,    86,    87,   206,   207,    88,    89,   208,   209,
-     210,    90,    92,    91,   211,   198,   214,   216,   285,    76,
-     259,   262,   195,   213,   196,   253,   250,   283,   196,   216,
-     195,   198,   214,    78,   268,    75,   284,   291,   298,   252,
-     214,   252,   212,   103,   252,   265,   294,   194,   216,   228,
-     252,    76,   231,   250,   250,   236,   236,   236,   238,   238,
-     239,   239,   240,   240,   240,   240,   241,   241,   242,   243,
-     244,   245,   246,   247,   252,   250,   195,   196,   253,   283,
-     213,   196,   253,   282,   193,   294,   299,   300,   194,   194,
-      76,   194,   196,   212,   253,   213,   196,   283,   213,   196,
-     252,   214,   194,   285,   293,   287,   213,   286,   288,   289,
-     250,   196,   283,   213,   283,   194,   252,   288,    12,   283,
-     283,   214,   285
+     118,   119,   120,   121,   155,   157,   222,   225,   255,   256,
+     257,   258,   259,   264,   265,   266,   267,   270,   272,   273,
+     274,   275,   276,   277,   278,   279,   303,   304,   112,    33,
+      34,    37,    76,   216,    76,   270,   273,   278,   113,   113,
+     113,   113,   113,   194,   303,   215,   216,   290,   195,   199,
+       4,    33,    34,    35,   261,   262,   271,   199,   215,    76,
+     273,   273,   274,   196,   275,    76,   216,   274,   280,   281,
+     273,   275,    76,   268,   269,     9,    10,    11,    13,    14,
+      15,    16,    17,    18,    19,    75,    76,    77,    78,    79,
+      80,    81,    84,    85,   194,   200,   201,   202,   203,   215,
+     216,   217,   226,   227,   228,   230,   231,   232,   233,   234,
+     235,   236,   237,   238,   239,   240,   241,   242,   243,   244,
+     245,   246,   247,   248,   249,   250,   251,   253,   255,   256,
+     274,   285,   286,   287,   288,   291,   292,   293,   296,   297,
+     298,   302,   261,   260,   263,   274,   262,    76,   194,   196,
+     214,   197,   237,   250,   254,   274,   113,   280,    76,   282,
+     283,   217,   281,   215,   214,   195,   199,   215,   215,   286,
+     194,   194,   215,   215,   253,   194,   253,   213,   194,   237,
+     237,   253,   217,   291,    84,    85,   196,   198,   195,   195,
+     199,    74,   251,   194,    93,    94,    95,    96,    97,    98,
+      99,   100,   101,   102,   214,   252,   237,   204,   205,   206,
+     200,   201,    82,    83,    86,    87,   207,   208,    88,    89,
+     209,   210,   211,    90,    92,    91,   212,   199,   215,   217,
+     286,    76,   260,   263,   196,   214,   197,   254,   251,   284,
+     197,   217,   196,   199,   215,    78,   269,    75,   285,   292,
+     299,   253,   215,   253,   213,   103,   253,   266,   295,   195,
+     217,   229,   253,    76,   232,   251,   251,   237,   237,   237,
+     239,   239,   240,   240,   241,   241,   241,   241,   242,   242,
+     243,   244,   245,   246,   247,   248,   253,   251,   196,   197,
+     254,   284,   214,   197,   254,   283,   194,   295,   300,   301,
+     195,   195,    76,   195,   197,   213,   254,   214,   197,   284,
+     214,   197,   253,   215,   195,   286,   294,   288,   214,   287,
+     289,   290,   251,   197,   284,   214,   284,   195,   253,   289,
+      12,   284,   284,   215,   286
 };
 
 #define yyerrok		(yyerrstatus = 0)
@@ -1955,18 +1961,9 @@ static const yytype_uint16 yystos[] =
 
 /* Like YYERROR except do call yyerror.  This remains here temporarily
    to ease the transition to the new meaning of YYERROR, for GCC.
-   Once GCC version 2 has supplanted version 1, this can go.  However,
-   YYFAIL appears to be in use.  Nevertheless, it is formally deprecated
-   in Bison 2.4.2's NEWS entry, where a plan to phase it out is
-   discussed.  */
+   Once GCC version 2 has supplanted version 1, this can go.  */
 
 #define YYFAIL		goto yyerrlab
-#if defined YYFAIL
-  /* This is here to suppress warnings from the GCC cpp's
-     -Wunused-macros.  Normally we don't worry about that warning, but
-     some users do, and we want to make it easy for users to remove
-     YYFAIL uses, which will produce warnings from Bison 2.5.  */
-#endif
 
 #define YYRECOVERING()  (!!yyerrstatus)
 
@@ -2023,7 +2020,7 @@ while (YYID (0))
    we won't break user code: when these are the locations we know.  */
 
 #ifndef YY_LOCATION_PRINT
-# if defined YYLTYPE_IS_TRIVIAL && YYLTYPE_IS_TRIVIAL
+# if YYLTYPE_IS_TRIVIAL
 #  define YY_LOCATION_PRINT(File, Loc)			\
      fprintf (File, "%d.%d-%d.%d",			\
 	      (Loc).first_line, (Loc).first_column,	\
@@ -2565,7 +2562,7 @@ YYLTYPE yylloc;
     YYLTYPE *yylsp;
 
     /* The locations where the error started and ended.  */
-    YYLTYPE yyerror_range[3];
+    YYLTYPE yyerror_range[2];
 
     YYSIZE_T yystacksize;
 
@@ -2612,7 +2609,7 @@ YYLTYPE yylloc;
   yyvsp = yyvs;
   yylsp = yyls;
 
-#if defined YYLTYPE_IS_TRIVIAL && YYLTYPE_IS_TRIVIAL
+#if YYLTYPE_IS_TRIVIAL
   /* Initialize the default location before parsing starts.  */
   yylloc.first_line   = yylloc.last_line   = 1;
   yylloc.first_column = yylloc.last_column = 1;
@@ -2620,7 +2617,7 @@ YYLTYPE yylloc;
 
 /* User initialization code.  */
 
-/* Line 1251 of yacc.c  */
+/* Line 1242 of yacc.c  */
 #line 41 "glsl_parser.ypp"
 {
    yylloc.first_line = 1;
@@ -2630,8 +2627,8 @@ YYLTYPE yylloc;
    yylloc.source = 0;
 }
 
-/* Line 1251 of yacc.c  */
-#line 2635 "glsl_parser.cpp"
+/* Line 1242 of yacc.c  */
+#line 2632 "glsl_parser.cpp"
   yylsp[0] = yylloc;
 
   goto yysetstate;
@@ -2818,8 +2815,8 @@ yyreduce:
     {
         case 2:
 
-/* Line 1464 of yacc.c  */
-#line 212 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 213 "glsl_parser.ypp"
     {
 	   _mesa_glsl_initialize_types(state);
 	;}
@@ -2827,8 +2824,8 @@ yyreduce:
 
   case 5:
 
-/* Line 1464 of yacc.c  */
-#line 221 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 222 "glsl_parser.ypp"
     {
 	   switch ((yyvsp[(2) - (3)].n)) {
 	   case 100:
@@ -2847,10 +2844,28 @@ yyreduce:
 	;}
     break;
 
-  case 12:
+  case 10:
 
-/* Line 1464 of yacc.c  */
-#line 253 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 246 "glsl_parser.ypp"
+    {
+	   if (state->language_version < 120) {
+	      _mesa_glsl_warning(& (yylsp[(1) - (2)]), state,
+				 "pragma `invariant(all)' not supported in "
+				 "GLSL%s %d.%02d",
+				 state->es_shader ? " ES" : "",
+				 state->language_version / 100,
+				 state->language_version % 100);
+	   } else {
+	      state->all_invariant = true;
+	   }
+	;}
+    break;
+
+  case 13:
+
+/* Line 1455 of yacc.c  */
+#line 267 "glsl_parser.ypp"
     {
 	   if (!_mesa_glsl_process_extension((yyvsp[(2) - (5)].identifier), & (yylsp[(2) - (5)]), (yyvsp[(4) - (5)].identifier), & (yylsp[(4) - (5)]), state)) {
 	      YYERROR;
@@ -2858,10 +2873,10 @@ yyreduce:
 	;}
     break;
 
-  case 13:
+  case 14:
 
-/* Line 1464 of yacc.c  */
-#line 262 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 276 "glsl_parser.ypp"
     {
 	   /* FINISHME: The NULL test is only required because 'precision'
 	    * FINISHME: statements are not yet supported.
@@ -2871,10 +2886,10 @@ yyreduce:
 	;}
     break;
 
-  case 14:
+  case 15:
 
-/* Line 1464 of yacc.c  */
-#line 270 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 284 "glsl_parser.ypp"
     {
 	   /* FINISHME: The NULL test is only required because 'precision'
 	    * FINISHME: statements are not yet supported.
@@ -2884,10 +2899,10 @@ yyreduce:
 	;}
     break;
 
-  case 16:
+  case 17:
 
-/* Line 1464 of yacc.c  */
-#line 285 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 299 "glsl_parser.ypp"
     {
 	   void *ctx = state;
 	   (yyval.expression) = new(ctx) ast_expression(ast_identifier, NULL, NULL, NULL);
@@ -2896,10 +2911,10 @@ yyreduce:
 	;}
     break;
 
-  case 17:
+  case 18:
 
-/* Line 1464 of yacc.c  */
-#line 292 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 306 "glsl_parser.ypp"
     {
 	   void *ctx = state;
 	   (yyval.expression) = new(ctx) ast_expression(ast_int_constant, NULL, NULL, NULL);
@@ -2908,10 +2923,10 @@ yyreduce:
 	;}
     break;
 
-  case 18:
+  case 19:
 
-/* Line 1464 of yacc.c  */
-#line 299 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 313 "glsl_parser.ypp"
     {
 	   void *ctx = state;
 	   (yyval.expression) = new(ctx) ast_expression(ast_uint_constant, NULL, NULL, NULL);
@@ -2920,10 +2935,10 @@ yyreduce:
 	;}
     break;
 
-  case 19:
+  case 20:
 
-/* Line 1464 of yacc.c  */
-#line 306 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 320 "glsl_parser.ypp"
     {
 	   void *ctx = state;
 	   (yyval.expression) = new(ctx) ast_expression(ast_float_constant, NULL, NULL, NULL);
@@ -2932,10 +2947,10 @@ yyreduce:
 	;}
     break;
 
-  case 20:
+  case 21:
 
-/* Line 1464 of yacc.c  */
-#line 313 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 327 "glsl_parser.ypp"
     {
 	   void *ctx = state;
 	   (yyval.expression) = new(ctx) ast_expression(ast_bool_constant, NULL, NULL, NULL);
@@ -2944,19 +2959,19 @@ yyreduce:
 	;}
     break;
 
-  case 21:
+  case 22:
 
-/* Line 1464 of yacc.c  */
-#line 320 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 334 "glsl_parser.ypp"
     {
 	   (yyval.expression) = (yyvsp[(2) - (3)].expression);
 	;}
     break;
 
-  case 23:
+  case 24:
 
-/* Line 1464 of yacc.c  */
-#line 328 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 342 "glsl_parser.ypp"
     {
 	   void *ctx = state;
 	   (yyval.expression) = new(ctx) ast_expression(ast_array_index, (yyvsp[(1) - (4)].expression), (yyvsp[(3) - (4)].expression), NULL);
@@ -2964,19 +2979,19 @@ yyreduce:
 	;}
     break;
 
-  case 24:
+  case 25:
 
-/* Line 1464 of yacc.c  */
-#line 334 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 348 "glsl_parser.ypp"
     {
 	   (yyval.expression) = (yyvsp[(1) - (1)].expression);
 	;}
     break;
 
-  case 25:
+  case 26:
 
-/* Line 1464 of yacc.c  */
-#line 338 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 352 "glsl_parser.ypp"
     {
 	   void *ctx = state;
 	   (yyval.expression) = new(ctx) ast_expression(ast_field_selection, (yyvsp[(1) - (3)].expression), NULL, NULL);
@@ -2985,10 +3000,10 @@ yyreduce:
 	;}
     break;
 
-  case 26:
+  case 27:
 
-/* Line 1464 of yacc.c  */
-#line 345 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 359 "glsl_parser.ypp"
     {
 	   void *ctx = state;
 	   (yyval.expression) = new(ctx) ast_expression(ast_post_inc, (yyvsp[(1) - (2)].expression), NULL, NULL);
@@ -2996,10 +3011,10 @@ yyreduce:
 	;}
     break;
 
-  case 27:
+  case 28:
 
-/* Line 1464 of yacc.c  */
-#line 351 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 365 "glsl_parser.ypp"
     {
 	   void *ctx = state;
 	   (yyval.expression) = new(ctx) ast_expression(ast_post_dec, (yyvsp[(1) - (2)].expression), NULL, NULL);
@@ -3007,10 +3022,10 @@ yyreduce:
 	;}
     break;
 
-  case 31:
+  case 32:
 
-/* Line 1464 of yacc.c  */
-#line 369 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 383 "glsl_parser.ypp"
     {
 	   void *ctx = state;
 	   (yyval.expression) = new(ctx) ast_expression(ast_field_selection, (yyvsp[(1) - (3)].expression), (yyvsp[(3) - (3)].expression), NULL);
@@ -3018,10 +3033,10 @@ yyreduce:
 	;}
     break;
 
-  case 36:
+  case 37:
 
-/* Line 1464 of yacc.c  */
-#line 388 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 402 "glsl_parser.ypp"
     {
 	   (yyval.expression) = (yyvsp[(1) - (2)].expression);
 	   (yyval.expression)->set_location(yylloc);
@@ -3029,10 +3044,10 @@ yyreduce:
 	;}
     break;
 
-  case 37:
+  case 38:
 
-/* Line 1464 of yacc.c  */
-#line 394 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 408 "glsl_parser.ypp"
     {
 	   (yyval.expression) = (yyvsp[(1) - (3)].expression);
 	   (yyval.expression)->set_location(yylloc);
@@ -3040,10 +3055,10 @@ yyreduce:
 	;}
     break;
 
-  case 39:
+  case 40:
 
-/* Line 1464 of yacc.c  */
-#line 410 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 424 "glsl_parser.ypp"
     {
 	   void *ctx = state;
 	   (yyval.expression) = new(ctx) ast_function_expression((yyvsp[(1) - (1)].type_specifier));
@@ -3051,10 +3066,10 @@ yyreduce:
    	;}
     break;
 
-  case 40:
+  case 41:
 
-/* Line 1464 of yacc.c  */
-#line 416 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 430 "glsl_parser.ypp"
     {
 	   void *ctx = state;
 	   ast_expression *callee = new(ctx) ast_expression((yyvsp[(1) - (1)].identifier));
@@ -3063,10 +3078,10 @@ yyreduce:
    	;}
     break;
 
-  case 41:
+  case 42:
 
-/* Line 1464 of yacc.c  */
-#line 423 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 437 "glsl_parser.ypp"
     {
 	   void *ctx = state;
 	   ast_expression *callee = new(ctx) ast_expression((yyvsp[(1) - (1)].identifier));
@@ -3075,10 +3090,10 @@ yyreduce:
    	;}
     break;
 
-  case 43:
+  case 44:
 
-/* Line 1464 of yacc.c  */
-#line 435 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 449 "glsl_parser.ypp"
     {
 	   void *ctx = state;
 	   (yyval.expression) = new(ctx) ast_expression(ast_pre_inc, (yyvsp[(2) - (2)].expression), NULL, NULL);
@@ -3086,10 +3101,10 @@ yyreduce:
 	;}
     break;
 
-  case 44:
+  case 45:
 
-/* Line 1464 of yacc.c  */
-#line 441 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 455 "glsl_parser.ypp"
     {
 	   void *ctx = state;
 	   (yyval.expression) = new(ctx) ast_expression(ast_pre_dec, (yyvsp[(2) - (2)].expression), NULL, NULL);
@@ -3097,10 +3112,10 @@ yyreduce:
 	;}
     break;
 
-  case 45:
+  case 46:
 
-/* Line 1464 of yacc.c  */
-#line 447 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 461 "glsl_parser.ypp"
     {
 	   void *ctx = state;
 	   (yyval.expression) = new(ctx) ast_expression((yyvsp[(1) - (2)].n), (yyvsp[(2) - (2)].expression), NULL, NULL);
@@ -3108,38 +3123,38 @@ yyreduce:
 	;}
     break;
 
-  case 46:
+  case 47:
 
-/* Line 1464 of yacc.c  */
-#line 456 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 470 "glsl_parser.ypp"
     { (yyval.n) = ast_plus; ;}
     break;
 
-  case 47:
+  case 48:
 
-/* Line 1464 of yacc.c  */
-#line 457 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 471 "glsl_parser.ypp"
     { (yyval.n) = ast_neg; ;}
     break;
 
-  case 48:
+  case 49:
 
-/* Line 1464 of yacc.c  */
-#line 458 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 472 "glsl_parser.ypp"
     { (yyval.n) = ast_logic_not; ;}
     break;
 
-  case 49:
+  case 50:
 
-/* Line 1464 of yacc.c  */
-#line 459 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 473 "glsl_parser.ypp"
     { (yyval.n) = ast_bit_not; ;}
     break;
 
-  case 51:
+  case 52:
 
-/* Line 1464 of yacc.c  */
-#line 465 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 479 "glsl_parser.ypp"
     {
 	   void *ctx = state;
 	   (yyval.expression) = new(ctx) ast_expression_bin(ast_mul, (yyvsp[(1) - (3)].expression), (yyvsp[(3) - (3)].expression));
@@ -3147,10 +3162,10 @@ yyreduce:
 	;}
     break;
 
-  case 52:
+  case 53:
 
-/* Line 1464 of yacc.c  */
-#line 471 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 485 "glsl_parser.ypp"
     {
 	   void *ctx = state;
 	   (yyval.expression) = new(ctx) ast_expression_bin(ast_div, (yyvsp[(1) - (3)].expression), (yyvsp[(3) - (3)].expression));
@@ -3158,10 +3173,10 @@ yyreduce:
 	;}
     break;
 
-  case 53:
+  case 54:
 
-/* Line 1464 of yacc.c  */
-#line 477 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 491 "glsl_parser.ypp"
     {
 	   void *ctx = state;
 	   (yyval.expression) = new(ctx) ast_expression_bin(ast_mod, (yyvsp[(1) - (3)].expression), (yyvsp[(3) - (3)].expression));
@@ -3169,10 +3184,10 @@ yyreduce:
 	;}
     break;
 
-  case 55:
+  case 56:
 
-/* Line 1464 of yacc.c  */
-#line 487 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 501 "glsl_parser.ypp"
     {
 	   void *ctx = state;
 	   (yyval.expression) = new(ctx) ast_expression_bin(ast_add, (yyvsp[(1) - (3)].expression), (yyvsp[(3) - (3)].expression));
@@ -3180,10 +3195,10 @@ yyreduce:
 	;}
     break;
 
-  case 56:
+  case 57:
 
-/* Line 1464 of yacc.c  */
-#line 493 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 507 "glsl_parser.ypp"
     {
 	   void *ctx = state;
 	   (yyval.expression) = new(ctx) ast_expression_bin(ast_sub, (yyvsp[(1) - (3)].expression), (yyvsp[(3) - (3)].expression));
@@ -3191,10 +3206,10 @@ yyreduce:
 	;}
     break;
 
-  case 58:
+  case 59:
 
-/* Line 1464 of yacc.c  */
-#line 503 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 517 "glsl_parser.ypp"
     {
 	   void *ctx = state;
 	   (yyval.expression) = new(ctx) ast_expression_bin(ast_lshift, (yyvsp[(1) - (3)].expression), (yyvsp[(3) - (3)].expression));
@@ -3202,10 +3217,10 @@ yyreduce:
 	;}
     break;
 
-  case 59:
+  case 60:
 
-/* Line 1464 of yacc.c  */
-#line 509 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 523 "glsl_parser.ypp"
     {
 	   void *ctx = state;
 	   (yyval.expression) = new(ctx) ast_expression_bin(ast_rshift, (yyvsp[(1) - (3)].expression), (yyvsp[(3) - (3)].expression));
@@ -3213,10 +3228,10 @@ yyreduce:
 	;}
     break;
 
-  case 61:
+  case 62:
 
-/* Line 1464 of yacc.c  */
-#line 519 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 533 "glsl_parser.ypp"
     {
 	   void *ctx = state;
 	   (yyval.expression) = new(ctx) ast_expression_bin(ast_less, (yyvsp[(1) - (3)].expression), (yyvsp[(3) - (3)].expression));
@@ -3224,10 +3239,10 @@ yyreduce:
 	;}
     break;
 
-  case 62:
+  case 63:
 
-/* Line 1464 of yacc.c  */
-#line 525 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 539 "glsl_parser.ypp"
     {
 	   void *ctx = state;
 	   (yyval.expression) = new(ctx) ast_expression_bin(ast_greater, (yyvsp[(1) - (3)].expression), (yyvsp[(3) - (3)].expression));
@@ -3235,10 +3250,10 @@ yyreduce:
 	;}
     break;
 
-  case 63:
+  case 64:
 
-/* Line 1464 of yacc.c  */
-#line 531 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 545 "glsl_parser.ypp"
     {
 	   void *ctx = state;
 	   (yyval.expression) = new(ctx) ast_expression_bin(ast_lequal, (yyvsp[(1) - (3)].expression), (yyvsp[(3) - (3)].expression));
@@ -3246,10 +3261,10 @@ yyreduce:
 	;}
     break;
 
-  case 64:
+  case 65:
 
-/* Line 1464 of yacc.c  */
-#line 537 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 551 "glsl_parser.ypp"
     {
 	   void *ctx = state;
 	   (yyval.expression) = new(ctx) ast_expression_bin(ast_gequal, (yyvsp[(1) - (3)].expression), (yyvsp[(3) - (3)].expression));
@@ -3257,10 +3272,10 @@ yyreduce:
 	;}
     break;
 
-  case 66:
+  case 67:
 
-/* Line 1464 of yacc.c  */
-#line 547 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 561 "glsl_parser.ypp"
     {
 	   void *ctx = state;
 	   (yyval.expression) = new(ctx) ast_expression_bin(ast_equal, (yyvsp[(1) - (3)].expression), (yyvsp[(3) - (3)].expression));
@@ -3268,10 +3283,10 @@ yyreduce:
 	;}
     break;
 
-  case 67:
+  case 68:
 
-/* Line 1464 of yacc.c  */
-#line 553 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 567 "glsl_parser.ypp"
     {
 	   void *ctx = state;
 	   (yyval.expression) = new(ctx) ast_expression_bin(ast_nequal, (yyvsp[(1) - (3)].expression), (yyvsp[(3) - (3)].expression));
@@ -3279,10 +3294,10 @@ yyreduce:
 	;}
     break;
 
-  case 69:
+  case 70:
 
-/* Line 1464 of yacc.c  */
-#line 563 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 577 "glsl_parser.ypp"
     {
 	   void *ctx = state;
 	   (yyval.expression) = new(ctx) ast_expression_bin(ast_bit_and, (yyvsp[(1) - (3)].expression), (yyvsp[(3) - (3)].expression));
@@ -3290,10 +3305,10 @@ yyreduce:
 	;}
     break;
 
-  case 71:
+  case 72:
 
-/* Line 1464 of yacc.c  */
-#line 573 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 587 "glsl_parser.ypp"
     {
 	   void *ctx = state;
 	   (yyval.expression) = new(ctx) ast_expression_bin(ast_bit_xor, (yyvsp[(1) - (3)].expression), (yyvsp[(3) - (3)].expression));
@@ -3301,10 +3316,10 @@ yyreduce:
 	;}
     break;
 
-  case 73:
+  case 74:
 
-/* Line 1464 of yacc.c  */
-#line 583 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 597 "glsl_parser.ypp"
     {
 	   void *ctx = state;
 	   (yyval.expression) = new(ctx) ast_expression_bin(ast_bit_or, (yyvsp[(1) - (3)].expression), (yyvsp[(3) - (3)].expression));
@@ -3312,10 +3327,10 @@ yyreduce:
 	;}
     break;
 
-  case 75:
+  case 76:
 
-/* Line 1464 of yacc.c  */
-#line 593 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 607 "glsl_parser.ypp"
     {
 	   void *ctx = state;
 	   (yyval.expression) = new(ctx) ast_expression_bin(ast_logic_and, (yyvsp[(1) - (3)].expression), (yyvsp[(3) - (3)].expression));
@@ -3323,10 +3338,10 @@ yyreduce:
 	;}
     break;
 
-  case 77:
+  case 78:
 
-/* Line 1464 of yacc.c  */
-#line 603 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 617 "glsl_parser.ypp"
     {
 	   void *ctx = state;
 	   (yyval.expression) = new(ctx) ast_expression_bin(ast_logic_xor, (yyvsp[(1) - (3)].expression), (yyvsp[(3) - (3)].expression));
@@ -3334,10 +3349,10 @@ yyreduce:
 	;}
     break;
 
-  case 79:
+  case 80:
 
-/* Line 1464 of yacc.c  */
-#line 613 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 627 "glsl_parser.ypp"
     {
 	   void *ctx = state;
 	   (yyval.expression) = new(ctx) ast_expression_bin(ast_logic_or, (yyvsp[(1) - (3)].expression), (yyvsp[(3) - (3)].expression));
@@ -3345,10 +3360,10 @@ yyreduce:
 	;}
     break;
 
-  case 81:
+  case 82:
 
-/* Line 1464 of yacc.c  */
-#line 623 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 637 "glsl_parser.ypp"
     {
 	   void *ctx = state;
 	   (yyval.expression) = new(ctx) ast_expression(ast_conditional, (yyvsp[(1) - (5)].expression), (yyvsp[(3) - (5)].expression), (yyvsp[(5) - (5)].expression));
@@ -3356,10 +3371,10 @@ yyreduce:
 	;}
     break;
 
-  case 83:
+  case 84:
 
-/* Line 1464 of yacc.c  */
-#line 633 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 647 "glsl_parser.ypp"
     {
 	   void *ctx = state;
 	   (yyval.expression) = new(ctx) ast_expression((yyvsp[(2) - (3)].n), (yyvsp[(1) - (3)].expression), (yyvsp[(3) - (3)].expression), NULL);
@@ -3367,96 +3382,96 @@ yyreduce:
 	;}
     break;
 
-  case 84:
+  case 85:
 
-/* Line 1464 of yacc.c  */
-#line 641 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 655 "glsl_parser.ypp"
     { (yyval.n) = ast_assign; ;}
     break;
 
-  case 85:
+  case 86:
 
-/* Line 1464 of yacc.c  */
-#line 642 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 656 "glsl_parser.ypp"
     { (yyval.n) = ast_mul_assign; ;}
     break;
 
-  case 86:
+  case 87:
 
-/* Line 1464 of yacc.c  */
-#line 643 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 657 "glsl_parser.ypp"
     { (yyval.n) = ast_div_assign; ;}
     break;
 
-  case 87:
+  case 88:
 
-/* Line 1464 of yacc.c  */
-#line 644 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 658 "glsl_parser.ypp"
     { (yyval.n) = ast_mod_assign; ;}
     break;
 
-  case 88:
+  case 89:
 
-/* Line 1464 of yacc.c  */
-#line 645 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 659 "glsl_parser.ypp"
     { (yyval.n) = ast_add_assign; ;}
     break;
 
-  case 89:
+  case 90:
 
-/* Line 1464 of yacc.c  */
-#line 646 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 660 "glsl_parser.ypp"
     { (yyval.n) = ast_sub_assign; ;}
     break;
 
-  case 90:
+  case 91:
 
-/* Line 1464 of yacc.c  */
-#line 647 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 661 "glsl_parser.ypp"
     { (yyval.n) = ast_ls_assign; ;}
     break;
 
-  case 91:
+  case 92:
 
-/* Line 1464 of yacc.c  */
-#line 648 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 662 "glsl_parser.ypp"
     { (yyval.n) = ast_rs_assign; ;}
     break;
 
-  case 92:
+  case 93:
 
-/* Line 1464 of yacc.c  */
-#line 649 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 663 "glsl_parser.ypp"
     { (yyval.n) = ast_and_assign; ;}
     break;
 
-  case 93:
+  case 94:
 
-/* Line 1464 of yacc.c  */
-#line 650 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 664 "glsl_parser.ypp"
     { (yyval.n) = ast_xor_assign; ;}
     break;
 
-  case 94:
+  case 95:
 
-/* Line 1464 of yacc.c  */
-#line 651 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 665 "glsl_parser.ypp"
     { (yyval.n) = ast_or_assign; ;}
     break;
 
-  case 95:
+  case 96:
 
-/* Line 1464 of yacc.c  */
-#line 656 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 670 "glsl_parser.ypp"
     {
 	   (yyval.expression) = (yyvsp[(1) - (1)].expression);
 	;}
     break;
 
-  case 96:
+  case 97:
 
-/* Line 1464 of yacc.c  */
-#line 660 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 674 "glsl_parser.ypp"
     {
 	   void *ctx = state;
 	   if ((yyvsp[(1) - (3)].expression)->oper != ast_sequence) {
@@ -3471,28 +3486,28 @@ yyreduce:
 	;}
     break;
 
-  case 98:
+  case 99:
 
-/* Line 1464 of yacc.c  */
-#line 680 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 694 "glsl_parser.ypp"
     {
 	   (yyval.node) = (yyvsp[(1) - (2)].function);
 	;}
     break;
 
-  case 99:
+  case 100:
 
-/* Line 1464 of yacc.c  */
-#line 684 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 698 "glsl_parser.ypp"
     {
 	   (yyval.node) = (yyvsp[(1) - (2)].declarator_list);
 	;}
     break;
 
-  case 100:
+  case 101:
 
-/* Line 1464 of yacc.c  */
-#line 688 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 702 "glsl_parser.ypp"
     {
 	   if (((yyvsp[(3) - (4)].type_specifier)->type_specifier != ast_float)
 	       && ((yyvsp[(3) - (4)].type_specifier)->type_specifier != ast_int)) {
@@ -3505,30 +3520,30 @@ yyreduce:
 	;}
     break;
 
-  case 104:
+  case 105:
 
-/* Line 1464 of yacc.c  */
-#line 711 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 725 "glsl_parser.ypp"
     {
 	   (yyval.function) = (yyvsp[(1) - (2)].function);
 	   (yyval.function)->parameters.push_tail(& (yyvsp[(2) - (2)].parameter_declarator)->link);
 	;}
     break;
 
-  case 105:
+  case 106:
 
-/* Line 1464 of yacc.c  */
-#line 716 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 730 "glsl_parser.ypp"
     {
 	   (yyval.function) = (yyvsp[(1) - (3)].function);
 	   (yyval.function)->parameters.push_tail(& (yyvsp[(3) - (3)].parameter_declarator)->link);
 	;}
     break;
 
-  case 106:
+  case 107:
 
-/* Line 1464 of yacc.c  */
-#line 724 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 738 "glsl_parser.ypp"
     {
 	   void *ctx = state;
 	   (yyval.function) = new(ctx) ast_function();
@@ -3538,10 +3553,10 @@ yyreduce:
 	;}
     break;
 
-  case 107:
+  case 108:
 
-/* Line 1464 of yacc.c  */
-#line 735 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 749 "glsl_parser.ypp"
     {
 	   void *ctx = state;
 	   (yyval.parameter_declarator) = new(ctx) ast_parameter_declarator();
@@ -3553,10 +3568,10 @@ yyreduce:
 	;}
     break;
 
-  case 108:
+  case 109:
 
-/* Line 1464 of yacc.c  */
-#line 745 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 759 "glsl_parser.ypp"
     {
 	   void *ctx = state;
 	   (yyval.parameter_declarator) = new(ctx) ast_parameter_declarator();
@@ -3570,10 +3585,10 @@ yyreduce:
 	;}
     break;
 
-  case 109:
+  case 110:
 
-/* Line 1464 of yacc.c  */
-#line 760 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 774 "glsl_parser.ypp"
     {
 	   (yyvsp[(1) - (3)].type_qualifier).flags.i |= (yyvsp[(2) - (3)].type_qualifier).flags.i;
 
@@ -3582,20 +3597,20 @@ yyreduce:
 	;}
     break;
 
-  case 110:
+  case 111:
 
-/* Line 1464 of yacc.c  */
-#line 767 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 781 "glsl_parser.ypp"
     {
 	   (yyval.parameter_declarator) = (yyvsp[(2) - (2)].parameter_declarator);
 	   (yyval.parameter_declarator)->type->qualifier = (yyvsp[(1) - (2)].type_qualifier);
 	;}
     break;
 
-  case 111:
+  case 112:
 
-/* Line 1464 of yacc.c  */
-#line 772 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 786 "glsl_parser.ypp"
     {
 	   void *ctx = state;
 	   (yyvsp[(1) - (3)].type_qualifier).flags.i |= (yyvsp[(2) - (3)].type_qualifier).flags.i;
@@ -3608,10 +3623,10 @@ yyreduce:
 	;}
     break;
 
-  case 112:
+  case 113:
 
-/* Line 1464 of yacc.c  */
-#line 783 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 797 "glsl_parser.ypp"
     {
 	   void *ctx = state;
 	   (yyval.parameter_declarator) = new(ctx) ast_parameter_declarator();
@@ -3622,39 +3637,39 @@ yyreduce:
 	;}
     break;
 
-  case 113:
+  case 114:
 
-/* Line 1464 of yacc.c  */
-#line 795 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 809 "glsl_parser.ypp"
     {
 	   memset(& (yyval.type_qualifier), 0, sizeof((yyval.type_qualifier)));
 	;}
     break;
 
-  case 114:
+  case 115:
 
-/* Line 1464 of yacc.c  */
-#line 799 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 813 "glsl_parser.ypp"
     {
 	   memset(& (yyval.type_qualifier), 0, sizeof((yyval.type_qualifier)));
 	   (yyval.type_qualifier).flags.q.in = 1;
 	;}
     break;
 
-  case 115:
+  case 116:
 
-/* Line 1464 of yacc.c  */
-#line 804 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 818 "glsl_parser.ypp"
     {
 	   memset(& (yyval.type_qualifier), 0, sizeof((yyval.type_qualifier)));
 	   (yyval.type_qualifier).flags.q.out = 1;
 	;}
     break;
 
-  case 116:
+  case 117:
 
-/* Line 1464 of yacc.c  */
-#line 809 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 823 "glsl_parser.ypp"
     {
 	   memset(& (yyval.type_qualifier), 0, sizeof((yyval.type_qualifier)));
 	   (yyval.type_qualifier).flags.q.in = 1;
@@ -3662,10 +3677,10 @@ yyreduce:
 	;}
     break;
 
-  case 119:
+  case 120:
 
-/* Line 1464 of yacc.c  */
-#line 823 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 837 "glsl_parser.ypp"
     {
 	   void *ctx = state;
 	   ast_declaration *decl = new(ctx) ast_declaration((yyvsp[(3) - (3)].identifier), false, NULL, NULL);
@@ -3676,10 +3691,10 @@ yyreduce:
 	;}
     break;
 
-  case 120:
+  case 121:
 
-/* Line 1464 of yacc.c  */
-#line 832 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 846 "glsl_parser.ypp"
     {
 	   void *ctx = state;
 	   ast_declaration *decl = new(ctx) ast_declaration((yyvsp[(3) - (5)].identifier), true, NULL, NULL);
@@ -3690,10 +3705,10 @@ yyreduce:
 	;}
     break;
 
-  case 121:
+  case 122:
 
-/* Line 1464 of yacc.c  */
-#line 841 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 855 "glsl_parser.ypp"
     {
 	   void *ctx = state;
 	   ast_declaration *decl = new(ctx) ast_declaration((yyvsp[(3) - (6)].identifier), true, (yyvsp[(5) - (6)].expression), NULL);
@@ -3704,10 +3719,10 @@ yyreduce:
 	;}
     break;
 
-  case 122:
+  case 123:
 
-/* Line 1464 of yacc.c  */
-#line 850 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 864 "glsl_parser.ypp"
     {
 	   void *ctx = state;
 	   ast_declaration *decl = new(ctx) ast_declaration((yyvsp[(3) - (7)].identifier), true, NULL, (yyvsp[(7) - (7)].expression));
@@ -3718,10 +3733,10 @@ yyreduce:
 	;}
     break;
 
-  case 123:
+  case 124:
 
-/* Line 1464 of yacc.c  */
-#line 859 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 873 "glsl_parser.ypp"
     {
 	   void *ctx = state;
 	   ast_declaration *decl = new(ctx) ast_declaration((yyvsp[(3) - (8)].identifier), true, (yyvsp[(5) - (8)].expression), (yyvsp[(8) - (8)].expression));
@@ -3732,10 +3747,10 @@ yyreduce:
 	;}
     break;
 
-  case 124:
+  case 125:
 
-/* Line 1464 of yacc.c  */
-#line 868 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 882 "glsl_parser.ypp"
     {
 	   void *ctx = state;
 	   ast_declaration *decl = new(ctx) ast_declaration((yyvsp[(3) - (5)].identifier), false, NULL, (yyvsp[(5) - (5)].expression));
@@ -3746,10 +3761,10 @@ yyreduce:
 	;}
     break;
 
-  case 125:
+  case 126:
 
-/* Line 1464 of yacc.c  */
-#line 881 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 895 "glsl_parser.ypp"
     {
 	   void *ctx = state;
 	   if ((yyvsp[(1) - (1)].fully_specified_type)->specifier->type_specifier != ast_struct) {
@@ -3762,10 +3777,10 @@ yyreduce:
 	;}
     break;
 
-  case 126:
+  case 127:
 
-/* Line 1464 of yacc.c  */
-#line 892 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 906 "glsl_parser.ypp"
     {
 	   void *ctx = state;
 	   ast_declaration *decl = new(ctx) ast_declaration((yyvsp[(2) - (2)].identifier), false, NULL, NULL);
@@ -3776,10 +3791,10 @@ yyreduce:
 	;}
     break;
 
-  case 127:
+  case 128:
 
-/* Line 1464 of yacc.c  */
-#line 901 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 915 "glsl_parser.ypp"
     {
 	   void *ctx = state;
 	   ast_declaration *decl = new(ctx) ast_declaration((yyvsp[(2) - (4)].identifier), true, NULL, NULL);
@@ -3790,10 +3805,10 @@ yyreduce:
 	;}
     break;
 
-  case 128:
+  case 129:
 
-/* Line 1464 of yacc.c  */
-#line 910 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 924 "glsl_parser.ypp"
     {
 	   void *ctx = state;
 	   ast_declaration *decl = new(ctx) ast_declaration((yyvsp[(2) - (5)].identifier), true, (yyvsp[(4) - (5)].expression), NULL);
@@ -3804,10 +3819,10 @@ yyreduce:
 	;}
     break;
 
-  case 129:
+  case 130:
 
-/* Line 1464 of yacc.c  */
-#line 919 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 933 "glsl_parser.ypp"
     {
 	   void *ctx = state;
 	   ast_declaration *decl = new(ctx) ast_declaration((yyvsp[(2) - (6)].identifier), true, NULL, (yyvsp[(6) - (6)].expression));
@@ -3818,10 +3833,10 @@ yyreduce:
 	;}
     break;
 
-  case 130:
+  case 131:
 
-/* Line 1464 of yacc.c  */
-#line 928 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 942 "glsl_parser.ypp"
     {
 	   void *ctx = state;
 	   ast_declaration *decl = new(ctx) ast_declaration((yyvsp[(2) - (7)].identifier), true, (yyvsp[(4) - (7)].expression), (yyvsp[(7) - (7)].expression));
@@ -3832,10 +3847,10 @@ yyreduce:
 	;}
     break;
 
-  case 131:
+  case 132:
 
-/* Line 1464 of yacc.c  */
-#line 937 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 951 "glsl_parser.ypp"
     {
 	   void *ctx = state;
 	   ast_declaration *decl = new(ctx) ast_declaration((yyvsp[(2) - (4)].identifier), false, NULL, (yyvsp[(4) - (4)].expression));
@@ -3846,10 +3861,10 @@ yyreduce:
 	;}
     break;
 
-  case 132:
+  case 133:
 
-/* Line 1464 of yacc.c  */
-#line 946 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 960 "glsl_parser.ypp"
     {
 	   void *ctx = state;
 	   ast_declaration *decl = new(ctx) ast_declaration((yyvsp[(2) - (2)].identifier), false, NULL, NULL);
@@ -3862,10 +3877,10 @@ yyreduce:
 	;}
     break;
 
-  case 133:
+  case 134:
 
-/* Line 1464 of yacc.c  */
-#line 960 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 974 "glsl_parser.ypp"
     {
 	   void *ctx = state;
 	   (yyval.fully_specified_type) = new(ctx) ast_fully_specified_type();
@@ -3874,10 +3889,10 @@ yyreduce:
 	;}
     break;
 
-  case 134:
+  case 135:
 
-/* Line 1464 of yacc.c  */
-#line 967 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 981 "glsl_parser.ypp"
     {
 	   void *ctx = state;
 	   (yyval.fully_specified_type) = new(ctx) ast_fully_specified_type();
@@ -3887,19 +3902,19 @@ yyreduce:
 	;}
     break;
 
-  case 135:
+  case 136:
 
-/* Line 1464 of yacc.c  */
-#line 978 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 992 "glsl_parser.ypp"
     {
 	  (yyval.type_qualifier) = (yyvsp[(3) - (4)].type_qualifier);
 	;}
     break;
 
-  case 137:
+  case 138:
 
-/* Line 1464 of yacc.c  */
-#line 986 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1000 "glsl_parser.ypp"
     {
 	   if (((yyvsp[(1) - (3)].type_qualifier).flags.i & (yyvsp[(3) - (3)].type_qualifier).flags.i) != 0) {
 	      _mesa_glsl_error(& (yylsp[(3) - (3)]), state,
@@ -3917,10 +3932,10 @@ yyreduce:
 	;}
     break;
 
-  case 138:
+  case 139:
 
-/* Line 1464 of yacc.c  */
-#line 1005 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1019 "glsl_parser.ypp"
     {
 	   bool got_one = false;
 
@@ -3951,10 +3966,10 @@ yyreduce:
 	;}
     break;
 
-  case 139:
+  case 140:
 
-/* Line 1464 of yacc.c  */
-#line 1034 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1048 "glsl_parser.ypp"
     {
 	   bool got_one = false;
 
@@ -3994,80 +4009,80 @@ yyreduce:
 	;}
     break;
 
-  case 140:
+  case 141:
 
-/* Line 1464 of yacc.c  */
-#line 1075 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1089 "glsl_parser.ypp"
     {
 	   memset(& (yyval.type_qualifier), 0, sizeof((yyval.type_qualifier)));
 	   (yyval.type_qualifier).flags.q.smooth = 1;
 	;}
     break;
 
-  case 141:
+  case 142:
 
-/* Line 1464 of yacc.c  */
-#line 1080 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1094 "glsl_parser.ypp"
     {
 	   memset(& (yyval.type_qualifier), 0, sizeof((yyval.type_qualifier)));
 	   (yyval.type_qualifier).flags.q.flat = 1;
 	;}
     break;
 
-  case 142:
+  case 143:
 
-/* Line 1464 of yacc.c  */
-#line 1085 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1099 "glsl_parser.ypp"
     {
 	   memset(& (yyval.type_qualifier), 0, sizeof((yyval.type_qualifier)));
 	   (yyval.type_qualifier).flags.q.noperspective = 1;
 	;}
     break;
 
-  case 143:
+  case 144:
 
-/* Line 1464 of yacc.c  */
-#line 1093 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1107 "glsl_parser.ypp"
     {
 	   memset(& (yyval.type_qualifier), 0, sizeof((yyval.type_qualifier)));
 	   (yyval.type_qualifier).flags.q.constant = 1;
 	;}
     break;
 
-  case 146:
+  case 147:
 
-/* Line 1464 of yacc.c  */
-#line 1103 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1117 "glsl_parser.ypp"
     {
 	   (yyval.type_qualifier) = (yyvsp[(1) - (2)].type_qualifier);
 	   (yyval.type_qualifier).flags.i |= (yyvsp[(2) - (2)].type_qualifier).flags.i;
 	;}
     break;
 
-  case 148:
+  case 149:
 
-/* Line 1464 of yacc.c  */
-#line 1109 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1123 "glsl_parser.ypp"
     {
 	   (yyval.type_qualifier) = (yyvsp[(1) - (2)].type_qualifier);
 	   (yyval.type_qualifier).flags.i |= (yyvsp[(2) - (2)].type_qualifier).flags.i;
 	;}
     break;
 
-  case 149:
+  case 150:
 
-/* Line 1464 of yacc.c  */
-#line 1114 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1128 "glsl_parser.ypp"
     {
 	   (yyval.type_qualifier) = (yyvsp[(2) - (2)].type_qualifier);
 	   (yyval.type_qualifier).flags.q.invariant = 1;
 	;}
     break;
 
-  case 150:
+  case 151:
 
-/* Line 1464 of yacc.c  */
-#line 1119 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1133 "glsl_parser.ypp"
     {
 	   (yyval.type_qualifier) = (yyvsp[(2) - (3)].type_qualifier);
 	   (yyval.type_qualifier).flags.i |= (yyvsp[(3) - (3)].type_qualifier).flags.i;
@@ -4075,50 +4090,50 @@ yyreduce:
 	;}
     break;
 
-  case 151:
+  case 152:
 
-/* Line 1464 of yacc.c  */
-#line 1125 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1139 "glsl_parser.ypp"
     {
 	   memset(& (yyval.type_qualifier), 0, sizeof((yyval.type_qualifier)));
 	   (yyval.type_qualifier).flags.q.invariant = 1;
 	;}
     break;
 
-  case 152:
+  case 153:
 
-/* Line 1464 of yacc.c  */
-#line 1133 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1147 "glsl_parser.ypp"
     {
 	   memset(& (yyval.type_qualifier), 0, sizeof((yyval.type_qualifier)));
 	   (yyval.type_qualifier).flags.q.constant = 1;
 	;}
     break;
 
-  case 153:
+  case 154:
 
-/* Line 1464 of yacc.c  */
-#line 1138 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1152 "glsl_parser.ypp"
     {
 	   memset(& (yyval.type_qualifier), 0, sizeof((yyval.type_qualifier)));
 	   (yyval.type_qualifier).flags.q.attribute = 1;
 	;}
     break;
 
-  case 154:
+  case 155:
 
-/* Line 1464 of yacc.c  */
-#line 1143 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1157 "glsl_parser.ypp"
     {
 	   memset(& (yyval.type_qualifier), 0, sizeof((yyval.type_qualifier)));
 	   (yyval.type_qualifier).flags.q.varying = 1;
 	;}
     break;
 
-  case 155:
+  case 156:
 
-/* Line 1464 of yacc.c  */
-#line 1148 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1162 "glsl_parser.ypp"
     {
 	   memset(& (yyval.type_qualifier), 0, sizeof((yyval.type_qualifier)));
 	   (yyval.type_qualifier).flags.q.centroid = 1;
@@ -4126,70 +4141,70 @@ yyreduce:
 	;}
     break;
 
-  case 156:
+  case 157:
 
-/* Line 1464 of yacc.c  */
-#line 1154 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1168 "glsl_parser.ypp"
     {
 	   memset(& (yyval.type_qualifier), 0, sizeof((yyval.type_qualifier)));
 	   (yyval.type_qualifier).flags.q.in = 1;
 	;}
     break;
 
-  case 157:
+  case 158:
 
-/* Line 1464 of yacc.c  */
-#line 1159 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1173 "glsl_parser.ypp"
     {
 	   memset(& (yyval.type_qualifier), 0, sizeof((yyval.type_qualifier)));
 	   (yyval.type_qualifier).flags.q.out = 1;
 	;}
     break;
 
-  case 158:
+  case 159:
 
-/* Line 1464 of yacc.c  */
-#line 1164 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1178 "glsl_parser.ypp"
     {
 	   memset(& (yyval.type_qualifier), 0, sizeof((yyval.type_qualifier)));
 	   (yyval.type_qualifier).flags.q.centroid = 1; (yyval.type_qualifier).flags.q.in = 1;
 	;}
     break;
 
-  case 159:
+  case 160:
 
-/* Line 1464 of yacc.c  */
-#line 1169 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1183 "glsl_parser.ypp"
     {
 	   memset(& (yyval.type_qualifier), 0, sizeof((yyval.type_qualifier)));
 	   (yyval.type_qualifier).flags.q.centroid = 1; (yyval.type_qualifier).flags.q.out = 1;
 	;}
     break;
 
-  case 160:
+  case 161:
 
-/* Line 1464 of yacc.c  */
-#line 1174 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1188 "glsl_parser.ypp"
     {
 	   memset(& (yyval.type_qualifier), 0, sizeof((yyval.type_qualifier)));
 	   (yyval.type_qualifier).flags.q.uniform = 1;
 	;}
     break;
 
-  case 162:
+  case 163:
 
-/* Line 1464 of yacc.c  */
-#line 1183 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1197 "glsl_parser.ypp"
     {
 	   (yyval.type_specifier) = (yyvsp[(2) - (2)].type_specifier);
 	   (yyval.type_specifier)->precision = (yyvsp[(1) - (2)].n);
 	;}
     break;
 
-  case 164:
+  case 165:
 
-/* Line 1464 of yacc.c  */
-#line 1192 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1206 "glsl_parser.ypp"
     {
 	   (yyval.type_specifier) = (yyvsp[(1) - (3)].type_specifier);
 	   (yyval.type_specifier)->is_array = true;
@@ -4197,10 +4212,10 @@ yyreduce:
 	;}
     break;
 
-  case 165:
+  case 166:
 
-/* Line 1464 of yacc.c  */
-#line 1198 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1212 "glsl_parser.ypp"
     {
 	   (yyval.type_specifier) = (yyvsp[(1) - (4)].type_specifier);
 	   (yyval.type_specifier)->is_array = true;
@@ -4208,10 +4223,10 @@ yyreduce:
 	;}
     break;
 
-  case 166:
+  case 167:
 
-/* Line 1464 of yacc.c  */
-#line 1207 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1221 "glsl_parser.ypp"
     {
 	   void *ctx = state;
 	   (yyval.type_specifier) = new(ctx) ast_type_specifier((yyvsp[(1) - (1)].n));
@@ -4219,10 +4234,10 @@ yyreduce:
 	;}
     break;
 
-  case 167:
+  case 168:
 
-/* Line 1464 of yacc.c  */
-#line 1213 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1227 "glsl_parser.ypp"
     {
 	   void *ctx = state;
 	   (yyval.type_specifier) = new(ctx) ast_type_specifier((yyvsp[(1) - (1)].struct_specifier));
@@ -4230,10 +4245,10 @@ yyreduce:
 	;}
     break;
 
-  case 168:
+  case 169:
 
-/* Line 1464 of yacc.c  */
-#line 1219 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1233 "glsl_parser.ypp"
     {
 	   void *ctx = state;
 	   (yyval.type_specifier) = new(ctx) ast_type_specifier((yyvsp[(1) - (1)].identifier));
@@ -4241,367 +4256,367 @@ yyreduce:
 	;}
     break;
 
-  case 169:
+  case 170:
 
-/* Line 1464 of yacc.c  */
-#line 1227 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1241 "glsl_parser.ypp"
     { (yyval.n) = ast_void; ;}
     break;
 
-  case 170:
+  case 171:
 
-/* Line 1464 of yacc.c  */
-#line 1228 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1242 "glsl_parser.ypp"
     { (yyval.n) = ast_float; ;}
     break;
 
-  case 171:
+  case 172:
 
-/* Line 1464 of yacc.c  */
-#line 1229 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1243 "glsl_parser.ypp"
     { (yyval.n) = ast_int; ;}
     break;
 
-  case 172:
+  case 173:
 
-/* Line 1464 of yacc.c  */
-#line 1230 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1244 "glsl_parser.ypp"
     { (yyval.n) = ast_uint; ;}
     break;
 
-  case 173:
+  case 174:
 
-/* Line 1464 of yacc.c  */
-#line 1231 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1245 "glsl_parser.ypp"
     { (yyval.n) = ast_bool; ;}
     break;
 
-  case 174:
+  case 175:
 
-/* Line 1464 of yacc.c  */
-#line 1232 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1246 "glsl_parser.ypp"
     { (yyval.n) = ast_vec2; ;}
     break;
 
-  case 175:
+  case 176:
 
-/* Line 1464 of yacc.c  */
-#line 1233 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1247 "glsl_parser.ypp"
     { (yyval.n) = ast_vec3; ;}
     break;
 
-  case 176:
+  case 177:
 
-/* Line 1464 of yacc.c  */
-#line 1234 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1248 "glsl_parser.ypp"
     { (yyval.n) = ast_vec4; ;}
     break;
 
-  case 177:
+  case 178:
 
-/* Line 1464 of yacc.c  */
-#line 1235 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1249 "glsl_parser.ypp"
     { (yyval.n) = ast_bvec2; ;}
     break;
 
-  case 178:
+  case 179:
 
-/* Line 1464 of yacc.c  */
-#line 1236 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1250 "glsl_parser.ypp"
     { (yyval.n) = ast_bvec3; ;}
     break;
 
-  case 179:
+  case 180:
 
-/* Line 1464 of yacc.c  */
-#line 1237 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1251 "glsl_parser.ypp"
     { (yyval.n) = ast_bvec4; ;}
     break;
 
-  case 180:
+  case 181:
 
-/* Line 1464 of yacc.c  */
-#line 1238 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1252 "glsl_parser.ypp"
     { (yyval.n) = ast_ivec2; ;}
     break;
 
-  case 181:
+  case 182:
 
-/* Line 1464 of yacc.c  */
-#line 1239 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1253 "glsl_parser.ypp"
     { (yyval.n) = ast_ivec3; ;}
     break;
 
-  case 182:
+  case 183:
 
-/* Line 1464 of yacc.c  */
-#line 1240 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1254 "glsl_parser.ypp"
     { (yyval.n) = ast_ivec4; ;}
     break;
 
-  case 183:
+  case 184:
 
-/* Line 1464 of yacc.c  */
-#line 1241 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1255 "glsl_parser.ypp"
     { (yyval.n) = ast_uvec2; ;}
     break;
 
-  case 184:
+  case 185:
 
-/* Line 1464 of yacc.c  */
-#line 1242 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1256 "glsl_parser.ypp"
     { (yyval.n) = ast_uvec3; ;}
     break;
 
-  case 185:
+  case 186:
 
-/* Line 1464 of yacc.c  */
-#line 1243 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1257 "glsl_parser.ypp"
     { (yyval.n) = ast_uvec4; ;}
     break;
 
-  case 186:
+  case 187:
 
-/* Line 1464 of yacc.c  */
-#line 1244 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1258 "glsl_parser.ypp"
     { (yyval.n) = ast_mat2; ;}
     break;
 
-  case 187:
+  case 188:
 
-/* Line 1464 of yacc.c  */
-#line 1245 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1259 "glsl_parser.ypp"
     { (yyval.n) = ast_mat2x3; ;}
     break;
 
-  case 188:
+  case 189:
 
-/* Line 1464 of yacc.c  */
-#line 1246 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1260 "glsl_parser.ypp"
     { (yyval.n) = ast_mat2x4; ;}
     break;
 
-  case 189:
+  case 190:
 
-/* Line 1464 of yacc.c  */
-#line 1247 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1261 "glsl_parser.ypp"
     { (yyval.n) = ast_mat3x2; ;}
     break;
 
-  case 190:
+  case 191:
 
-/* Line 1464 of yacc.c  */
-#line 1248 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1262 "glsl_parser.ypp"
     { (yyval.n) = ast_mat3; ;}
     break;
 
-  case 191:
+  case 192:
 
-/* Line 1464 of yacc.c  */
-#line 1249 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1263 "glsl_parser.ypp"
     { (yyval.n) = ast_mat3x4; ;}
     break;
 
-  case 192:
+  case 193:
 
-/* Line 1464 of yacc.c  */
-#line 1250 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1264 "glsl_parser.ypp"
     { (yyval.n) = ast_mat4x2; ;}
     break;
 
-  case 193:
+  case 194:
 
-/* Line 1464 of yacc.c  */
-#line 1251 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1265 "glsl_parser.ypp"
     { (yyval.n) = ast_mat4x3; ;}
     break;
 
-  case 194:
+  case 195:
 
-/* Line 1464 of yacc.c  */
-#line 1252 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1266 "glsl_parser.ypp"
     { (yyval.n) = ast_mat4; ;}
     break;
 
-  case 195:
+  case 196:
 
-/* Line 1464 of yacc.c  */
-#line 1253 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1267 "glsl_parser.ypp"
     { (yyval.n) = ast_sampler1d; ;}
     break;
 
-  case 196:
+  case 197:
 
-/* Line 1464 of yacc.c  */
-#line 1254 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1268 "glsl_parser.ypp"
     { (yyval.n) = ast_sampler2d; ;}
     break;
 
-  case 197:
+  case 198:
 
-/* Line 1464 of yacc.c  */
-#line 1255 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1269 "glsl_parser.ypp"
     { (yyval.n) = ast_sampler2drect; ;}
     break;
 
-  case 198:
+  case 199:
 
-/* Line 1464 of yacc.c  */
-#line 1256 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1270 "glsl_parser.ypp"
     { (yyval.n) = ast_sampler3d; ;}
     break;
 
-  case 199:
+  case 200:
 
-/* Line 1464 of yacc.c  */
-#line 1257 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1271 "glsl_parser.ypp"
     { (yyval.n) = ast_samplercube; ;}
     break;
 
-  case 200:
+  case 201:
 
-/* Line 1464 of yacc.c  */
-#line 1258 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1272 "glsl_parser.ypp"
     { (yyval.n) = ast_sampler1dshadow; ;}
     break;
 
-  case 201:
+  case 202:
 
-/* Line 1464 of yacc.c  */
-#line 1259 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1273 "glsl_parser.ypp"
     { (yyval.n) = ast_sampler2dshadow; ;}
     break;
 
-  case 202:
+  case 203:
 
-/* Line 1464 of yacc.c  */
-#line 1260 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1274 "glsl_parser.ypp"
     { (yyval.n) = ast_sampler2drectshadow; ;}
     break;
 
-  case 203:
+  case 204:
 
-/* Line 1464 of yacc.c  */
-#line 1261 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1275 "glsl_parser.ypp"
     { (yyval.n) = ast_samplercubeshadow; ;}
     break;
 
-  case 204:
+  case 205:
 
-/* Line 1464 of yacc.c  */
-#line 1262 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1276 "glsl_parser.ypp"
     { (yyval.n) = ast_sampler1darray; ;}
     break;
 
-  case 205:
+  case 206:
 
-/* Line 1464 of yacc.c  */
-#line 1263 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1277 "glsl_parser.ypp"
     { (yyval.n) = ast_sampler2darray; ;}
     break;
 
-  case 206:
+  case 207:
 
-/* Line 1464 of yacc.c  */
-#line 1264 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1278 "glsl_parser.ypp"
     { (yyval.n) = ast_sampler1darrayshadow; ;}
     break;
 
-  case 207:
+  case 208:
 
-/* Line 1464 of yacc.c  */
-#line 1265 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1279 "glsl_parser.ypp"
     { (yyval.n) = ast_sampler2darrayshadow; ;}
     break;
 
-  case 208:
+  case 209:
 
-/* Line 1464 of yacc.c  */
-#line 1266 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1280 "glsl_parser.ypp"
     { (yyval.n) = ast_isampler1d; ;}
     break;
 
-  case 209:
+  case 210:
 
-/* Line 1464 of yacc.c  */
-#line 1267 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1281 "glsl_parser.ypp"
     { (yyval.n) = ast_isampler2d; ;}
     break;
 
-  case 210:
+  case 211:
 
-/* Line 1464 of yacc.c  */
-#line 1268 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1282 "glsl_parser.ypp"
     { (yyval.n) = ast_isampler3d; ;}
     break;
 
-  case 211:
+  case 212:
 
-/* Line 1464 of yacc.c  */
-#line 1269 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1283 "glsl_parser.ypp"
     { (yyval.n) = ast_isamplercube; ;}
     break;
 
-  case 212:
+  case 213:
 
-/* Line 1464 of yacc.c  */
-#line 1270 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1284 "glsl_parser.ypp"
     { (yyval.n) = ast_isampler1darray; ;}
     break;
 
-  case 213:
+  case 214:
 
-/* Line 1464 of yacc.c  */
-#line 1271 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1285 "glsl_parser.ypp"
     { (yyval.n) = ast_isampler2darray; ;}
     break;
 
-  case 214:
+  case 215:
 
-/* Line 1464 of yacc.c  */
-#line 1272 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1286 "glsl_parser.ypp"
     { (yyval.n) = ast_usampler1d; ;}
     break;
 
-  case 215:
+  case 216:
 
-/* Line 1464 of yacc.c  */
-#line 1273 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1287 "glsl_parser.ypp"
     { (yyval.n) = ast_usampler2d; ;}
     break;
 
-  case 216:
+  case 217:
 
-/* Line 1464 of yacc.c  */
-#line 1274 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1288 "glsl_parser.ypp"
     { (yyval.n) = ast_usampler3d; ;}
     break;
 
-  case 217:
+  case 218:
 
-/* Line 1464 of yacc.c  */
-#line 1275 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1289 "glsl_parser.ypp"
     { (yyval.n) = ast_usamplercube; ;}
     break;
 
-  case 218:
+  case 219:
 
-/* Line 1464 of yacc.c  */
-#line 1276 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1290 "glsl_parser.ypp"
     { (yyval.n) = ast_usampler1darray; ;}
     break;
 
-  case 219:
+  case 220:
 
-/* Line 1464 of yacc.c  */
-#line 1277 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1291 "glsl_parser.ypp"
     { (yyval.n) = ast_usampler2darray; ;}
     break;
 
-  case 220:
+  case 221:
 
-/* Line 1464 of yacc.c  */
-#line 1281 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1295 "glsl_parser.ypp"
     {
 		     if (!state->es_shader && state->language_version < 130)
 			_mesa_glsl_error(& (yylsp[(1) - (1)]), state,
@@ -4615,10 +4630,10 @@ yyreduce:
 		  ;}
     break;
 
-  case 221:
+  case 222:
 
-/* Line 1464 of yacc.c  */
-#line 1292 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1306 "glsl_parser.ypp"
     {
 		     if (!state->es_shader && state->language_version < 130)
 			_mesa_glsl_error(& (yylsp[(1) - (1)]), state,
@@ -4632,10 +4647,10 @@ yyreduce:
 		  ;}
     break;
 
-  case 222:
+  case 223:
 
-/* Line 1464 of yacc.c  */
-#line 1303 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1317 "glsl_parser.ypp"
     {
 		     if (!state->es_shader && state->language_version < 130)
 			_mesa_glsl_error(& (yylsp[(1) - (1)]), state,
@@ -4649,10 +4664,10 @@ yyreduce:
 		  ;}
     break;
 
-  case 223:
+  case 224:
 
-/* Line 1464 of yacc.c  */
-#line 1318 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1332 "glsl_parser.ypp"
     {
 	   void *ctx = state;
 	   (yyval.struct_specifier) = new(ctx) ast_struct_specifier((yyvsp[(2) - (5)].identifier), (yyvsp[(4) - (5)].node));
@@ -4660,10 +4675,10 @@ yyreduce:
 	;}
     break;
 
-  case 224:
+  case 225:
 
-/* Line 1464 of yacc.c  */
-#line 1324 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1338 "glsl_parser.ypp"
     {
 	   void *ctx = state;
 	   (yyval.struct_specifier) = new(ctx) ast_struct_specifier(NULL, (yyvsp[(3) - (4)].node));
@@ -4671,30 +4686,30 @@ yyreduce:
 	;}
     break;
 
-  case 225:
+  case 226:
 
-/* Line 1464 of yacc.c  */
-#line 1333 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1347 "glsl_parser.ypp"
     {
 	   (yyval.node) = (ast_node *) (yyvsp[(1) - (1)].declarator_list);
 	   (yyvsp[(1) - (1)].declarator_list)->link.self_link();
 	;}
     break;
 
-  case 226:
+  case 227:
 
-/* Line 1464 of yacc.c  */
-#line 1338 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1352 "glsl_parser.ypp"
     {
 	   (yyval.node) = (ast_node *) (yyvsp[(1) - (2)].node);
 	   (yyval.node)->link.insert_before(& (yyvsp[(2) - (2)].declarator_list)->link);
 	;}
     break;
 
-  case 227:
+  case 228:
 
-/* Line 1464 of yacc.c  */
-#line 1346 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1360 "glsl_parser.ypp"
     {
 	   void *ctx = state;
 	   ast_fully_specified_type *type = new(ctx) ast_fully_specified_type();
@@ -4708,30 +4723,30 @@ yyreduce:
 	;}
     break;
 
-  case 228:
+  case 229:
 
-/* Line 1464 of yacc.c  */
-#line 1361 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1375 "glsl_parser.ypp"
     {
 	   (yyval.declaration) = (yyvsp[(1) - (1)].declaration);
 	   (yyvsp[(1) - (1)].declaration)->link.self_link();
 	;}
     break;
 
-  case 229:
+  case 230:
 
-/* Line 1464 of yacc.c  */
-#line 1366 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1380 "glsl_parser.ypp"
     {
 	   (yyval.declaration) = (yyvsp[(1) - (3)].declaration);
 	   (yyval.declaration)->link.insert_before(& (yyvsp[(3) - (3)].declaration)->link);
 	;}
     break;
 
-  case 230:
+  case 231:
 
-/* Line 1464 of yacc.c  */
-#line 1374 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1388 "glsl_parser.ypp"
     {
 	   void *ctx = state;
 	   (yyval.declaration) = new(ctx) ast_declaration((yyvsp[(1) - (1)].identifier), false, NULL, NULL);
@@ -4739,10 +4754,10 @@ yyreduce:
 	;}
     break;
 
-  case 231:
+  case 232:
 
-/* Line 1464 of yacc.c  */
-#line 1380 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1394 "glsl_parser.ypp"
     {
 	   void *ctx = state;
 	   (yyval.declaration) = new(ctx) ast_declaration((yyvsp[(1) - (4)].identifier), true, (yyvsp[(3) - (4)].expression), NULL);
@@ -4750,31 +4765,31 @@ yyreduce:
 	;}
     break;
 
-  case 234:
+  case 235:
 
-/* Line 1464 of yacc.c  */
-#line 1398 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1412 "glsl_parser.ypp"
     { (yyval.node) = (ast_node *) (yyvsp[(1) - (1)].compound_statement); ;}
     break;
 
-  case 239:
+  case 240:
 
-/* Line 1464 of yacc.c  */
-#line 1406 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1420 "glsl_parser.ypp"
     { (yyval.node) = NULL; ;}
     break;
 
-  case 240:
+  case 241:
 
-/* Line 1464 of yacc.c  */
-#line 1407 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1421 "glsl_parser.ypp"
     { (yyval.node) = NULL; ;}
     break;
 
-  case 243:
+  case 244:
 
-/* Line 1464 of yacc.c  */
-#line 1414 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1428 "glsl_parser.ypp"
     {
 	   void *ctx = state;
 	   (yyval.compound_statement) = new(ctx) ast_compound_statement(true, NULL);
@@ -4782,10 +4797,10 @@ yyreduce:
 	;}
     break;
 
-  case 244:
+  case 245:
 
-/* Line 1464 of yacc.c  */
-#line 1420 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1434 "glsl_parser.ypp"
     {
 	   void *ctx = state;
 	   (yyval.compound_statement) = new(ctx) ast_compound_statement(true, (yyvsp[(2) - (3)].node));
@@ -4793,17 +4808,17 @@ yyreduce:
 	;}
     break;
 
-  case 245:
+  case 246:
 
-/* Line 1464 of yacc.c  */
-#line 1428 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1442 "glsl_parser.ypp"
     { (yyval.node) = (ast_node *) (yyvsp[(1) - (1)].compound_statement); ;}
     break;
 
-  case 247:
+  case 248:
 
-/* Line 1464 of yacc.c  */
-#line 1434 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1448 "glsl_parser.ypp"
     {
 	   void *ctx = state;
 	   (yyval.compound_statement) = new(ctx) ast_compound_statement(false, NULL);
@@ -4811,10 +4826,10 @@ yyreduce:
 	;}
     break;
 
-  case 248:
+  case 249:
 
-/* Line 1464 of yacc.c  */
-#line 1440 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1454 "glsl_parser.ypp"
     {
 	   void *ctx = state;
 	   (yyval.compound_statement) = new(ctx) ast_compound_statement(false, (yyvsp[(2) - (3)].node));
@@ -4822,10 +4837,10 @@ yyreduce:
 	;}
     break;
 
-  case 249:
+  case 250:
 
-/* Line 1464 of yacc.c  */
-#line 1449 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1463 "glsl_parser.ypp"
     {
 	   if ((yyvsp[(1) - (1)].node) == NULL) {
 	      _mesa_glsl_error(& (yylsp[(1) - (1)]), state, "<nil> statement\n");
@@ -4837,10 +4852,10 @@ yyreduce:
 	;}
     break;
 
-  case 250:
+  case 251:
 
-/* Line 1464 of yacc.c  */
-#line 1459 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1473 "glsl_parser.ypp"
     {
 	   if ((yyvsp[(2) - (2)].node) == NULL) {
 	      _mesa_glsl_error(& (yylsp[(2) - (2)]), state, "<nil> statement\n");
@@ -4851,10 +4866,10 @@ yyreduce:
 	;}
     break;
 
-  case 251:
+  case 252:
 
-/* Line 1464 of yacc.c  */
-#line 1471 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1485 "glsl_parser.ypp"
     {
 	   void *ctx = state;
 	   (yyval.node) = new(ctx) ast_expression_statement(NULL);
@@ -4862,10 +4877,10 @@ yyreduce:
 	;}
     break;
 
-  case 252:
+  case 253:
 
-/* Line 1464 of yacc.c  */
-#line 1477 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1491 "glsl_parser.ypp"
     {
 	   void *ctx = state;
 	   (yyval.node) = new(ctx) ast_expression_statement((yyvsp[(1) - (2)].expression));
@@ -4873,10 +4888,10 @@ yyreduce:
 	;}
     break;
 
-  case 253:
+  case 254:
 
-/* Line 1464 of yacc.c  */
-#line 1486 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1500 "glsl_parser.ypp"
     {
 	   (yyval.node) = new(state) ast_selection_statement((yyvsp[(3) - (5)].expression), (yyvsp[(5) - (5)].selection_rest_statement).then_statement,
 						   (yyvsp[(5) - (5)].selection_rest_statement).else_statement);
@@ -4884,39 +4899,39 @@ yyreduce:
 	;}
     break;
 
-  case 254:
+  case 255:
 
-/* Line 1464 of yacc.c  */
-#line 1495 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1509 "glsl_parser.ypp"
     {
 	   (yyval.selection_rest_statement).then_statement = (yyvsp[(1) - (3)].node);
 	   (yyval.selection_rest_statement).else_statement = (yyvsp[(3) - (3)].node);
 	;}
     break;
 
-  case 255:
+  case 256:
 
-/* Line 1464 of yacc.c  */
-#line 1500 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1514 "glsl_parser.ypp"
     {
 	   (yyval.selection_rest_statement).then_statement = (yyvsp[(1) - (1)].node);
 	   (yyval.selection_rest_statement).else_statement = NULL;
 	;}
     break;
 
-  case 256:
+  case 257:
 
-/* Line 1464 of yacc.c  */
-#line 1508 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1522 "glsl_parser.ypp"
     {
 	   (yyval.node) = (ast_node *) (yyvsp[(1) - (1)].expression);
 	;}
     break;
 
-  case 257:
+  case 258:
 
-/* Line 1464 of yacc.c  */
-#line 1512 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1526 "glsl_parser.ypp"
     {
 	   void *ctx = state;
 	   ast_declaration *decl = new(ctx) ast_declaration((yyvsp[(2) - (4)].identifier), false, NULL, (yyvsp[(4) - (4)].expression));
@@ -4929,10 +4944,10 @@ yyreduce:
 	;}
     break;
 
-  case 261:
+  case 262:
 
-/* Line 1464 of yacc.c  */
-#line 1535 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1549 "glsl_parser.ypp"
     {
 	   void *ctx = state;
 	   (yyval.node) = new(ctx) ast_iteration_statement(ast_iteration_statement::ast_while,
@@ -4941,10 +4956,10 @@ yyreduce:
 	;}
     break;
 
-  case 262:
+  case 263:
 
-/* Line 1464 of yacc.c  */
-#line 1542 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1556 "glsl_parser.ypp"
     {
 	   void *ctx = state;
 	   (yyval.node) = new(ctx) ast_iteration_statement(ast_iteration_statement::ast_do_while,
@@ -4953,10 +4968,10 @@ yyreduce:
 	;}
     break;
 
-  case 263:
+  case 264:
 
-/* Line 1464 of yacc.c  */
-#line 1549 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1563 "glsl_parser.ypp"
     {
 	   void *ctx = state;
 	   (yyval.node) = new(ctx) ast_iteration_statement(ast_iteration_statement::ast_for,
@@ -4965,39 +4980,39 @@ yyreduce:
 	;}
     break;
 
-  case 267:
+  case 268:
 
-/* Line 1464 of yacc.c  */
-#line 1565 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1579 "glsl_parser.ypp"
     {
 	   (yyval.node) = NULL;
 	;}
     break;
 
-  case 268:
+  case 269:
 
-/* Line 1464 of yacc.c  */
-#line 1572 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1586 "glsl_parser.ypp"
     {
 	   (yyval.for_rest_statement).cond = (yyvsp[(1) - (2)].node);
 	   (yyval.for_rest_statement).rest = NULL;
 	;}
     break;
 
-  case 269:
+  case 270:
 
-/* Line 1464 of yacc.c  */
-#line 1577 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1591 "glsl_parser.ypp"
     {
 	   (yyval.for_rest_statement).cond = (yyvsp[(1) - (3)].node);
 	   (yyval.for_rest_statement).rest = (yyvsp[(3) - (3)].expression);
 	;}
     break;
 
-  case 270:
+  case 271:
 
-/* Line 1464 of yacc.c  */
-#line 1586 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1600 "glsl_parser.ypp"
     {
 	   void *ctx = state;
 	   (yyval.node) = new(ctx) ast_jump_statement(ast_jump_statement::ast_continue, NULL);
@@ -5005,10 +5020,10 @@ yyreduce:
 	;}
     break;
 
-  case 271:
+  case 272:
 
-/* Line 1464 of yacc.c  */
-#line 1592 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1606 "glsl_parser.ypp"
     {
 	   void *ctx = state;
 	   (yyval.node) = new(ctx) ast_jump_statement(ast_jump_statement::ast_break, NULL);
@@ -5016,10 +5031,10 @@ yyreduce:
 	;}
     break;
 
-  case 272:
+  case 273:
 
-/* Line 1464 of yacc.c  */
-#line 1598 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1612 "glsl_parser.ypp"
     {
 	   void *ctx = state;
 	   (yyval.node) = new(ctx) ast_jump_statement(ast_jump_statement::ast_return, NULL);
@@ -5027,10 +5042,10 @@ yyreduce:
 	;}
     break;
 
-  case 273:
+  case 274:
 
-/* Line 1464 of yacc.c  */
-#line 1604 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1618 "glsl_parser.ypp"
     {
 	   void *ctx = state;
 	   (yyval.node) = new(ctx) ast_jump_statement(ast_jump_statement::ast_return, (yyvsp[(2) - (3)].expression));
@@ -5038,10 +5053,10 @@ yyreduce:
 	;}
     break;
 
-  case 274:
+  case 275:
 
-/* Line 1464 of yacc.c  */
-#line 1610 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1624 "glsl_parser.ypp"
     {
 	   void *ctx = state;
 	   (yyval.node) = new(ctx) ast_jump_statement(ast_jump_statement::ast_discard, NULL);
@@ -5049,31 +5064,31 @@ yyreduce:
 	;}
     break;
 
-  case 275:
+  case 276:
 
-/* Line 1464 of yacc.c  */
-#line 1618 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1632 "glsl_parser.ypp"
     { (yyval.node) = (yyvsp[(1) - (1)].function_definition); ;}
     break;
 
-  case 276:
+  case 277:
 
-/* Line 1464 of yacc.c  */
-#line 1619 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1633 "glsl_parser.ypp"
     { (yyval.node) = (yyvsp[(1) - (1)].node); ;}
     break;
 
-  case 277:
+  case 278:
 
-/* Line 1464 of yacc.c  */
-#line 1620 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1634 "glsl_parser.ypp"
     { (yyval.node) = NULL; ;}
     break;
 
-  case 278:
+  case 279:
 
-/* Line 1464 of yacc.c  */
-#line 1625 "glsl_parser.ypp"
+/* Line 1455 of yacc.c  */
+#line 1639 "glsl_parser.ypp"
     {
 	   void *ctx = state;
 	   (yyval.function_definition) = new(ctx) ast_function_definition();
@@ -5085,8 +5100,8 @@ yyreduce:
 
 
 
-/* Line 1464 of yacc.c  */
-#line 5090 "glsl_parser.cpp"
+/* Line 1455 of yacc.c  */
+#line 5105 "glsl_parser.cpp"
       default: break;
     }
   YY_SYMBOL_PRINT ("-> $$ =", yyr1[yyn], &yyval, &yyloc);
@@ -5158,7 +5173,7 @@ yyerrlab:
 #endif
     }
 
-  yyerror_range[1] = yylloc;
+  yyerror_range[0] = yylloc;
 
   if (yyerrstatus == 3)
     {
@@ -5195,7 +5210,7 @@ yyerrorlab:
   if (/*CONSTCOND*/ 0)
      goto yyerrorlab;
 
-  yyerror_range[1] = yylsp[1-yylen];
+  yyerror_range[0] = yylsp[1-yylen];
   /* Do not reclaim the symbols of the rule which action triggered
      this YYERROR.  */
   YYPOPSTACK (yylen);
@@ -5229,7 +5244,7 @@ yyerrlab1:
       if (yyssp == yyss)
 	YYABORT;
 
-      yyerror_range[1] = *yylsp;
+      yyerror_range[0] = *yylsp;
       yydestruct ("Error: popping",
 		  yystos[yystate], yyvsp, yylsp, state);
       YYPOPSTACK (1);
@@ -5239,10 +5254,10 @@ yyerrlab1:
 
   *++yyvsp = yylval;
 
-  yyerror_range[2] = yylloc;
+  yyerror_range[1] = yylloc;
   /* Using YYLLOC is tempting, but would change the location of
      the lookahead.  YYLOC is available though.  */
-  YYLLOC_DEFAULT (yyloc, yyerror_range, 2);
+  YYLLOC_DEFAULT (yyloc, (yyerror_range - 1), 2);
   *++yylsp = yyloc;
 
   /* Shift the error token.  */
diff --git a/src/glsl/glsl_parser.h b/src/glsl/glsl_parser.h
index 83aa56f7cbe..01b407d4dbe 100644
--- a/src/glsl/glsl_parser.h
+++ b/src/glsl/glsl_parser.h
@@ -1,9 +1,10 @@
-/* A Bison parser, made by GNU Bison 2.4.3.  */
+
+/* A Bison parser, made by GNU Bison 2.4.1.  */
 
 /* Skeleton interface for Bison's Yacc-like parsers in C
    
-      Copyright (C) 1984, 1989, 1990, 2000, 2001, 2002, 2003, 2004, 2005, 2006,
-   2009, 2010 Free Software Foundation, Inc.
+      Copyright (C) 1984, 1989, 1990, 2000, 2001, 2002, 2003, 2004, 2005, 2006
+   Free Software Foundation, Inc.
    
    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -155,79 +156,80 @@
      PRAGMA_DEBUG_OFF = 372,
      PRAGMA_OPTIMIZE_ON = 373,
      PRAGMA_OPTIMIZE_OFF = 374,
-     LAYOUT_TOK = 375,
-     ASM = 376,
-     CLASS = 377,
-     UNION = 378,
-     ENUM = 379,
-     TYPEDEF = 380,
-     TEMPLATE = 381,
-     THIS = 382,
-     PACKED_TOK = 383,
-     GOTO = 384,
-     INLINE_TOK = 385,
-     NOINLINE = 386,
-     VOLATILE = 387,
-     PUBLIC_TOK = 388,
-     STATIC = 389,
-     EXTERN = 390,
-     EXTERNAL = 391,
-     LONG_TOK = 392,
-     SHORT_TOK = 393,
-     DOUBLE_TOK = 394,
-     HALF = 395,
-     FIXED_TOK = 396,
-     UNSIGNED = 397,
-     INPUT_TOK = 398,
-     OUPTUT = 399,
-     HVEC2 = 400,
-     HVEC3 = 401,
-     HVEC4 = 402,
-     DVEC2 = 403,
-     DVEC3 = 404,
-     DVEC4 = 405,
-     FVEC2 = 406,
-     FVEC3 = 407,
-     FVEC4 = 408,
-     SAMPLER2DRECT = 409,
-     SAMPLER3DRECT = 410,
-     SAMPLER2DRECTSHADOW = 411,
-     SIZEOF = 412,
-     CAST = 413,
-     NAMESPACE = 414,
-     USING = 415,
-     ERROR_TOK = 416,
-     COMMON = 417,
-     PARTITION = 418,
-     ACTIVE = 419,
-     SAMPLERBUFFER = 420,
-     FILTER = 421,
-     IMAGE1D = 422,
-     IMAGE2D = 423,
-     IMAGE3D = 424,
-     IMAGECUBE = 425,
-     IMAGE1DARRAY = 426,
-     IMAGE2DARRAY = 427,
-     IIMAGE1D = 428,
-     IIMAGE2D = 429,
-     IIMAGE3D = 430,
-     IIMAGECUBE = 431,
-     IIMAGE1DARRAY = 432,
-     IIMAGE2DARRAY = 433,
-     UIMAGE1D = 434,
-     UIMAGE2D = 435,
-     UIMAGE3D = 436,
-     UIMAGECUBE = 437,
-     UIMAGE1DARRAY = 438,
-     UIMAGE2DARRAY = 439,
-     IMAGE1DSHADOW = 440,
-     IMAGE2DSHADOW = 441,
-     IMAGEBUFFER = 442,
-     IIMAGEBUFFER = 443,
-     UIMAGEBUFFER = 444,
-     IMAGE1DARRAYSHADOW = 445,
-     IMAGE2DARRAYSHADOW = 446,
-     ROW_MAJOR = 447
+     PRAGMA_INVARIANT_ALL = 375,
+     LAYOUT_TOK = 376,
+     ASM = 377,
+     CLASS = 378,
+     UNION = 379,
+     ENUM = 380,
+     TYPEDEF = 381,
+     TEMPLATE = 382,
+     THIS = 383,
+     PACKED_TOK = 384,
+     GOTO = 385,
+     INLINE_TOK = 386,
+     NOINLINE = 387,
+     VOLATILE = 388,
+     PUBLIC_TOK = 389,
+     STATIC = 390,
+     EXTERN = 391,
+     EXTERNAL = 392,
+     LONG_TOK = 393,
+     SHORT_TOK = 394,
+     DOUBLE_TOK = 395,
+     HALF = 396,
+     FIXED_TOK = 397,
+     UNSIGNED = 398,
+     INPUT_TOK = 399,
+     OUPTUT = 400,
+     HVEC2 = 401,
+     HVEC3 = 402,
+     HVEC4 = 403,
+     DVEC2 = 404,
+     DVEC3 = 405,
+     DVEC4 = 406,
+     FVEC2 = 407,
+     FVEC3 = 408,
+     FVEC4 = 409,
+     SAMPLER2DRECT = 410,
+     SAMPLER3DRECT = 411,
+     SAMPLER2DRECTSHADOW = 412,
+     SIZEOF = 413,
+     CAST = 414,
+     NAMESPACE = 415,
+     USING = 416,
+     ERROR_TOK = 417,
+     COMMON = 418,
+     PARTITION = 419,
+     ACTIVE = 420,
+     SAMPLERBUFFER = 421,
+     FILTER = 422,
+     IMAGE1D = 423,
+     IMAGE2D = 424,
+     IMAGE3D = 425,
+     IMAGECUBE = 426,
+     IMAGE1DARRAY = 427,
+     IMAGE2DARRAY = 428,
+     IIMAGE1D = 429,
+     IIMAGE2D = 430,
+     IIMAGE3D = 431,
+     IIMAGECUBE = 432,
+     IIMAGE1DARRAY = 433,
+     IIMAGE2DARRAY = 434,
+     UIMAGE1D = 435,
+     UIMAGE2D = 436,
+     UIMAGE3D = 437,
+     UIMAGECUBE = 438,
+     UIMAGE1DARRAY = 439,
+     UIMAGE2DARRAY = 440,
+     IMAGE1DSHADOW = 441,
+     IMAGE2DSHADOW = 442,
+     IMAGEBUFFER = 443,
+     IIMAGEBUFFER = 444,
+     UIMAGEBUFFER = 445,
+     IMAGE1DARRAYSHADOW = 446,
+     IMAGE2DARRAYSHADOW = 447,
+     ROW_MAJOR = 448
    };
 #endif
 
@@ -237,7 +239,7 @@
 typedef union YYSTYPE
 {
 
-/* Line 1685 of yacc.c  */
+/* Line 1676 of yacc.c  */
 #line 52 "glsl_parser.ypp"
 
    int n;
@@ -270,8 +272,8 @@ typedef union YYSTYPE
 
 
 
-/* Line 1685 of yacc.c  */
-#line 275 "glsl_parser.h"
+/* Line 1676 of yacc.c  */
+#line 277 "glsl_parser.h"
 } YYSTYPE;
 # define YYSTYPE_IS_TRIVIAL 1
 # define yystype YYSTYPE /* obsolescent; will be withdrawn */
diff --git a/src/glsl/glsl_parser.ypp b/src/glsl/glsl_parser.ypp
index 6d7d148eb04..41b51a74725 100644
--- a/src/glsl/glsl_parser.ypp
+++ b/src/glsl/glsl_parser.ypp
@@ -108,6 +108,7 @@
 %token VERSION EXTENSION LINE COLON EOL INTERFACE OUTPUT
 %token PRAGMA_DEBUG_ON PRAGMA_DEBUG_OFF
 %token PRAGMA_OPTIMIZE_ON PRAGMA_OPTIMIZE_OFF
+%token PRAGMA_INVARIANT_ALL
 %token LAYOUT_TOK
 
    /* Reserved words that are not actually used in the grammar.
@@ -241,6 +242,19 @@ pragma_statement:
 	| PRAGMA_DEBUG_OFF EOL
 	| PRAGMA_OPTIMIZE_ON EOL
 	| PRAGMA_OPTIMIZE_OFF EOL
+	| PRAGMA_INVARIANT_ALL EOL
+	{
+	   if (state->language_version < 120) {
+	      _mesa_glsl_warning(& @1, state,
+				 "pragma `invariant(all)' not supported in "
+				 "GLSL%s %d.%02d",
+				 state->es_shader ? " ES" : "",
+				 state->language_version / 100,
+				 state->language_version % 100);
+	   } else {
+	      state->all_invariant = true;
+	   }
+	}
 	;
 
 extension_statement_list:
diff --git a/src/glsl/glsl_parser_extras.h b/src/glsl/glsl_parser_extras.h
index 98c4e70173d..13d3a2979e4 100644
--- a/src/glsl/glsl_parser_extras.h
+++ b/src/glsl/glsl_parser_extras.h
@@ -108,6 +108,13 @@ struct _mesa_glsl_parse_state {
    /** Was there an error during compilation? */
    bool error;
 
+   /**
+    * Are all shader inputs / outputs invariant?
+    *
+    * This is set when the 'STDGL invariant(all)' pragma is used.
+    */
+   bool all_invariant;
+
    /** Loop or switch statement containing the current instructions. */
    class ir_instruction *loop_or_switch_nesting;
    class ast_iteration_statement *loop_or_switch_nesting_ast;
diff --git a/src/glsl/ir.h b/src/glsl/ir.h
index 102a68b6551..9668c9439ad 100644
--- a/src/glsl/ir.h
+++ b/src/glsl/ir.h
@@ -1582,8 +1582,7 @@ _mesa_glsl_initialize_variables(exec_list *instructions,
 				struct _mesa_glsl_parse_state *state);
 
 extern void
-_mesa_glsl_initialize_functions(exec_list *instructions,
-				struct _mesa_glsl_parse_state *state);
+_mesa_glsl_initialize_functions(_mesa_glsl_parse_state *state);
 
 extern void
 _mesa_glsl_release_functions(void);
diff --git a/src/glsl/ir_optimization.h b/src/glsl/ir_optimization.h
index f264265f4b1..dbc9f4ab975 100644
--- a/src/glsl/ir_optimization.h
+++ b/src/glsl/ir_optimization.h
@@ -53,7 +53,7 @@ bool do_lower_jumps(exec_list *instructions, bool pull_out_jumps = true, bool lo
 bool do_lower_texture_projection(exec_list *instructions);
 bool do_if_simplification(exec_list *instructions);
 bool do_discard_simplification(exec_list *instructions);
-bool do_if_to_cond_assign(exec_list *instructions);
+bool lower_if_to_cond_assign(exec_list *instructions, unsigned max_depth = 0);
 bool do_mat_op_to_vec(exec_list *instructions);
 bool do_mod_to_fract(exec_list *instructions);
 bool do_noop_swizzle(exec_list *instructions);
diff --git a/src/glsl/lower_if_to_cond_assign.cpp b/src/glsl/lower_if_to_cond_assign.cpp
index cf48cfb8d61..40ffc45c86c 100644
--- a/src/glsl/lower_if_to_cond_assign.cpp
+++ b/src/glsl/lower_if_to_cond_assign.cpp
@@ -24,12 +24,25 @@
 /**
  * \file lower_if_to_cond_assign.cpp
  *
- * This attempts to flatten all if statements to conditional
- * assignments for GPUs that don't do control flow.
+ * This attempts to flatten if-statements to conditional assignments for
+ * GPUs with limited or no flow control support.
  *
  * It can't handle other control flow being inside of its block, such
  * as calls or loops.  Hopefully loop unrolling and inlining will take
  * care of those.
+ *
+ * Drivers for GPUs with no control flow support should simply call
+ *
+ *    lower_if_to_cond_assign(instructions)
+ *
+ * to attempt to flatten all if-statements.
+ *
+ * Some GPUs (such as i965 prior to gen6) do support control flow, but have a
+ * maximum nesting depth N.  Drivers for such hardware can call
+ *
+ *    lower_if_to_cond_assign(instructions, N)
+ *
+ * to attempt to flatten any if-statements appearing at depth > N.
  */
 
 #include "glsl_types.h"
@@ -37,20 +50,25 @@
 
 class ir_if_to_cond_assign_visitor : public ir_hierarchical_visitor {
 public:
-   ir_if_to_cond_assign_visitor()
+   ir_if_to_cond_assign_visitor(unsigned max_depth)
    {
       this->progress = false;
+      this->max_depth = max_depth;
+      this->depth = 0;
    }
 
+   ir_visitor_status visit_enter(ir_if *);
    ir_visitor_status visit_leave(ir_if *);
 
    bool progress;
+   unsigned max_depth;
+   unsigned depth;
 };
 
 bool
-do_if_to_cond_assign(exec_list *instructions)
+lower_if_to_cond_assign(exec_list *instructions, unsigned max_depth)
 {
-   ir_if_to_cond_assign_visitor v;
+   ir_if_to_cond_assign_visitor v(max_depth);
 
    visit_list_elements(&v, instructions);
 
@@ -120,8 +138,22 @@ move_block_to_cond_assign(void *mem_ctx,
 }
 
 ir_visitor_status
+ir_if_to_cond_assign_visitor::visit_enter(ir_if *ir)
+{
+   (void) ir;
+   this->depth++;
+   return visit_continue;
+}
+
+ir_visitor_status
 ir_if_to_cond_assign_visitor::visit_leave(ir_if *ir)
 {
+   /* Only flatten when beyond the GPU's maximum supported nesting depth. */
+   if (this->depth <= this->max_depth)
+      return visit_continue;
+
+   this->depth--;
+
    bool found_control_flow = false;
    ir_variable *cond_var;
    ir_assignment *assign;
diff --git a/src/mapi/mapi/entry.c b/src/mapi/mapi/entry.c
index fdf2a89c52f..69b6134bd27 100644
--- a/src/mapi/mapi/entry.c
+++ b/src/mapi/mapi/entry.c
@@ -40,9 +40,10 @@
 
 #include <stdlib.h>
 #include "u_current.h"
-#include "table.h"
 
 /* C version of the public entries */
+#define MAPI_TMP_DEFINES
+#define MAPI_TMP_PUBLIC_DECLARES
 #define MAPI_TMP_PUBLIC_ENTRIES
 #include "mapi_tmp.h"
 
@@ -52,6 +53,13 @@ entry_patch_public(void)
 }
 
 mapi_func
+entry_get_public(int slot)
+{
+   /* pubic_entries are defined by MAPI_TMP_PUBLIC_ENTRIES */
+   return public_entries[slot];
+}
+
+mapi_func
 entry_generate(int slot)
 {
    return NULL;
diff --git a/src/mapi/mapi/entry.h b/src/mapi/mapi/entry.h
index 48ed3f9ec4a..9df81000470 100644
--- a/src/mapi/mapi/entry.h
+++ b/src/mapi/mapi/entry.h
@@ -30,17 +30,16 @@
 #define _ENTRY_H_
 
 #include "u_compiler.h"
-#include "stub.h"
 
-/* declare public entries */
-#define MAPI_TMP_DEFINES
-#define MAPI_TMP_PUBLIC_DECLARES
-#include "mapi_tmp.h"
+typedef void (*mapi_func)(void);
 
 void
 entry_patch_public(void);
 
 mapi_func
+entry_get_public(int slot);
+
+mapi_func
 entry_generate(int slot);
 
 void
diff --git a/src/mapi/mapi/entry_x86-64_tls.h b/src/mapi/mapi/entry_x86-64_tls.h
index 2c02933802f..d14bf1c3946 100644
--- a/src/mapi/mapi/entry_x86-64_tls.h
+++ b/src/mapi/mapi/entry_x86-64_tls.h
@@ -49,6 +49,9 @@ __asm__("x86_64_current_tls:\n\t"
 	"movq u_current_table@GOTTPOFF(%rip), %rax\n\t"
 	"ret");
 
+__asm__(".balign 32\n"
+        "x86_64_entry_start:");
+
 #define STUB_ASM_ENTRY(func)                             \
    ".globl " func "\n"                                   \
    ".type " func ", @function\n"                         \
@@ -71,6 +74,13 @@ entry_patch_public(void)
 {
 }
 
+mapi_func
+entry_get_public(int slot)
+{
+   extern char x86_64_entry_start[];
+   return (mapi_func) (x86_64_entry_start + slot * 32);
+}
+
 void
 entry_patch(mapi_func entry, int slot)
 {
diff --git a/src/mapi/mapi/entry_x86_tls.h b/src/mapi/mapi/entry_x86_tls.h
index 3d0b7caffe1..ea63490e1c7 100644
--- a/src/mapi/mapi/entry_x86_tls.h
+++ b/src/mapi/mapi/entry_x86_tls.h
@@ -54,11 +54,12 @@ __asm__("x86_current_tls:\n\t"
 	"ret");
 
 #ifndef GLX_X86_READONLY_TEXT
-__asm__(".section wtext, \"awx\", @progbits\n"
-        ".balign 16\n"
-        "x86_entry_start:");
+__asm__(".section wtext, \"awx\", @progbits");
 #endif /* GLX_X86_READONLY_TEXT */
 
+__asm__(".balign 16\n"
+        "x86_entry_start:");
+
 #define STUB_ASM_ENTRY(func)     \
    ".globl " func "\n"           \
    ".type " func ", @function\n" \
@@ -101,6 +102,13 @@ entry_patch_public(void)
 #endif
 }
 
+mapi_func
+entry_get_public(int slot)
+{
+   extern char x86_entry_start[];
+   return (mapi_func) (x86_entry_start + slot * 16);
+}
+
 void
 entry_patch(mapi_func entry, int slot)
 {
diff --git a/src/mapi/mapi/entry_x86_tsd.h b/src/mapi/mapi/entry_x86_tsd.h
index f37c7473a6f..0a07ad74247 100644
--- a/src/mapi/mapi/entry_x86_tsd.h
+++ b/src/mapi/mapi/entry_x86_tsd.h
@@ -32,7 +32,9 @@
 
 #define X86_ENTRY_SIZE 32
 
-__asm__(".text");
+__asm__(".text\n"
+        ".balign 32\n"
+        "x86_entry_start:");
 
 #define STUB_ASM_ENTRY(func)        \
    ".globl " func "\n"              \
@@ -60,6 +62,13 @@ entry_patch_public(void)
 {
 }
 
+mapi_func
+entry_get_public(int slot)
+{
+   extern const char x86_entry_start[];
+   return (mapi_func) (x86_entry_start + slot * X86_ENTRY_SIZE);
+}
+
 void
 entry_patch(mapi_func entry, int slot)
 {
diff --git a/src/mapi/mapi/mapi.c b/src/mapi/mapi/mapi.c
index 5476d37fdc9..b471c40b144 100644
--- a/src/mapi/mapi/mapi.c
+++ b/src/mapi/mapi/mapi.c
@@ -132,7 +132,7 @@ mapi_get_proc_address(const char *name)
    if (!stub)
       stub = stub_find_dynamic(name, 0);
 
-   return (stub) ? (mapi_proc) stub->addr : NULL;
+   return (stub) ? (mapi_proc) stub_get_addr(stub) : NULL;
 }
 
 /**
@@ -172,11 +172,12 @@ mapi_table_fill(struct mapi_table *tbl, const mapi_proc *procs)
 
    for (i = 0; i < mapi_num_stubs; i++) {
       const struct mapi_stub *stub = mapi_stub_map[i];
+      int slot = stub_get_slot(stub);
       mapi_func func = (mapi_func) procs[i];
 
       if (!func)
-         func = table_get_func(noop, stub);
-      table_set_func(tbl, stub, func);
+         func = table_get_func(noop, slot);
+      table_set_func(tbl, slot, func);
    }
 }
 
diff --git a/src/mapi/mapi/mapi_abi.py b/src/mapi/mapi/mapi_abi.py
index 3a872666f96..5c212420a80 100644
--- a/src/mapi/mapi/mapi_abi.py
+++ b/src/mapi/mapi/mapi_abi.py
@@ -45,6 +45,7 @@ class ABIEntry(object):
         self.slot = attrs['slot']
         self.hidden = attrs['hidden']
         self.alias = attrs['alias']
+        self.handcode = attrs['handcode']
 
     def c_prototype(self):
         return '%s %s(%s)' % (self.c_return(), self.name, self.c_params())
@@ -132,6 +133,7 @@ def abi_parse_line(line):
             'slot': -1,
             'hidden': False,
             'alias': None,
+            'handcode': None,
     }
 
     # extract attributes from the first column
@@ -144,6 +146,8 @@ def abi_parse_line(line):
             attrs['hidden'] = True
         elif val.startswith('alias='):
             attrs['alias'] = val[6:]
+        elif val.startswith('handcode='):
+            attrs['handcode'] = val[9:]
         elif not val:
             pass
         else:
@@ -166,10 +170,13 @@ def abi_parse(filename):
         # post-process attributes
         if attrs['alias']:
             try:
-                ent = entry_dict[attrs['alias']]
-                slot = ent.slot
+                alias = entry_dict[attrs['alias']]
             except KeyError:
                 raise Exception('failed to alias %s' % attrs['alias'])
+            if alias.alias:
+                raise Exception('recursive alias %s' % ent.name)
+            slot = alias.slot
+            attrs['alias'] = alias
         else:
             slot = next_slot
             next_slot += 1
@@ -194,8 +201,15 @@ def abi_parse(filename):
             raise Exception('entries are not ordered by slots')
         if entries[i].alias:
             raise Exception('first entry of slot %d aliases %s'
-                    % (slot, entries[i].alias))
+                    % (slot, entries[i].alias.name))
+        handcode = None
         while i < len(entries) and entries[i].slot == slot:
+            ent = entries[i]
+            if not handcode and ent.handcode:
+                handcode = ent.handcode
+            elif ent.handcode != handcode:
+                raise Exception('two aliases with handcode %s != %s',
+                        ent.handcode, handcode)
             i += 1
     if i < len(entries):
         raise Exception('there are %d invalid entries' % (len(entries) - 1))
@@ -222,15 +236,37 @@ class ABIPrinter(object):
         self.api_entry = 'KHRONOS_APIENTRY'
         self.api_attrs = 'KHRONOS_APIATTRIBUTES'
 
-    def c_header(self):
+        self.lib_need_table_size = True
+        self.lib_need_noop_array = True
+        self.lib_need_stubs = True
+        self.lib_need_entries = True
+
+    def c_notice(self):
         return '/* This file is automatically generated by mapi_abi.py.  Do not modify. */'
 
-    def c_includes(self):
+    def c_public_includes(self):
         """Return includes of the client API headers."""
         defines = ['#define ' + d for d in self.api_defines]
         includes = ['#include ' + h for h in self.api_headers]
         return "\n".join(defines + includes)
 
+    def need_entry_point(self, ent):
+        """Return True if an entry point is needed for the entry."""
+        # non-handcode hidden aliases may share the entry they alias
+        use_alias = (ent.hidden and ent.alias and not ent.handcode)
+        return not use_alias
+
+    def c_public_declarations(self, prefix):
+        """Return the declarations of public entry points."""
+        decls = []
+        for ent in self.entries:
+            if not self.need_entry_point(ent):
+                continue
+            export = self.api_call if not ent.hidden else ''
+            decls.append(self._c_decl(ent, prefix, True, export) + ';')
+
+        return "\n".join(decls)
+
     def c_mapi_table(self):
         """Return defines of the dispatch table size."""
         num_static_entries = 0
@@ -244,8 +280,9 @@ class ABIPrinter(object):
 
     def c_mapi_table_initializer(self, prefix):
         """Return the array initializer for mapi_table_fill."""
-        entries = [ent.name for ent in self.entries if not ent.alias]
-        pre = self.indent + '(mapi_proc) ' + prefix
+        entries = [self._c_function(ent, prefix)
+                for ent in self.entries if not ent.alias]
+        pre = self.indent + '(mapi_proc) '
         return pre + (',\n' + pre).join(entries)
 
     def c_mapi_table_spec(self):
@@ -263,11 +300,34 @@ class ABIPrinter(object):
 
         return self.indent + self.indent.join(specv1)
 
-    def _c_decl(self, ent, prefix, need_attr=True):
+    def _c_function(self, ent, prefix, mangle=False, stringify=False):
+        """Return the function name of an entry."""
+        formats = { True: '"%s%s"', False: '%s%s' }
+        fmt = formats[stringify]
+        name = ent.name
+        if mangle and ent.hidden:
+            name = '_dispatch_stub_' + str(ent.slot)
+        return fmt % (prefix, name)
+
+    def _c_function_call(self, ent, prefix):
+        """Return the function name used for calling."""
+        if ent.handcode:
+            # _c_function does not handle this case
+            fmt = '%s%s'
+            name = fmt % (prefix, ent.handcode)
+        elif self.need_entry_point(ent):
+            name = self._c_function(ent, prefix, True)
+        else:
+            name = self._c_function(ent.alias, prefix, True)
+        return name
+
+    def _c_decl(self, ent, prefix, mangle=False, export=''):
         """Return the C declaration for the entry."""
-        decl = '%s %s %s%s(%s)' % (ent.c_return(), self.api_entry,
-                prefix, ent.name, ent.c_params())
-        if need_attr and self.api_attrs:
+        decl = '%s %s %s(%s)' % (ent.c_return(), self.api_entry,
+                self._c_function(ent, prefix, mangle), ent.c_params())
+        if export:
+            decl = export + ' ' + decl
+        if self.api_attrs:
             decl += ' ' + self.api_attrs
 
         return decl
@@ -281,19 +341,21 @@ class ABIPrinter(object):
 
     def c_private_declarations(self, prefix):
         """Return the declarations of private functions."""
-        decls = [self._c_decl(ent, prefix)
+        decls = [self._c_decl(ent, prefix) + ';'
                 for ent in self.entries if not ent.alias]
 
-        return ";\n".join(decls) + ";"
+        return "\n".join(decls)
 
     def c_public_dispatches(self, prefix):
         """Return the public dispatch functions."""
         dispatches = []
         for ent in self.entries:
-            if ent.hidden:
+            if not self.need_entry_point(ent):
                 continue
 
-            proto = self.api_call + ' ' + self._c_decl(ent, prefix)
+            export = self.api_call if not ent.hidden else ''
+
+            proto = self._c_decl(ent, prefix, True, export)
             cast = self._c_cast(ent)
 
             ret = ''
@@ -308,10 +370,27 @@ class ABIPrinter(object):
             stmt3 += '%s((%s) func)(%s);' % (ret, cast, ent.c_args())
 
             disp = '%s\n{\n%s\n%s\n%s\n}' % (proto, stmt1, stmt2, stmt3)
+
+            if ent.handcode:
+                disp = '#if 0\n' + disp + '\n#endif'
+
             dispatches.append(disp)
 
         return '\n\n'.join(dispatches)
 
+    def c_public_initializer(self, prefix):
+        """Return the initializer for public dispatch functions."""
+        names = []
+        for ent in self.entries:
+            if ent.alias:
+                continue
+
+            name = '%s(mapi_func) %s' % (self.indent,
+                    self._c_function_call(ent, prefix))
+            names.append(name)
+
+        return ',\n'.join(names)
+
     def c_stub_string_pool(self):
         """Return the string pool for use by stubs."""
         # sort entries by their names
@@ -334,8 +413,8 @@ class ABIPrinter(object):
         """Return the initializer for struct mapi_stub array."""
         stubs = []
         for ent in self.entries_sorted_by_names:
-            stubs.append('%s{ (mapi_func) %s%s, %d, (void *) %d }' % (
-                self.indent, prefix, ent.name, ent.slot, pool_offsets[ent]))
+            stubs.append('%s{ (void *) %d, %d, NULL }' % (
+                self.indent, pool_offsets[ent], ent.slot))
 
         return ',\n'.join(stubs)
 
@@ -346,10 +425,10 @@ class ABIPrinter(object):
             if ent.alias:
                 continue
 
-            proto = 'static ' + self._c_decl(ent, prefix)
+            proto = self._c_decl(ent, prefix, False, 'static')
 
-            stmt1 = self.indent + '%s("%s%s");' % (
-                    self.noop_warn, warn_prefix, ent.name)
+            stmt1 = self.indent + '%s(%s);' % (self.noop_warn,
+                    self._c_function(ent, warn_prefix, False, True))
 
             if ent.ret:
                 stmt2 = self.indent + 'return (%s) 0;' % (ent.ret)
@@ -363,7 +442,8 @@ class ABIPrinter(object):
 
     def c_noop_initializer(self, prefix, use_generic):
         """Return an initializer for the noop dispatch table."""
-        entries = [prefix + ent.name for ent in self.entries if not ent.alias]
+        entries = [self._c_function(ent, prefix)
+                for ent in self.entries if not ent.alias]
         if use_generic:
             entries = [self.noop_generic] * len(entries)
 
@@ -374,84 +454,105 @@ class ABIPrinter(object):
 
     def c_asm_gcc(self, prefix):
         asm = []
-        to_name = None
 
         asm.append('__asm__(')
         for ent in self.entries:
-            name = prefix + ent.name
+            if not self.need_entry_point(ent):
+                continue
+
+            name = self._c_function(ent, prefix, True, True)
+
+            if ent.handcode:
+                asm.append('#if 0')
 
             if ent.hidden:
-                asm.append('".hidden %s\\n"' % (name))
+                asm.append('".hidden "%s"\\n"' % (name))
 
             if ent.alias:
-                asm.append('".globl %s\\n"' % (name))
-                asm.append('".set %s, %s\\n"' % (name, to_name))
+                asm.append('".globl "%s"\\n"' % (name))
+                asm.append('".set "%s", "%s"\\n"' % (name,
+                    self._c_function(ent.alias, prefix, True, True)))
             else:
-                asm.append('STUB_ASM_ENTRY("%s")"\\n"' % (name))
+                asm.append('STUB_ASM_ENTRY(%s)"\\n"' % (name))
                 asm.append('"\\t"STUB_ASM_CODE("%d")"\\n"' % (ent.slot))
-                to_name = name
+
+            if ent.handcode:
+                asm.append('#endif')
+            asm.append('')
         asm.append(');')
 
         return "\n".join(asm)
 
     def output_for_lib(self):
-        print self.c_header()
+        print self.c_notice()
         print
         print '#ifdef MAPI_TMP_DEFINES'
-        print self.c_includes()
+        print self.c_public_includes()
+        print
+        print self.c_public_declarations(self.prefix_lib)
         print '#undef MAPI_TMP_DEFINES'
         print '#endif /* MAPI_TMP_DEFINES */'
-        print
-        print '#ifdef MAPI_TMP_TABLE'
-        print self.c_mapi_table()
-        print '#undef MAPI_TMP_TABLE'
-        print '#endif /* MAPI_TMP_TABLE */'
-        print
-
-        pool, pool_offsets = self.c_stub_string_pool()
-        print '#ifdef MAPI_TMP_PUBLIC_STUBS'
-        print 'static const char public_string_pool[] ='
-        print pool
-        print
-        print 'static const struct mapi_stub public_stubs[] = {'
-        print self.c_stub_initializer(self.prefix_lib, pool_offsets)
-        print '};'
-        print '#undef MAPI_TMP_PUBLIC_STUBS'
-        print '#endif /* MAPI_TMP_PUBLIC_STUBS */'
-        print
-
-        print '#ifdef MAPI_TMP_PUBLIC_ENTRIES'
-        print self.c_public_dispatches(self.prefix_lib)
-        print '#undef MAPI_TMP_PUBLIC_ENTRIES'
-        print '#endif /* MAPI_TMP_PUBLIC_ENTRIES */'
-        print
-
-        print '#ifdef MAPI_TMP_NOOP_ARRAY'
-        print '#ifdef DEBUG'
-        print
-        print self.c_noop_functions(self.prefix_noop, self.prefix_lib)
-        print
-        print 'const mapi_func table_%s_array[] = {' % (self.prefix_noop)
-        print self.c_noop_initializer(self.prefix_noop, False)
-        print '};'
-        print
-        print '#else /* DEBUG */'
-        print
-        print 'const mapi_func table_%s_array[] = {' % (self.prefix_noop)
-        print self.c_noop_initializer(self.prefix_noop, True)
-        print '};'
-        print '#endif /* DEBUG */'
-        print '#undef MAPI_TMP_NOOP_ARRAY'
-        print '#endif /* MAPI_TMP_NOOP_ARRAY */'
-        print
 
-        print '#ifdef MAPI_TMP_STUB_ASM_GCC'
-        print self.c_asm_gcc(self.prefix_lib)
-        print '#undef MAPI_TMP_STUB_ASM_GCC'
-        print '#endif /* MAPI_TMP_STUB_ASM_GCC */'
+        if self.lib_need_table_size:
+            print
+            print '#ifdef MAPI_TMP_TABLE'
+            print self.c_mapi_table()
+            print '#undef MAPI_TMP_TABLE'
+            print '#endif /* MAPI_TMP_TABLE */'
+
+        if self.lib_need_noop_array:
+            print
+            print '#ifdef MAPI_TMP_NOOP_ARRAY'
+            print '#ifdef DEBUG'
+            print
+            print self.c_noop_functions(self.prefix_noop, self.prefix_lib)
+            print
+            print 'const mapi_func table_%s_array[] = {' % (self.prefix_noop)
+            print self.c_noop_initializer(self.prefix_noop, False)
+            print '};'
+            print
+            print '#else /* DEBUG */'
+            print
+            print 'const mapi_func table_%s_array[] = {' % (self.prefix_noop)
+            print self.c_noop_initializer(self.prefix_noop, True)
+            print '};'
+            print
+            print '#endif /* DEBUG */'
+            print '#undef MAPI_TMP_NOOP_ARRAY'
+            print '#endif /* MAPI_TMP_NOOP_ARRAY */'
+
+        if self.lib_need_stubs:
+            pool, pool_offsets = self.c_stub_string_pool()
+            print
+            print '#ifdef MAPI_TMP_PUBLIC_STUBS'
+            print 'static const char public_string_pool[] ='
+            print pool
+            print
+            print 'static const struct mapi_stub public_stubs[] = {'
+            print self.c_stub_initializer(self.prefix_lib, pool_offsets)
+            print '};'
+            print '#undef MAPI_TMP_PUBLIC_STUBS'
+            print '#endif /* MAPI_TMP_PUBLIC_STUBS */'
+
+        if self.lib_need_entries:
+            print
+            print '#ifdef MAPI_TMP_PUBLIC_ENTRIES'
+            print self.c_public_dispatches(self.prefix_lib)
+            print
+            print 'static const mapi_func public_entries[] = {'
+            print self.c_public_initializer(self.prefix_lib)
+            print '};'
+            print '#undef MAPI_TMP_PUBLIC_ENTRIES'
+            print '#endif /* MAPI_TMP_PUBLIC_ENTRIES */'
+
+            print
+            print '#ifdef MAPI_TMP_STUB_ASM_GCC'
+            print self.c_asm_gcc(self.prefix_lib)
+            print '#undef MAPI_TMP_STUB_ASM_GCC'
+            print '#endif /* MAPI_TMP_STUB_ASM_GCC */'
 
     def output_for_app(self):
-        print self.c_header()
+        print self.c_notice()
         print
         print self.c_private_declarations(self.prefix_app)
         print
diff --git a/src/mapi/mapi/stub.c b/src/mapi/mapi/stub.c
index 3594eacb4ec..99b475a3b4b 100644
--- a/src/mapi/mapi/stub.c
+++ b/src/mapi/mapi/stub.c
@@ -27,7 +27,6 @@
  */
 
 #include <stdlib.h>
-#include <stddef.h> /* for offsetof */
 #include <string.h>
 #include <assert.h>
 
@@ -39,6 +38,12 @@
 
 #define ARRAY_SIZE(x) (sizeof(x)/sizeof((x)[0]))
 
+struct mapi_stub {
+   const void *name;
+   int slot;
+   mapi_func addr;
+};
+
 /* define public_string_pool and public_stubs */
 #define MAPI_TMP_PUBLIC_STUBS
 #include "mapi_tmp.h"
@@ -164,3 +169,39 @@ stub_fix_dynamic(struct mapi_stub *stub, const struct mapi_stub *alias)
    entry_patch(stub->addr, slot);
    stub->slot = slot;
 }
+
+/**
+ * Return the name of a stub.
+ */
+const char *
+stub_get_name(const struct mapi_stub *stub)
+{
+   const char *name;
+
+   if (stub >= public_stubs &&
+       stub < public_stubs + ARRAY_SIZE(public_stubs))
+      name = &public_string_pool[(unsigned long) stub->name];
+   else
+      name = (const char *) stub->name;
+
+   return name;
+}
+
+/**
+ * Return the slot of a stub.
+ */
+int
+stub_get_slot(const struct mapi_stub *stub)
+{
+   return stub->slot;
+}
+
+/**
+ * Return the address of a stub.
+ */
+mapi_func
+stub_get_addr(const struct mapi_stub *stub)
+{
+   assert(stub->addr || (unsigned int) stub->slot < MAPI_TABLE_NUM_STATIC);
+   return (stub->addr) ? stub->addr : entry_get_public(stub->slot);
+}
diff --git a/src/mapi/mapi/stub.h b/src/mapi/mapi/stub.h
index c7e194cf4fc..b2b6f1839c6 100644
--- a/src/mapi/mapi/stub.h
+++ b/src/mapi/mapi/stub.h
@@ -29,13 +29,9 @@
 #ifndef _STUB_H_
 #define _STUB_H_
 
-typedef void (*mapi_func)(void);
+#include "entry.h"
 
-struct mapi_stub {
-   mapi_func addr;
-   int slot;
-   const void *name;
-};
+struct mapi_stub;
 
 void
 stub_init_once(void);
@@ -49,4 +45,13 @@ stub_find_dynamic(const char *name, int generate);
 void
 stub_fix_dynamic(struct mapi_stub *stub, const struct mapi_stub *alias);
 
+const char *
+stub_get_name(const struct mapi_stub *stub);
+
+int
+stub_get_slot(const struct mapi_stub *stub);
+
+mapi_func
+stub_get_addr(const struct mapi_stub *stub);
+
 #endif /* _STUB_H_ */
diff --git a/src/mapi/mapi/table.c b/src/mapi/mapi/table.c
index 8f4f700b920..9bb9f654a2a 100644
--- a/src/mapi/mapi/table.c
+++ b/src/mapi/mapi/table.c
@@ -29,7 +29,6 @@
 #include <stdlib.h>
 #include <stdio.h>
 
-#include "stub.h"
 #include "table.h"
 
 static void
@@ -41,7 +40,7 @@ noop_warn(const char *name)
       debug = (getenv("MESA_DEBUG") || getenv("LIBGL_DEBUG"));
 
    if (debug)
-      fprintf(stderr, "%s is no-op", name);
+      fprintf(stderr, "%s is no-op\n", name);
 }
 
 static int
@@ -52,5 +51,6 @@ noop_generic(void)
 }
 
 /* define noop_array */
+#define MAPI_TMP_DEFINES
 #define MAPI_TMP_NOOP_ARRAY
 #include "mapi_tmp.h"
diff --git a/src/mapi/mapi/table.h b/src/mapi/mapi/table.h
index ca2be568c70..d84523f7777 100644
--- a/src/mapi/mapi/table.h
+++ b/src/mapi/mapi/table.h
@@ -30,9 +30,8 @@
 #define _TABLE_H_
 
 #include "u_compiler.h"
-#include "stub.h"
+#include "entry.h"
 
-#define MAPI_TMP_DEFINES
 #define MAPI_TMP_TABLE
 #include "mapi_tmp.h"
 
@@ -51,24 +50,23 @@ table_get_noop(void)
 }
 
 /**
- * Update the dispatch table to dispatch a stub to the given function.
+ * Set the function of a slot.
  */
 static INLINE void
-table_set_func(struct mapi_table *tbl,
-               const struct mapi_stub *stub, mapi_func func)
+table_set_func(struct mapi_table *tbl, int slot, mapi_func func)
 {
    mapi_func *funcs = (mapi_func *) tbl;
-   funcs[stub->slot] = func;
+   funcs[slot] = func;
 }
 
 /**
- * Return the dispatched function of a stub.
+ * Return the function of a slot.
  */
 static INLINE mapi_func
-table_get_func(const struct mapi_table *tbl, const struct mapi_stub *stub)
+table_get_func(const struct mapi_table *tbl, int slot)
 {
    const mapi_func *funcs = (const mapi_func *) tbl;
-   return funcs[stub->slot];
+   return funcs[slot];
 }
 
 #endif /* _TABLE_H_ */
diff --git a/src/mesa/drivers/dri/i915/i830_reg.h b/src/mesa/drivers/dri/i915/i830_reg.h
index ae1317029a2..99ee1bb4e90 100644
--- a/src/mesa/drivers/dri/i915/i830_reg.h
+++ b/src/mesa/drivers/dri/i915/i830_reg.h
@@ -585,6 +585,8 @@
 #define TM0S2_VERITCAL_LINE_STRIDE_OFF  (1<<12)
 #define TM0S2_OUTPUT_CHAN_SHIFT         10
 #define TM0S2_OUTPUT_CHAN_MASK          (3<<10)
+#define TM0S2_BASE_MIP_LEVEL_SHIFT      1
+#define TM0S2_LOD_PRECLAMP              (1 << 0)
 
 #define TM0S3_MIP_FILTER_MASK           (0x3<<30)
 #define TM0S3_MIP_FILTER_SHIFT          30
@@ -605,6 +607,8 @@
 #define TM0S3_MAX_MIP_MASK		(0xff<<9)
 #define TM0S3_MIN_MIP_SHIFT		3
 #define TM0S3_MIN_MIP_MASK		(0x3f<<3)
+#define TM0S3_MIN_MIP_SHIFT_830		5
+#define TM0S3_MIN_MIP_MASK_830		(0x3f<<5)
 #define TM0S3_KILL_PIXEL		(1<<2)
 #define TM0S3_KEYED_FILTER		(1<<1)
 #define TM0S3_CHROMA_KEY		(1<<0)
diff --git a/src/mesa/drivers/dri/i915/i830_texstate.c b/src/mesa/drivers/dri/i915/i830_texstate.c
index b3bb8837cca..c35b4b5ed06 100644
--- a/src/mesa/drivers/dri/i915/i830_texstate.c
+++ b/src/mesa/drivers/dri/i915/i830_texstate.c
@@ -28,13 +28,14 @@
 #include "main/mtypes.h"
 #include "main/enums.h"
 #include "main/colormac.h"
+#include "main/macros.h"
 
 #include "intel_mipmap_tree.h"
 #include "intel_tex.h"
 
 #include "i830_context.h"
 #include "i830_reg.h"
-
+#include "intel_chipset.h"
 
 
 static GLuint
@@ -139,9 +140,9 @@ i830_update_tex_unit(struct intel_context *intel, GLuint unit, GLuint ss3)
    /* Get first image here, since intelObj->firstLevel will get set in
     * the intel_finalize_mipmap_tree() call above.
     */
-   firstImage = tObj->Image[0][intelObj->firstLevel];
+   firstImage = tObj->Image[0][tObj->BaseLevel];
 
-   intel_miptree_get_image_offset(intelObj->mt, intelObj->firstLevel, 0, 0,
+   intel_miptree_get_image_offset(intelObj->mt, tObj->BaseLevel, 0, 0,
 				  &dst_x, &dst_y);
 
    drm_intel_bo_reference(intelObj->mt->region->buffer);
@@ -189,6 +190,8 @@ i830_update_tex_unit(struct intel_context *intel, GLuint unit, GLuint ss3)
 
    {
       GLuint minFilt, mipFilt, magFilt;
+      float maxlod;
+      uint32_t minlod_fixed, maxlod_fixed;
 
       switch (tObj->MinFilter) {
       case GL_NEAREST:
@@ -252,10 +255,24 @@ i830_update_tex_unit(struct intel_context *intel, GLuint unit, GLuint ss3)
          state[I830_TEXREG_TM0S3] |= SS2_COLORSPACE_CONVERSION;
 #endif
 
-      state[I830_TEXREG_TM0S3] |= ((intelObj->lastLevel -
-                                    intelObj->firstLevel) *
-                                   4) << TM0S3_MIN_MIP_SHIFT;
-
+      /* We get one field with fraction bits for the maximum
+       * addressable (smallest resolution) LOD.  Use it to cover both
+       * MAX_LEVEL and MAX_LOD.
+       */
+      minlod_fixed = U_FIXED(CLAMP(tObj->MinLod, 0.0, 11), 4);
+      maxlod = MIN2(tObj->MaxLod, tObj->_MaxLevel - tObj->BaseLevel);
+      if (intel->intelScreen->deviceID == PCI_CHIP_I855_GM ||
+	  intel->intelScreen->deviceID == PCI_CHIP_I865_G) {
+	 maxlod_fixed = U_FIXED(CLAMP(maxlod, 0.0, 11.75), 2);
+	 maxlod_fixed = MAX2(maxlod_fixed, (minlod_fixed + 3) >> 2);
+	 state[I830_TEXREG_TM0S3] |= maxlod_fixed << TM0S3_MIN_MIP_SHIFT;
+	 state[I830_TEXREG_TM0S2] |= TM0S2_LOD_PRECLAMP;
+      } else {
+	 maxlod_fixed = U_FIXED(CLAMP(maxlod, 0.0, 11), 0);
+	 maxlod_fixed = MAX2(maxlod_fixed, (minlod_fixed + 15) >> 4);
+	 state[I830_TEXREG_TM0S3] |= maxlod_fixed << TM0S3_MIN_MIP_SHIFT_830;
+      }
+      state[I830_TEXREG_TM0S3] |= minlod_fixed << TM0S3_MAX_MIP_SHIFT;
       state[I830_TEXREG_TM0S3] |= ((minFilt << TM0S3_MIN_FILTER_SHIFT) |
                                    (mipFilt << TM0S3_MIP_FILTER_SHIFT) |
                                    (magFilt << TM0S3_MAG_FILTER_SHIFT));
diff --git a/src/mesa/drivers/dri/i915/i830_vtbl.c b/src/mesa/drivers/dri/i915/i830_vtbl.c
index 1621c9544ac..ebdefeac874 100644
--- a/src/mesa/drivers/dri/i915/i830_vtbl.c
+++ b/src/mesa/drivers/dri/i915/i830_vtbl.c
@@ -535,14 +535,9 @@ i830_emit_state(struct intel_context *intel)
          BEGIN_BATCH(I830_TEX_SETUP_SIZE + 1);
          OUT_BATCH(state->Tex[i][I830_TEXREG_TM0LI]);
 
-         if (state->tex_buffer[i]) {
-            OUT_RELOC(state->tex_buffer[i],
-		      I915_GEM_DOMAIN_SAMPLER, 0,
-                      state->tex_offset[i]);
-         }
-	 else {
-	    OUT_BATCH(state->tex_offset[i]);
-	 }
+	 OUT_RELOC(state->tex_buffer[i],
+		   I915_GEM_DOMAIN_SAMPLER, 0,
+		   state->tex_offset[i]);
 
          OUT_BATCH(state->Tex[i][I830_TEXREG_TM0S1]);
          OUT_BATCH(state->Tex[i][I830_TEXREG_TM0S2]);
@@ -585,6 +580,27 @@ i830_destroy_context(struct intel_context *intel)
    _tnl_free_vertices(&intel->ctx);
 }
 
+static uint32_t i830_render_target_format_for_mesa_format[MESA_FORMAT_COUNT] =
+{
+   [MESA_FORMAT_ARGB8888] = DV_PF_8888,
+   [MESA_FORMAT_XRGB8888] = DV_PF_8888,
+   [MESA_FORMAT_RGB565] = DV_PF_565,
+   [MESA_FORMAT_ARGB1555] = DV_PF_1555,
+   [MESA_FORMAT_ARGB4444] = DV_PF_4444,
+};
+
+static bool
+i830_render_target_supported(gl_format format)
+{
+   if (format == MESA_FORMAT_S8_Z24 ||
+       format == MESA_FORMAT_X8_Z24 ||
+       format == MESA_FORMAT_Z16) {
+      return true;
+   }
+
+   return i830_render_target_format_for_mesa_format[format] != 0;
+}
+
 static void
 i830_set_draw_region(struct intel_context *intel,
                      struct intel_region *color_regions[],
@@ -624,24 +640,7 @@ i830_set_draw_region(struct intel_context *intel,
             DSTORG_VERT_BIAS(0x8) | DEPTH_IS_Z);    /* .5 */
 
    if (irb != NULL) {
-      switch (irb->Base.Format) {
-      case MESA_FORMAT_ARGB8888:
-      case MESA_FORMAT_XRGB8888:
-	 value |= DV_PF_8888;
-	 break;
-      case MESA_FORMAT_RGB565:
-	 value |= DV_PF_565;
-	 break;
-      case MESA_FORMAT_ARGB1555:
-	 value |= DV_PF_1555;
-	 break;
-      case MESA_FORMAT_ARGB4444:
-	 value |= DV_PF_4444;
-	 break;
-      default:
-	 _mesa_problem(ctx, "Bad renderbuffer format: %d\n",
-		       irb->Base.Format);
-      }
+      value |= i830_render_target_format_for_mesa_format[irb->Base.Format];
    }
 
    if (depth_region && depth_region->cpp == 4) {
@@ -729,4 +728,5 @@ i830InitVtbl(struct i830_context *i830)
    i830->intel.vtbl.assert_not_dirty = i830_assert_not_dirty;
    i830->intel.vtbl.finish_batch = intel_finish_vb;
    i830->intel.vtbl.invalidate_state = i830_invalidate_state;
+   i830->intel.vtbl.render_target_supported = i830_render_target_supported;
 }
diff --git a/src/mesa/drivers/dri/i915/i915_tex_layout.c b/src/mesa/drivers/dri/i915/i915_tex_layout.c
index 6e4512129cd..f3a9fd4828d 100644
--- a/src/mesa/drivers/dri/i915/i915_tex_layout.c
+++ b/src/mesa/drivers/dri/i915/i915_tex_layout.c
@@ -127,7 +127,7 @@ i915_miptree_layout_cube(struct intel_context *intel,
    mt->total_width = dim * 2;
    mt->total_height = dim * 4;
 
-   for (level = mt->first_level; level <= mt->last_level; level++) {
+   for (level = 0; level < mt->levels; level++) {
       intel_miptree_set_level_info(mt, level, 6,
 				   0, 0,
 				   lvlWidth, lvlHeight,
@@ -141,12 +141,12 @@ i915_miptree_layout_cube(struct intel_context *intel,
       GLuint y = initial_offsets[face][1] * dim;
       GLuint d = dim;
 
-      for (level = mt->first_level; level <= mt->last_level; level++) {
+      for (level = 0; level < mt->levels; level++) {
 	 intel_miptree_set_image_offset(mt, level, face, x, y);
 
 	 if (d == 0)
-	    printf("cube mipmap %d/%d (%d..%d) is 0x0\n",
-		   face, level, mt->first_level, mt->last_level);
+	    printf("cube mipmap %d/%d (%d) is 0x0\n",
+		   face, level, mt->levels);
 
 	 d >>= 1;
 	 x += step_offsets[face][0] * d;
@@ -170,7 +170,7 @@ i915_miptree_layout_3d(struct intel_context *intel,
    mt->total_width = mt->width0;
 
    /* XXX: hardware expects/requires 9 levels at minimum. */
-   for (level = mt->first_level; level <= MAX2(8, mt->last_level); level++) {
+   for (level = 0; level < MAX2(9, mt->levels); level++) {
       intel_miptree_set_level_info(mt, level, depth, 0, mt->total_height,
 				   width, height, depth);
 
@@ -183,7 +183,7 @@ i915_miptree_layout_3d(struct intel_context *intel,
 
    /* Fixup depth image_offsets: */
    depth = mt->depth0;
-   for (level = mt->first_level; level <= mt->last_level; level++) {
+   for (level = 0; level < mt->levels; level++) {
       GLuint i;
       for (i = 0; i < depth; i++) {
 	 intel_miptree_set_image_offset(mt, level, i,
@@ -213,7 +213,7 @@ i915_miptree_layout_2d(struct intel_context *intel,
    mt->total_width = mt->width0;
    mt->total_height = 0;
 
-   for (level = mt->first_level; level <= mt->last_level; level++) {
+   for (level = 0; level < mt->levels; level++) {
       intel_miptree_set_level_info(mt, level, 1,
 				   0, mt->total_height,
 				   width, height, 1);
@@ -345,7 +345,7 @@ i945_miptree_layout_cube(struct intel_context *intel,
       mt->total_height = 4;
 
    /* Set all the levels to effectively occupy the whole rectangular region. */
-   for (level = mt->first_level; level <= mt->last_level; level++) {
+   for (level = 0; level < mt->levels; level++) {
       intel_miptree_set_level_info(mt, level, 6,
 				   0, 0,
 				   lvlWidth, lvlHeight, 1);
@@ -361,12 +361,12 @@ i945_miptree_layout_cube(struct intel_context *intel,
       if (dim == 4 && face >= 4) {
 	 y = mt->total_height - 4;
 	 x = (face - 4) * 8;
-      } else if (dim < 4 && (face > 0 || mt->first_level > 0)) {
+      } else if (dim < 4 && face > 0) {
 	 y = mt->total_height - 4;
 	 x = face * 8;
       }
 
-      for (level = mt->first_level; level <= mt->last_level; level++) {
+      for (level = 0; level < mt->levels; level++) {
 	 intel_miptree_set_image_offset(mt, level, face, x, y);
 
 	 d >>= 1;
@@ -429,7 +429,7 @@ i945_miptree_layout_3d(struct intel_context *intel,
    pack_x_pitch = mt->total_width;
    pack_x_nr = 1;
 
-   for (level = mt->first_level; level <= mt->last_level; level++) {
+   for (level = 0; level < mt->levels; level++) {
       GLint x = 0;
       GLint y = 0;
       GLint q, j;
diff --git a/src/mesa/drivers/dri/i915/i915_texstate.c b/src/mesa/drivers/dri/i915/i915_texstate.c
index c724a214967..af140c85f50 100644
--- a/src/mesa/drivers/dri/i915/i915_texstate.c
+++ b/src/mesa/drivers/dri/i915/i915_texstate.c
@@ -156,7 +156,7 @@ i915_update_tex_unit(struct intel_context *intel, GLuint unit, GLuint ss3)
    /* Get first image here, since intelObj->firstLevel will get set in
     * the intel_finalize_mipmap_tree() call above.
     */
-   firstImage = tObj->Image[0][intelObj->firstLevel];
+   firstImage = tObj->Image[0][tObj->BaseLevel];
 
    drm_intel_bo_reference(intelObj->mt->region->buffer);
    i915->state.tex_buffer[unit] = intelObj->mt->region->buffer;
diff --git a/src/mesa/drivers/dri/i915/i915_vtbl.c b/src/mesa/drivers/dri/i915/i915_vtbl.c
index 8d9020f5ef3..a94b9571275 100644
--- a/src/mesa/drivers/dri/i915/i915_vtbl.c
+++ b/src/mesa/drivers/dri/i915/i915_vtbl.c
@@ -436,15 +436,9 @@ i915_emit_state(struct intel_context *intel)
       OUT_BATCH((dirty & I915_UPLOAD_TEX_ALL) >> I915_UPLOAD_TEX_0_SHIFT);
       for (i = 0; i < I915_TEX_UNITS; i++)
          if (dirty & I915_UPLOAD_TEX(i)) {
-
-            if (state->tex_buffer[i]) {
-               OUT_RELOC(state->tex_buffer[i],
-			 I915_GEM_DOMAIN_SAMPLER, 0,
-                         state->tex_offset[i]);
-            }
-            else {
-               OUT_BATCH(state->tex_offset[i]);
-            }
+	    OUT_RELOC(state->tex_buffer[i],
+		      I915_GEM_DOMAIN_SAMPLER, 0,
+		      state->tex_offset[i]);
 
             OUT_BATCH(state->Tex[i][I915_TEXREG_MS3]);
             OUT_BATCH(state->Tex[i][I915_TEXREG_MS4]);
@@ -524,6 +518,27 @@ i915_set_buf_info_for_region(uint32_t *state, struct intel_region *region,
    }
 }
 
+static uint32_t i915_render_target_format_for_mesa_format[MESA_FORMAT_COUNT] =
+{
+   [MESA_FORMAT_ARGB8888] = DV_PF_8888,
+   [MESA_FORMAT_XRGB8888] = DV_PF_8888,
+   [MESA_FORMAT_RGB565] = DV_PF_565 | DITHER_FULL_ALWAYS,
+   [MESA_FORMAT_ARGB1555] = DV_PF_1555 | DITHER_FULL_ALWAYS,
+   [MESA_FORMAT_ARGB4444] = DV_PF_4444 | DITHER_FULL_ALWAYS,
+};
+
+static bool
+i915_render_target_supported(gl_format format)
+{
+   if (format == MESA_FORMAT_S8_Z24 ||
+       format == MESA_FORMAT_X8_Z24 ||
+       format == MESA_FORMAT_Z16) {
+      return true;
+   }
+
+   return i915_render_target_format_for_mesa_format[format] != 0;
+}
+
 static void
 i915_set_draw_region(struct intel_context *intel,
                      struct intel_region *color_regions[],
@@ -563,24 +578,7 @@ i915_set_draw_region(struct intel_context *intel,
             DSTORG_VERT_BIAS(0x8) |     /* .5 */
             LOD_PRECLAMP_OGL | TEX_DEFAULT_COLOR_OGL);
    if (irb != NULL) {
-      switch (irb->Base.Format) {
-      case MESA_FORMAT_ARGB8888:
-      case MESA_FORMAT_XRGB8888:
-	 value |= DV_PF_8888;
-	 break;
-      case MESA_FORMAT_RGB565:
-	 value |= DV_PF_565 | DITHER_FULL_ALWAYS;
-	 break;
-      case MESA_FORMAT_ARGB1555:
-	 value |= DV_PF_1555 | DITHER_FULL_ALWAYS;
-	 break;
-      case MESA_FORMAT_ARGB4444:
-	 value |= DV_PF_4444 | DITHER_FULL_ALWAYS;
-	 break;
-      default:
-	 _mesa_problem(ctx, "Bad renderbuffer format: %d\n",
-		       irb->Base.Format);
-      }
+      value |= i915_render_target_format_for_mesa_format[irb->Base.Format];
    }
 
    /* This isn't quite safe, thus being hidden behind an option.  When changing
@@ -687,4 +685,5 @@ i915InitVtbl(struct i915_context *i915)
    i915->intel.vtbl.update_texture_state = i915UpdateTextureState;
    i915->intel.vtbl.assert_not_dirty = i915_assert_not_dirty;
    i915->intel.vtbl.finish_batch = intel_finish_vb;
+   i915->intel.vtbl.render_target_supported = i915_render_target_supported;
 }
diff --git a/src/mesa/drivers/dri/i965/brw_context.c b/src/mesa/drivers/dri/i965/brw_context.c
index 28549f2574a..8fc322fd82e 100644
--- a/src/mesa/drivers/dri/i965/brw_context.c
+++ b/src/mesa/drivers/dri/i965/brw_context.c
@@ -151,6 +151,13 @@ GLboolean brwCreateContext( int api,
       MIN2(ctx->Const.FragmentProgram.MaxNativeParameters,
 	   ctx->Const.FragmentProgram.MaxEnvParams);
 
+   /* Gen6 converts quads to polygon in beginning of 3D pipeline,
+      but we're not sure how it's actually done for vertex order,
+      that affect provoking vertex decision. Always use last vertex
+      convention for quad primitive which works as expected for now. */
+   if (intel->gen == 6)
+       ctx->Const.QuadsFollowProvokingVertexConvention = GL_FALSE;
+
    if (intel->is_g4x || intel->gen >= 5) {
       brw->CMD_VF_STATISTICS = CMD_VF_STATISTICS_GM45;
       brw->CMD_PIPELINE_SELECT = CMD_PIPELINE_SELECT_GM45;
diff --git a/src/mesa/drivers/dri/i965/brw_defines.h b/src/mesa/drivers/dri/i965/brw_defines.h
index b48a30d6be9..ea6ac340482 100644
--- a/src/mesa/drivers/dri/i965/brw_defines.h
+++ b/src/mesa/drivers/dri/i965/brw_defines.h
@@ -389,6 +389,7 @@
 #define BRW_SURFACEFORMAT_R8_SSCALED                     0x149
 #define BRW_SURFACEFORMAT_R8_USCALED                     0x14A
 #define BRW_SURFACEFORMAT_L8_UNORM_SRGB                  0x14C
+#define BRW_SURFACEFORMAT_DXT1_RGB_SRGB                  0x180
 #define BRW_SURFACEFORMAT_R1_UINT                        0x181 
 #define BRW_SURFACEFORMAT_YCRCB_NORMAL                   0x182 
 #define BRW_SURFACEFORMAT_YCRCB_SWAPUVY                  0x183 
@@ -844,7 +845,7 @@
 # define GEN6_BINDING_TABLE_MODIFY_GS	(1 << 9)
 # define GEN6_BINDING_TABLE_MODIFY_PS	(1 << 12)
 
-#define CMD_3D_SAMPLER_STATE_POINTERS			0x7802 /* SNB+ */
+#define _3DSTATE_SAMPLER_STATE_POINTERS		0x7802 /* GEN6+ */
 # define PS_SAMPLER_STATE_CHANGE				(1 << 12)
 # define GS_SAMPLER_STATE_CHANGE				(1 << 9)
 # define VS_SAMPLER_STATE_CHANGE				(1 << 8)
@@ -885,22 +886,22 @@
 #define CMD_INDEX_BUFFER              0x780a
 #define CMD_VF_STATISTICS_965          0x780b
 #define CMD_VF_STATISTICS_GM45        0x680b
-#define CMD_3D_CC_STATE_POINTERS      0x780e /* GEN6+ */
+#define _3DSTATE_CC_STATE_POINTERS		0x780e /* GEN6+ */
 
-#define CMD_URB					0x7805 /* GEN6+ */
+#define _3DSTATE_URB				0x7805 /* GEN6+ */
 # define GEN6_URB_VS_SIZE_SHIFT				16
 # define GEN6_URB_VS_ENTRIES_SHIFT			0
 # define GEN6_URB_GS_ENTRIES_SHIFT			8
 # define GEN6_URB_GS_SIZE_SHIFT				0
 
-#define CMD_VIEWPORT_STATE_POINTERS			0x780d /* GEN6+ */
+#define _3DSTATE_VIEWPORT_STATE_POINTERS	0x780d /* GEN6+ */
 # define GEN6_CC_VIEWPORT_MODIFY			(1 << 12)
 # define GEN6_SF_VIEWPORT_MODIFY			(1 << 11)
 # define GEN6_CLIP_VIEWPORT_MODIFY			(1 << 10)
 
-#define CMD_3D_SCISSOR_STATE_POINTERS		0x780f /* GEN6+ */
+#define _3DSTATE_SCISSOR_STATE_POINTERS		0x780f /* GEN6+ */
 
-#define CMD_3D_VS_STATE		      0x7810 /* GEN6+ */
+#define _3DSTATE_VS				0x7810 /* GEN6+ */
 /* DW2 */
 # define GEN6_VS_SPF_MODE				(1 << 31)
 # define GEN6_VS_VECTOR_MASK_ENABLE			(1 << 30)
@@ -918,7 +919,7 @@
 # define GEN6_VS_CACHE_DISABLE				(1 << 1)
 # define GEN6_VS_ENABLE					(1 << 0)
 
-#define CMD_3D_GS_STATE		      0x7811 /* GEN6+ */
+#define _3DSTATE_GS		      		0x7811 /* GEN6+ */
 /* DW2 */
 # define GEN6_GS_SPF_MODE				(1 << 31)
 # define GEN6_GS_VECTOR_MASK_ENABLE			(1 << 30)
@@ -936,7 +937,7 @@
 /* DW6 */
 # define GEN6_GS_ENABLE					(1 << 15)
 
-#define CMD_3D_CLIP_STATE		      0x7812 /* GEN6+ */
+#define _3DSTATE_CLIP				0x7812 /* GEN6+ */
 /* DW1 */
 # define GEN6_CLIP_STATISTICS_ENABLE			(1 << 10)
 /**
@@ -966,7 +967,7 @@
 # define GEN6_CLIP_MAX_POINT_WIDTH_SHIFT		6
 # define GEN6_CLIP_FORCE_ZERO_RTAINDEX			(1 << 5)
 
-#define CMD_3D_SF_STATE				0x7813 /* GEN6+ */
+#define _3DSTATE_SF				0x7813 /* GEN6+ */
 /* DW1 */
 # define GEN6_SF_NUM_OUTPUTS_SHIFT			22
 # define GEN6_SF_SWIZZLE_ENABLE				(1 << 21)
@@ -1043,7 +1044,7 @@
 /* DW18: attr 0-7 wrap shortest enables */
 /* DW19: attr 8-16 wrap shortest enables */
 
-#define CMD_3D_WM_STATE		      0x7814 /* GEN6+ */
+#define _3DSTATE_WM				0x7814 /* GEN6+ */
 /* DW1: kernel pointer */
 /* DW2 */
 # define GEN6_WM_SPF_MODE				(1 << 31)
@@ -1106,15 +1107,15 @@
 /* DW7: kernel 1 pointer */
 /* DW8: kernel 2 pointer */
 
-#define CMD_3D_CONSTANT_VS_STATE	      0x7815 /* GEN6+ */
-#define CMD_3D_CONSTANT_GS_STATE	      0x7816 /* GEN6+ */
-#define CMD_3D_CONSTANT_PS_STATE	      0x7817 /* GEN6+ */
+#define _3DSTATE_CONSTANT_VS		      0x7815 /* GEN6+ */
+#define _3DSTATE_CONSTANT_GS		      0x7816 /* GEN6+ */
+#define _3DSTATE_CONSTANT_PS		      0x7817 /* GEN6+ */
 # define GEN6_CONSTANT_BUFFER_3_ENABLE			(1 << 15)
 # define GEN6_CONSTANT_BUFFER_2_ENABLE			(1 << 14)
 # define GEN6_CONSTANT_BUFFER_1_ENABLE			(1 << 13)
 # define GEN6_CONSTANT_BUFFER_0_ENABLE			(1 << 12)
 
-#define CMD_3D_SAMPLE_MASK			0x7818 /* GEN6+ */
+#define _3DSTATE_SAMPLE_MASK			0x7818 /* GEN6+ */
 
 #define CMD_DRAW_RECT                 0x7900
 #define CMD_BLEND_CONSTANT_COLOR      0x7901
@@ -1133,7 +1134,7 @@
 /* DW2: SVB index */
 /* DW3: SVB maximum index */
 
-#define CMD_3D_MULTISAMPLE			0x790d /* SNB+ */
+#define _3DSTATE_MULTISAMPLE			0x790d /* GEN6+ */
 /* DW1 */
 # define MS_PIXEL_LOCATION_CENTER			(0 << 4)
 # define MS_PIXEL_LOCATION_UPPER_LEFT			(1 << 4)
diff --git a/src/mesa/drivers/dri/i965/brw_disasm.c b/src/mesa/drivers/dri/i965/brw_disasm.c
index 6b61f7af15d..111cb9974e1 100644
--- a/src/mesa/drivers/dri/i965/brw_disasm.c
+++ b/src/mesa/drivers/dri/i965/brw_disasm.c
@@ -973,7 +973,7 @@ int brw_disasm (FILE *file, struct brw_instruction *inst, int gen)
 			inst->bits3.dp_render_cache.send_commit_msg,
 			inst->bits3.dp_render_cache.msg_length,
 			inst->bits3.dp_render_cache.response_length);
-	    } else if (gen >= 5) {
+	    } else if (gen >= 5 /* FINISHME: || is_g4x */) {
 		format (file, " (%d, %d, %d)",
 			inst->bits3.dp_read_gen5.binding_table_index,
 			inst->bits3.dp_read_gen5.msg_control,
diff --git a/src/mesa/drivers/dri/i965/brw_eu_emit.c b/src/mesa/drivers/dri/i965/brw_eu_emit.c
index 6d48ca0e46d..88131c432ec 100644
--- a/src/mesa/drivers/dri/i965/brw_eu_emit.c
+++ b/src/mesa/drivers/dri/i965/brw_eu_emit.c
@@ -536,6 +536,16 @@ brw_set_dp_read_message(struct brw_context *brw,
        insn->bits3.dp_read_gen5.end_of_thread = 0;
        insn->bits2.send_gen5.sfid = BRW_MESSAGE_TARGET_DATAPORT_READ;
        insn->bits2.send_gen5.end_of_thread = 0;
+   } else if (intel->is_g4x) {
+       insn->bits3.dp_read_g4x.binding_table_index = binding_table_index; /*0:7*/
+       insn->bits3.dp_read_g4x.msg_control = msg_control;  /*8:10*/
+       insn->bits3.dp_read_g4x.msg_type = msg_type;  /*11:13*/
+       insn->bits3.dp_read_g4x.target_cache = target_cache;  /*14:15*/
+       insn->bits3.dp_read_g4x.response_length = response_length;  /*16:19*/
+       insn->bits3.dp_read_g4x.msg_length = msg_length;  /*20:23*/
+       insn->bits3.dp_read_g4x.msg_target = BRW_MESSAGE_TARGET_DATAPORT_READ; /*24:27*/
+       insn->bits3.dp_read_g4x.pad1 = 0;
+       insn->bits3.dp_read_g4x.end_of_thread = 0;
    } else {
        insn->bits3.dp_read.binding_table_index = binding_table_index; /*0:7*/
        insn->bits3.dp_read.msg_control = msg_control;  /*8:11*/
@@ -1717,6 +1727,7 @@ void brw_dp_READ_4_vs(struct brw_compile *p,
 
    /* Setup MRF[1] with location/offset into const buffer */
    brw_push_insn_state(p);
+   brw_set_access_mode(p, BRW_ALIGN_1);
    brw_set_compression_control(p, BRW_COMPRESSION_NONE);
    brw_set_mask_control(p, BRW_MASK_DISABLE);
    brw_set_predicate_control(p, BRW_PREDICATE_NONE);
@@ -1764,6 +1775,7 @@ void brw_dp_READ_4_vs_relative(struct brw_compile *p,
 
    /* Setup MRF[1] with offset into const buffer */
    brw_push_insn_state(p);
+   brw_set_access_mode(p, BRW_ALIGN_1);
    brw_set_compression_control(p, BRW_COMPRESSION_NONE);
    brw_set_mask_control(p, BRW_MASK_DISABLE);
    brw_set_predicate_control(p, BRW_PREDICATE_NONE);
diff --git a/src/mesa/drivers/dri/i965/brw_fallback.c b/src/mesa/drivers/dri/i965/brw_fallback.c
index 6796fb208dc..7262cf69582 100644
--- a/src/mesa/drivers/dri/i965/brw_fallback.c
+++ b/src/mesa/drivers/dri/i965/brw_fallback.c
@@ -63,8 +63,8 @@ static GLboolean do_check_fallback(struct brw_context *brw)
    for (i = 0; i < BRW_MAX_TEX_UNIT; i++) {
       struct gl_texture_unit *texUnit = &ctx->Texture.Unit[i];
       if (texUnit->_ReallyEnabled) {
-	 struct intel_texture_object *intelObj = intel_texture_object(texUnit->_Current);
-	 struct gl_texture_image *texImage = intelObj->base.Image[0][intelObj->firstLevel];
+	 struct gl_texture_object *tex_obj = texUnit->_Current;
+	 struct gl_texture_image *texImage = tex_obj->Image[0][tex_obj->BaseLevel];
 	 if (texImage->Border) {
 	    DBG("FALLBACK: texture border\n");
 	    return GL_TRUE;
diff --git a/src/mesa/drivers/dri/i965/brw_fs.cpp b/src/mesa/drivers/dri/i965/brw_fs.cpp
index 2de81b28371..22e6e2e7368 100644
--- a/src/mesa/drivers/dri/i965/brw_fs.cpp
+++ b/src/mesa/drivers/dri/i965/brw_fs.cpp
@@ -89,6 +89,9 @@ brw_compile_shader(struct gl_context *ctx, struct gl_shader *shader)
 GLboolean
 brw_link_shader(struct gl_context *ctx, struct gl_shader_program *prog)
 {
+   struct brw_context *brw = brw_context(ctx);
+   struct intel_context *intel = &brw->intel;
+
    struct brw_shader *shader =
       (struct brw_shader *)prog->_LinkedShaders[MESA_SHADER_FRAGMENT];
    if (shader != NULL) {
@@ -107,7 +110,15 @@ brw_link_shader(struct gl_context *ctx, struct gl_shader_program *prog)
 			 SUB_TO_ADD_NEG |
 			 EXP_TO_EXP2 |
 			 LOG_TO_LOG2);
+
+      /* Pre-gen6 HW can only nest if-statements 16 deep.  Beyond this,
+       * if-statements need to be flattened.
+       */
+      if (intel->gen < 6)
+	 lower_if_to_cond_assign(shader->ir, 16);
+
       do_lower_texture_projection(shader->ir);
+      do_vec_index_to_cond_assign(shader->ir);
       brw_do_cubemap_normalize(shader->ir);
 
       do {
@@ -775,6 +786,30 @@ fs_visitor::try_emit_saturate(ir_expression *ir)
    return true;
 }
 
+static uint32_t
+brw_conditional_for_comparison(unsigned int op)
+{
+   switch (op) {
+   case ir_binop_less:
+      return BRW_CONDITIONAL_L;
+   case ir_binop_greater:
+      return BRW_CONDITIONAL_G;
+   case ir_binop_lequal:
+      return BRW_CONDITIONAL_LE;
+   case ir_binop_gequal:
+      return BRW_CONDITIONAL_GE;
+   case ir_binop_equal:
+   case ir_binop_all_equal: /* same as equal for scalars */
+      return BRW_CONDITIONAL_Z;
+   case ir_binop_nequal:
+   case ir_binop_any_nequal: /* same as nequal for scalars */
+      return BRW_CONDITIONAL_NZ;
+   default:
+      assert(!"not reached: bad operation for comparison");
+      return BRW_CONDITIONAL_NZ;
+   }
+}
+
 void
 fs_visitor::visit(ir_expression *ir)
 {
@@ -890,35 +925,20 @@ fs_visitor::visit(ir_expression *ir)
       break;
 
    case ir_binop_less:
-      inst = emit(fs_inst(BRW_OPCODE_CMP, this->result, op[0], op[1]));
-      inst->conditional_mod = BRW_CONDITIONAL_L;
-      emit(fs_inst(BRW_OPCODE_AND, this->result, this->result, fs_reg(0x1)));
-      break;
    case ir_binop_greater:
-      inst = emit(fs_inst(BRW_OPCODE_CMP, this->result, op[0], op[1]));
-      inst->conditional_mod = BRW_CONDITIONAL_G;
-      emit(fs_inst(BRW_OPCODE_AND, this->result, this->result, fs_reg(0x1)));
-      break;
    case ir_binop_lequal:
-      inst = emit(fs_inst(BRW_OPCODE_CMP, this->result, op[0], op[1]));
-      inst->conditional_mod = BRW_CONDITIONAL_LE;
-      emit(fs_inst(BRW_OPCODE_AND, this->result, this->result, fs_reg(0x1)));
-      break;
    case ir_binop_gequal:
-      inst = emit(fs_inst(BRW_OPCODE_CMP, this->result, op[0], op[1]));
-      inst->conditional_mod = BRW_CONDITIONAL_GE;
-      emit(fs_inst(BRW_OPCODE_AND, this->result, this->result, fs_reg(0x1)));
-      break;
    case ir_binop_equal:
-   case ir_binop_all_equal: /* same as nequal for scalars */
-      inst = emit(fs_inst(BRW_OPCODE_CMP, this->result, op[0], op[1]));
-      inst->conditional_mod = BRW_CONDITIONAL_Z;
-      emit(fs_inst(BRW_OPCODE_AND, this->result, this->result, fs_reg(0x1)));
-      break;
+   case ir_binop_all_equal:
    case ir_binop_nequal:
-   case ir_binop_any_nequal: /* same as nequal for scalars */
-      inst = emit(fs_inst(BRW_OPCODE_CMP, this->result, op[0], op[1]));
-      inst->conditional_mod = BRW_CONDITIONAL_NZ;
+   case ir_binop_any_nequal:
+      temp = this->result;
+      /* original gen4 does implicit conversion before comparison. */
+      if (intel->gen < 5)
+	 temp.type = op[0].type;
+
+      inst = emit(fs_inst(BRW_OPCODE_CMP, temp, op[0], op[1]));
+      inst->conditional_mod = brw_conditional_for_comparison(ir->operation);
       emit(fs_inst(BRW_OPCODE_AND, this->result, this->result, fs_reg(0x1)));
       break;
 
@@ -963,7 +983,12 @@ fs_visitor::visit(ir_expression *ir)
       break;
    case ir_unop_f2b:
    case ir_unop_i2b:
-      inst = emit(fs_inst(BRW_OPCODE_CMP, this->result, op[0], fs_reg(0.0f)));
+      temp = this->result;
+      /* original gen4 does implicit conversion before comparison. */
+      if (intel->gen < 5)
+	 temp.type = op[0].type;
+
+      inst = emit(fs_inst(BRW_OPCODE_CMP, temp, op[0], fs_reg(0.0f)));
       inst->conditional_mod = BRW_CONDITIONAL_NZ;
       inst = emit(fs_inst(BRW_OPCODE_AND, this->result,
 			  this->result, fs_reg(1)));
@@ -1546,7 +1571,7 @@ fs_visitor::emit_bool_to_cond_code(ir_rvalue *ir)
 	    inst = emit(fs_inst(BRW_OPCODE_CMP, reg_null_d,
 				op[0], fs_reg(0.0f)));
 	 } else {
-	    inst = emit(fs_inst(BRW_OPCODE_MOV, reg_null_d, op[0]));
+	    inst = emit(fs_inst(BRW_OPCODE_MOV, reg_null_f, op[0]));
 	 }
 	 inst->conditional_mod = BRW_CONDITIONAL_NZ;
 	 break;
@@ -1561,31 +1586,18 @@ fs_visitor::emit_bool_to_cond_code(ir_rvalue *ir)
 	 break;
 
       case ir_binop_greater:
-	 inst = emit(fs_inst(BRW_OPCODE_CMP, reg_null_d, op[0], op[1]));
-	 inst->conditional_mod = BRW_CONDITIONAL_G;
-	 break;
       case ir_binop_gequal:
-	 inst = emit(fs_inst(BRW_OPCODE_CMP, reg_null_d, op[0], op[1]));
-	 inst->conditional_mod = BRW_CONDITIONAL_GE;
-	 break;
       case ir_binop_less:
-	 inst = emit(fs_inst(BRW_OPCODE_CMP, reg_null_d, op[0], op[1]));
-	 inst->conditional_mod = BRW_CONDITIONAL_L;
-	 break;
       case ir_binop_lequal:
-	 inst = emit(fs_inst(BRW_OPCODE_CMP, reg_null_d, op[0], op[1]));
-	 inst->conditional_mod = BRW_CONDITIONAL_LE;
-	 break;
       case ir_binop_equal:
       case ir_binop_all_equal:
-	 inst = emit(fs_inst(BRW_OPCODE_CMP, reg_null_d, op[0], op[1]));
-	 inst->conditional_mod = BRW_CONDITIONAL_Z;
-	 break;
       case ir_binop_nequal:
       case ir_binop_any_nequal:
-	 inst = emit(fs_inst(BRW_OPCODE_CMP, reg_null_d, op[0], op[1]));
-	 inst->conditional_mod = BRW_CONDITIONAL_NZ;
+	 inst = emit(fs_inst(BRW_OPCODE_CMP, reg_null_cmp, op[0], op[1]));
+	 inst->conditional_mod =
+	    brw_conditional_for_comparison(expr->operation);
 	 break;
+
       default:
 	 assert(!"not reached");
 	 this->fail = true;
@@ -1664,30 +1676,16 @@ fs_visitor::emit_if_gen6(ir_if *ir)
 	 return;
 
       case ir_binop_greater:
-	 inst = emit(fs_inst(BRW_OPCODE_IF, reg_null_d, op[0], op[1]));
-	 inst->conditional_mod = BRW_CONDITIONAL_G;
-	 return;
       case ir_binop_gequal:
-	 inst = emit(fs_inst(BRW_OPCODE_IF, reg_null_d, op[0], op[1]));
-	 inst->conditional_mod = BRW_CONDITIONAL_GE;
-	 return;
       case ir_binop_less:
-	 inst = emit(fs_inst(BRW_OPCODE_IF, reg_null_d, op[0], op[1]));
-	 inst->conditional_mod = BRW_CONDITIONAL_L;
-	 return;
       case ir_binop_lequal:
-	 inst = emit(fs_inst(BRW_OPCODE_IF, reg_null_d, op[0], op[1]));
-	 inst->conditional_mod = BRW_CONDITIONAL_LE;
-	 return;
       case ir_binop_equal:
       case ir_binop_all_equal:
-	 inst = emit(fs_inst(BRW_OPCODE_IF, reg_null_d, op[0], op[1]));
-	 inst->conditional_mod = BRW_CONDITIONAL_Z;
-	 return;
       case ir_binop_nequal:
       case ir_binop_any_nequal:
 	 inst = emit(fs_inst(BRW_OPCODE_IF, reg_null_d, op[0], op[1]));
-	 inst->conditional_mod = BRW_CONDITIONAL_NZ;
+	 inst->conditional_mod =
+	    brw_conditional_for_comparison(expr->operation);
 	 return;
       default:
 	 assert(!"not reached");
@@ -1769,32 +1767,9 @@ fs_visitor::visit(ir_loop *ir)
       this->base_ir = ir->to;
       ir->to->accept(this);
 
-      fs_inst *inst = emit(fs_inst(BRW_OPCODE_CMP, reg_null_d,
+      fs_inst *inst = emit(fs_inst(BRW_OPCODE_CMP, reg_null_cmp,
 				   counter, this->result));
-      switch (ir->cmp) {
-      case ir_binop_equal:
-	 inst->conditional_mod = BRW_CONDITIONAL_Z;
-	 break;
-      case ir_binop_nequal:
-	 inst->conditional_mod = BRW_CONDITIONAL_NZ;
-	 break;
-      case ir_binop_gequal:
-	 inst->conditional_mod = BRW_CONDITIONAL_GE;
-	 break;
-      case ir_binop_lequal:
-	 inst->conditional_mod = BRW_CONDITIONAL_LE;
-	 break;
-      case ir_binop_greater:
-	 inst->conditional_mod = BRW_CONDITIONAL_G;
-	 break;
-      case ir_binop_less:
-	 inst->conditional_mod = BRW_CONDITIONAL_L;
-	 break;
-      default:
-	 assert(!"not reached: unknown loop condition");
-	 this->fail = true;
-	 break;
-      }
+      inst->conditional_mod = brw_conditional_for_comparison(ir->cmp);
 
       inst = emit(fs_inst(BRW_OPCODE_BREAK));
       inst->predicated = true;
diff --git a/src/mesa/drivers/dri/i965/brw_fs.h b/src/mesa/drivers/dri/i965/brw_fs.h
index de7b15312a5..00a000855c5 100644
--- a/src/mesa/drivers/dri/i965/brw_fs.h
+++ b/src/mesa/drivers/dri/i965/brw_fs.h
@@ -348,6 +348,23 @@ public:
 					  hash_table_pointer_hash,
 					  hash_table_pointer_compare);
 
+      /* There's a question that appears to be left open in the spec:
+       * How do implicit dst conversions interact with the CMP
+       * instruction or conditional mods?  On gen6, the instruction:
+       *
+       * CMP null<d> src0<f> src1<f>
+       *
+       * will do src1 - src0 and compare that result as if it was an
+       * integer.  On gen4, it will do src1 - src0 as float, convert
+       * the result to int, and compare as int.  In between, it
+       * appears that it does src1 - src0 and does the compare in the
+       * execution type so dst type doesn't matter.
+       */
+      if (this->intel->gen > 4)
+	 this->reg_null_cmp = reg_null_d;
+      else
+	 this->reg_null_cmp = reg_null_f;
+
       this->frag_color = NULL;
       this->frag_data = NULL;
       this->frag_depth = NULL;
@@ -485,6 +502,7 @@ public:
    fs_reg pixel_w;
    fs_reg delta_x;
    fs_reg delta_y;
+   fs_reg reg_null_cmp;
 
    int grf_used;
 };
diff --git a/src/mesa/drivers/dri/i965/brw_gs.c b/src/mesa/drivers/dri/i965/brw_gs.c
index 73b41fdbcef..70c451d071d 100644
--- a/src/mesa/drivers/dri/i965/brw_gs.c
+++ b/src/mesa/drivers/dri/i965/brw_gs.c
@@ -96,6 +96,9 @@ static void compile_gs_prog( struct brw_context *brw,
       brw_gs_quad_strip( &c, key );
       break;
    case GL_LINE_LOOP:
+      /* Gen6: LINELOOP is converted to LINESTRIP at the beginning of the 3D pipeline */
+      if (intel->gen == 6)
+          return;
       brw_gs_lines( &c );
       break;
    case GL_LINES:
@@ -189,7 +192,7 @@ static void populate_key( struct brw_context *brw,
    }
 
    if (intel->gen == 6)
-       prim_gs_always = brw->primitive == GL_LINE_LOOP;
+       prim_gs_always = 0;
    else
        prim_gs_always = brw->primitive == GL_QUADS ||
 			brw->primitive == GL_QUAD_STRIP ||
diff --git a/src/mesa/drivers/dri/i965/brw_misc_state.c b/src/mesa/drivers/dri/i965/brw_misc_state.c
index a91b0528fac..c49ebe64a60 100644
--- a/src/mesa/drivers/dri/i965/brw_misc_state.c
+++ b/src/mesa/drivers/dri/i965/brw_misc_state.c
@@ -492,14 +492,14 @@ static void upload_invarient_state( struct brw_context *brw )
       int i;
 
       BEGIN_BATCH(3);
-      OUT_BATCH(CMD_3D_MULTISAMPLE << 16 | (3 - 2));
+      OUT_BATCH(_3DSTATE_MULTISAMPLE << 16 | (3 - 2));
       OUT_BATCH(MS_PIXEL_LOCATION_CENTER |
 		MS_NUMSAMPLES_1);
       OUT_BATCH(0); /* positions for 4/8-sample */
       ADVANCE_BATCH();
 
       BEGIN_BATCH(2);
-      OUT_BATCH(CMD_3D_SAMPLE_MASK << 16 | (2 - 2));
+      OUT_BATCH(_3DSTATE_SAMPLE_MASK << 16 | (2 - 2));
       OUT_BATCH(1);
       ADVANCE_BATCH();
 
diff --git a/src/mesa/drivers/dri/i965/brw_queryobj.c b/src/mesa/drivers/dri/i965/brw_queryobj.c
index f28f28663ea..656aad630a1 100644
--- a/src/mesa/drivers/dri/i965/brw_queryobj.c
+++ b/src/mesa/drivers/dri/i965/brw_queryobj.c
@@ -232,6 +232,12 @@ brw_prepare_query_begin(struct brw_context *brw)
       brw->query.bo = NULL;
 
       brw->query.bo = drm_intel_bo_alloc(intel->bufmgr, "query", 4096, 1);
+
+      /* clear target buffer */
+      drm_intel_bo_map(brw->query.bo, GL_TRUE);
+      memset((char *)brw->query.bo->virtual, 0, 4096);
+      drm_intel_bo_unmap(brw->query.bo);
+
       brw->query.index = 0;
    }
 
diff --git a/src/mesa/drivers/dri/i965/brw_state_cache.c b/src/mesa/drivers/dri/i965/brw_state_cache.c
index 58ff528d44b..7045888ad4a 100644
--- a/src/mesa/drivers/dri/i965/brw_state_cache.c
+++ b/src/mesa/drivers/dri/i965/brw_state_cache.c
@@ -58,8 +58,6 @@
 
 #include "main/imports.h"
 #include "brw_state.h"
-#include "intel_batchbuffer.h"
-#include "brw_wm.h"
 
 #define FILE_DEBUG_FLAG DEBUG_STATE
 
diff --git a/src/mesa/drivers/dri/i965/brw_structs.h b/src/mesa/drivers/dri/i965/brw_structs.h
index 461f27048cc..8b6646ca513 100644
--- a/src/mesa/drivers/dri/i965/brw_structs.h
+++ b/src/mesa/drivers/dri/i965/brw_structs.h
@@ -1665,6 +1665,18 @@ struct brw_instruction
 
       struct {
 	 GLuint binding_table_index:8;
+	 GLuint msg_control:3;
+	 GLuint msg_type:3;
+	 GLuint target_cache:2;
+	 GLuint response_length:4;
+	 GLuint msg_length:4;
+	 GLuint msg_target:4;
+	 GLuint pad1:3;
+	 GLuint end_of_thread:1;
+      } dp_read_g4x;
+
+      struct {
+	 GLuint binding_table_index:8;
 	 GLuint msg_control:3;  
 	 GLuint msg_type:3;  
 	 GLuint target_cache:2;    
diff --git a/src/mesa/drivers/dri/i965/brw_tex_layout.c b/src/mesa/drivers/dri/i965/brw_tex_layout.c
index 9ac0713a1d3..66adc49cc94 100644
--- a/src/mesa/drivers/dri/i965/brw_tex_layout.c
+++ b/src/mesa/drivers/dri/i965/brw_tex_layout.c
@@ -67,7 +67,7 @@ GLboolean brw_miptree_layout(struct intel_context *intel,
 
 	  i945_miptree_layout_2d(intel, mt, tiling, 6);
 
-          for (level = mt->first_level; level <= mt->last_level; level++) {
+          for (level = 0; level < mt->levels; level++) {
 	     for (q = 0; q < 6; q++) {
 		intel_miptree_set_image_offset(mt, level, q, 0, q * qpitch);
 	     }
@@ -101,7 +101,7 @@ GLboolean brw_miptree_layout(struct intel_context *intel,
       pack_x_pitch = width;
       pack_x_nr = 1;
 
-      for (level = mt->first_level ; level <= mt->last_level ; level++) {
+      for (level = 0; level < mt->levels; level++) {
 	 GLuint nr_images = mt->target == GL_TEXTURE_3D ? depth : 6;
 	 GLint x = 0;
 	 GLint y = 0;
diff --git a/src/mesa/drivers/dri/i965/brw_vs_emit.c b/src/mesa/drivers/dri/i965/brw_vs_emit.c
index 326bb1e562f..fe9737d043a 100644
--- a/src/mesa/drivers/dri/i965/brw_vs_emit.c
+++ b/src/mesa/drivers/dri/i965/brw_vs_emit.c
@@ -1136,7 +1136,7 @@ get_constant(struct brw_vs_compile *c,
    }
 
    /* replicate lower four floats into upper half (to get XYZWXYZW) */
-   const_reg = stride(const_reg, 0, 4, 0);
+   const_reg = stride(const_reg, 0, 4, 1);
    const_reg.subnr = 0;
 
    return const_reg;
@@ -1426,11 +1426,10 @@ static struct brw_reg get_arg( struct brw_vs_compile *c,
 					  GET_SWZ(src->Swizzle, 1),
 					  GET_SWZ(src->Swizzle, 2),
 					  GET_SWZ(src->Swizzle, 3));
-   }
 
-   /* Note this is ok for non-swizzle instructions: 
-    */
-   reg.negate = src->Negate ? 1 : 0;   
+      /* Note this is ok for non-swizzle ARB_vp instructions */
+      reg.negate = src->Negate ? 1 : 0;
+   }
 
    return reg;
 }
diff --git a/src/mesa/drivers/dri/i965/brw_vs_surface_state.c b/src/mesa/drivers/dri/i965/brw_vs_surface_state.c
index eabac511602..b0b05445eb9 100644
--- a/src/mesa/drivers/dri/i965/brw_vs_surface_state.c
+++ b/src/mesa/drivers/dri/i965/brw_vs_surface_state.c
@@ -82,6 +82,15 @@ prepare_vs_constants(struct brw_context *brw)
 	     params->ParameterValues[i],
 	     4 * sizeof(float));
    }
+
+   if (0) {
+      for (i = 0; i < params->NumParameters; i++) {
+	 float *row = (float *)brw->vs.const_bo->virtual + i * 4;
+	 printf("vs const surface %3d: %4.3f %4.3f %4.3f %4.3f\n",
+		i, row[0], row[1], row[2], row[3]);
+      }
+   }
+
    drm_intel_gem_bo_unmap_gtt(brw->vs.const_bo);
    brw->state.dirty.brw |= BRW_NEW_VS_CONSTBUF;
 }
diff --git a/src/mesa/drivers/dri/i965/brw_vtbl.c b/src/mesa/drivers/dri/i965/brw_vtbl.c
index 3d7a98c9812..100a21b59d7 100644
--- a/src/mesa/drivers/dri/i965/brw_vtbl.c
+++ b/src/mesa/drivers/dri/i965/brw_vtbl.c
@@ -203,4 +203,5 @@ void brwInitVtbl( struct brw_context *brw )
    brw->intel.vtbl.destroy = brw_destroy_context;
    brw->intel.vtbl.set_draw_region = brw_set_draw_region;
    brw->intel.vtbl.debug_batch = brw_debug_batch;
+   brw->intel.vtbl.render_target_supported = brw_render_target_supported;
 }
diff --git a/src/mesa/drivers/dri/i965/brw_wm.h b/src/mesa/drivers/dri/i965/brw_wm.h
index e7f3cfbb75f..d9cae75ab5b 100644
--- a/src/mesa/drivers/dri/i965/brw_wm.h
+++ b/src/mesa/drivers/dri/i965/brw_wm.h
@@ -474,5 +474,6 @@ struct gl_shader *brw_new_shader(struct gl_context *ctx, GLuint name, GLuint typ
 struct gl_shader_program *brw_new_shader_program(struct gl_context *ctx, GLuint name);
 
 bool brw_color_buffer_write_enabled(struct brw_context *brw);
+bool brw_render_target_supported(gl_format format);
 
 #endif
diff --git a/src/mesa/drivers/dri/i965/brw_wm_emit.c b/src/mesa/drivers/dri/i965/brw_wm_emit.c
index 56725c0d471..2336e27c1ef 100644
--- a/src/mesa/drivers/dri/i965/brw_wm_emit.c
+++ b/src/mesa/drivers/dri/i965/brw_wm_emit.c
@@ -1333,9 +1333,11 @@ static void fire_fb_write( struct brw_wm_compile *c,
       dst = retype(vec8(brw_null_reg()), BRW_REGISTER_TYPE_UW);
 
    /* Pass through control information:
+    * 
+    * Gen6 has done m1 mov in emit_fb_write() for current SIMD16 case.
     */
 /*  mov (8) m1.0<1>:ud   r1.0<8;8,1>:ud   { Align1 NoMask } */
-   if (intel->gen < 6) /* gen6, use headerless for fb write */
+   if (intel->gen < 6)
    {
       brw_push_insn_state(p);
       brw_set_mask_control(p, BRW_MASK_DISABLE); /* ? */
diff --git a/src/mesa/drivers/dri/i965/brw_wm_sampler_state.c b/src/mesa/drivers/dri/i965/brw_wm_sampler_state.c
index e7c97a1cb05..f830e256268 100644
--- a/src/mesa/drivers/dri/i965/brw_wm_sampler_state.c
+++ b/src/mesa/drivers/dri/i965/brw_wm_sampler_state.c
@@ -115,6 +115,7 @@ struct wm_sampler_key {
    struct wm_sampler_entry {
       GLenum tex_target;
       GLenum wrap_r, wrap_s, wrap_t;
+      uint32_t base_level;
       float maxlod, minlod;
       float lod_bias;
       float max_aniso;
@@ -243,14 +244,7 @@ static void brw_update_sampler_state(struct brw_context *brw,
    sampler->ss0.lod_preclamp = 1; /* OpenGL mode */
    sampler->ss0.default_color_mode = 0; /* OpenGL/DX10 mode */
 
-   /* Set BaseMipLevel, MaxLOD, MinLOD: 
-    *
-    * XXX: I don't think that using firstLevel, lastLevel works,
-    * because we always setup the surface state as if firstLevel ==
-    * level zero.  Probably have to subtract firstLevel from each of
-    * these:
-    */
-   sampler->ss0.base_level = U_FIXED(0, 1);
+   sampler->ss0.base_level = U_FIXED(key->base_level, 1);
 
    sampler->ss1.max_lod = U_FIXED(CLAMP(key->maxlod, 0, 13), 6);
    sampler->ss1.min_lod = U_FIXED(CLAMP(key->minlod, 0, 13), 6);
@@ -276,9 +270,8 @@ brw_wm_sampler_populate_key(struct brw_context *brw,
 	 struct wm_sampler_entry *entry = &key->sampler[unit];
 	 struct gl_texture_unit *texUnit = &ctx->Texture.Unit[unit];
 	 struct gl_texture_object *texObj = texUnit->_Current;
-	 struct intel_texture_object *intelObj = intel_texture_object(texObj);
 	 struct gl_texture_image *firstImage =
-	    texObj->Image[0][intelObj->firstLevel];
+	    texObj->Image[0][texObj->BaseLevel];
 
 	 memset(last_entry_end, 0, 
 		(char*)entry - last_entry_end + sizeof(*entry));
@@ -293,6 +286,7 @@ brw_wm_sampler_populate_key(struct brw_context *brw,
 	 entry->wrap_s = texObj->WrapS;
 	 entry->wrap_t = texObj->WrapT;
 
+	 entry->base_level = texObj->BaseLevel;
 	 entry->maxlod = texObj->MaxLod;
 	 entry->minlod = texObj->MinLod;
 	 entry->lod_bias = texUnit->LodBias + texObj->LodBias;
diff --git a/src/mesa/drivers/dri/i965/brw_wm_surface_state.c b/src/mesa/drivers/dri/i965/brw_wm_surface_state.c
index 1cd736a1119..4570af66cd9 100644
--- a/src/mesa/drivers/dri/i965/brw_wm_surface_state.c
+++ b/src/mesa/drivers/dri/i965/brw_wm_surface_state.c
@@ -42,7 +42,7 @@
 #include "brw_context.h"
 #include "brw_state.h"
 #include "brw_defines.h"
-
+#include "brw_wm.h"
 
 static GLuint translate_tex_target( GLenum target )
 {
@@ -92,7 +92,10 @@ static uint32_t brw_format_for_mesa_format[MESA_FORMAT_COUNT] =
    [MESA_FORMAT_RGBA_DXT1] = BRW_SURFACEFORMAT_BC1_UNORM,
    [MESA_FORMAT_RGBA_DXT3] = BRW_SURFACEFORMAT_BC2_UNORM,
    [MESA_FORMAT_RGBA_DXT5] = BRW_SURFACEFORMAT_BC3_UNORM,
-   [MESA_FORMAT_SRGB_DXT1] = BRW_SURFACEFORMAT_BC1_UNORM_SRGB,
+   [MESA_FORMAT_SRGB_DXT1] = BRW_SURFACEFORMAT_DXT1_RGB_SRGB,
+   [MESA_FORMAT_SRGBA_DXT1] = BRW_SURFACEFORMAT_BC1_UNORM_SRGB,
+   [MESA_FORMAT_SRGBA_DXT3] = BRW_SURFACEFORMAT_BC2_UNORM_SRGB,
+   [MESA_FORMAT_SRGBA_DXT5] = BRW_SURFACEFORMAT_BC3_UNORM_SRGB,
    [MESA_FORMAT_SARGB8] = BRW_SURFACEFORMAT_B8G8R8A8_UNORM_SRGB,
    [MESA_FORMAT_SLA8] = BRW_SURFACEFORMAT_L8A8_UNORM_SRGB,
    [MESA_FORMAT_SL8] = BRW_SURFACEFORMAT_L8_UNORM_SRGB,
@@ -100,6 +103,21 @@ static uint32_t brw_format_for_mesa_format[MESA_FORMAT_COUNT] =
    [MESA_FORMAT_SIGNED_RGBA8888_REV] = BRW_SURFACEFORMAT_R8G8B8A8_SNORM,
 };
 
+bool
+brw_render_target_supported(gl_format format)
+{
+   if (format == MESA_FORMAT_S8_Z24 ||
+       format == MESA_FORMAT_X8_Z24 ||
+       format == MESA_FORMAT_Z16) {
+      return true;
+   }
+
+   /* Not exactly true, as some of those formats are not renderable.
+    * But at least we know how to translate them.
+    */
+   return brw_format_for_mesa_format[format] != 0;
+}
+
 static GLuint translate_tex_format( gl_format mesa_format,
                                     GLenum internal_format,
 				    GLenum depth_mode )
@@ -160,7 +178,7 @@ brw_update_texture_surface( struct gl_context *ctx, GLuint unit )
    struct brw_context *brw = brw_context(ctx);
    struct gl_texture_object *tObj = ctx->Texture.Unit[unit]._Current;
    struct intel_texture_object *intelObj = intel_texture_object(tObj);
-   struct gl_texture_image *firstImage = tObj->Image[0][intelObj->firstLevel];
+   struct gl_texture_image *firstImage = tObj->Image[0][tObj->BaseLevel];
    const GLuint surf_index = SURF_INDEX_TEXTURE(unit);
    struct brw_surface_state surf;
    void *map;
@@ -178,15 +196,15 @@ brw_update_texture_surface( struct gl_context *ctx, GLuint unit )
 /*    surf.ss0.data_return_format = BRW_SURFACERETURNFORMAT_S1; */
    surf.ss1.base_addr = intelObj->mt->region->buffer->offset; /* reloc */
 
-   surf.ss2.mip_count = intelObj->lastLevel - intelObj->firstLevel;
-   surf.ss2.width = firstImage->Width - 1;
-   surf.ss2.height = firstImage->Height - 1;
+   /* mip_count is #levels - 1 */
+   surf.ss2.mip_count = intelObj->_MaxLevel - tObj->BaseLevel;
+   surf.ss2.width = intelObj->mt->width0 - 1;
+   surf.ss2.height = intelObj->mt->height0 - 1;
    brw_set_surface_tiling(&surf, intelObj->mt->region->tiling);
    surf.ss3.pitch = (intelObj->mt->region->pitch * intelObj->mt->cpp) - 1;
-   surf.ss3.depth = firstImage->Depth - 1;
+   surf.ss3.depth = intelObj->mt->depth0 - 1;
+   surf.ss4.min_lod = tObj->BaseLevel;
 
-   surf.ss4.min_lod = 0;
- 
    if (tObj->Target == GL_TEXTURE_CUBE_MAP) {
       surf.ss0.cube_pos_x = 1;
       surf.ss0.cube_pos_y = 1;
@@ -354,6 +372,38 @@ const struct brw_tracked_state brw_wm_constant_surface = {
    .emit = upload_wm_constant_surface,
 };
 
+static void
+brw_update_null_renderbuffer_surface(struct brw_context *brw, unsigned int unit)
+{
+   struct intel_context *intel = &brw->intel;
+   struct brw_surface_state surf;
+   void *map;
+
+   memset(&surf, 0, sizeof(surf));
+
+   surf.ss0.surface_type = BRW_SURFACE_NULL;
+   surf.ss0.surface_format = BRW_SURFACEFORMAT_B8G8R8A8_UNORM;
+   surf.ss1.base_addr = 0;
+
+   surf.ss2.width = 0;
+   surf.ss2.height = 0;
+   brw_set_surface_tiling(&surf, I915_TILING_NONE);
+   surf.ss3.pitch = 0;
+
+   if (intel->gen < 6) {
+      /* _NEW_COLOR */
+      surf.ss0.color_blend = 0;
+      surf.ss0.writedisable_red =   1;
+      surf.ss0.writedisable_green = 1;
+      surf.ss0.writedisable_blue =  1;
+      surf.ss0.writedisable_alpha = 1;
+   }
+
+   map = brw_state_batch(brw, sizeof(surf), 32,
+			 &brw->wm.surf_bo[unit],
+			 &brw->wm.surf_offset[unit]);
+   memcpy(map, &surf, sizeof(surf));
+}
 
 /**
  * Sets up a surface state structure to point at the given region.
@@ -367,97 +417,48 @@ brw_update_renderbuffer_surface(struct brw_context *brw,
 {
    struct intel_context *intel = &brw->intel;
    struct gl_context *ctx = &intel->ctx;
-   drm_intel_bo *region_bo = NULL;
    struct intel_renderbuffer *irb = intel_renderbuffer(rb);
-   struct intel_region *region = irb ? irb->region : NULL;
-   struct {
-      unsigned int surface_type;
-      unsigned int surface_format;
-      unsigned int width, height, pitch, cpp;
-      GLubyte color_mask[4];
-      GLboolean color_blend;
-      uint32_t tiling;
-      uint32_t draw_x;
-      uint32_t draw_y;
-   } key;
+   struct intel_region *region = irb->region;
    struct brw_surface_state surf;
    void *map;
 
-   memset(&key, 0, sizeof(key));
-
-   if (region != NULL) {
-      region_bo = region->buffer;
-
-      key.surface_type = BRW_SURFACE_2D;
-      switch (irb->Base.Format) {
-      case MESA_FORMAT_XRGB8888:
-	 /* XRGB is handled as ARGB because the chips in this family
-	  * cannot render to XRGB targets.  This means that we have to
-	  * mask writes to alpha (ala glColorMask) and reconfigure the
-	  * alpha blending hardware to use GL_ONE (or GL_ZERO) for
-	  * cases where GL_DST_ALPHA (or GL_ONE_MINUS_DST_ALPHA) is
-	  * used.
-	  */
-	 key.surface_format = BRW_SURFACEFORMAT_B8G8R8A8_UNORM;
-	 break;
-      default:
-	 key.surface_format = brw_format_for_mesa_format[irb->Base.Format];
-	 assert(key.surface_format != 0);
-      }
-      key.tiling = region->tiling;
-      key.width = rb->Width;
-      key.height = rb->Height;
-      key.pitch = region->pitch;
-      key.cpp = region->cpp;
-      key.draw_x = region->draw_x;
-      key.draw_y = region->draw_y;
-   } else {
-      key.surface_type = BRW_SURFACE_NULL;
-      key.surface_format = BRW_SURFACEFORMAT_B8G8R8A8_UNORM;
-      key.tiling = I915_TILING_X;
-      key.width = 1;
-      key.height = 1;
-      key.cpp = 4;
-      key.draw_x = 0;
-      key.draw_y = 0;
-   }
-
-   if (intel->gen < 6) {
-      /* _NEW_COLOR */
-      memcpy(key.color_mask, ctx->Color.ColorMask[unit],
-	     sizeof(key.color_mask));
+   memset(&surf, 0, sizeof(surf));
 
-      /* As mentioned above, disable writes to the alpha component when the
-       * renderbuffer is XRGB.
+   switch (irb->Base.Format) {
+   case MESA_FORMAT_XRGB8888:
+      /* XRGB is handled as ARGB because the chips in this family
+       * cannot render to XRGB targets.  This means that we have to
+       * mask writes to alpha (ala glColorMask) and reconfigure the
+       * alpha blending hardware to use GL_ONE (or GL_ZERO) for
+       * cases where GL_DST_ALPHA (or GL_ONE_MINUS_DST_ALPHA) is
+       * used.
        */
-      if (ctx->DrawBuffer->Visual.alphaBits == 0)
-	 key.color_mask[3] = GL_FALSE;
-
-      key.color_blend = (!ctx->Color._LogicOpEnabled &&
-			 (ctx->Color.BlendEnabled & (1 << unit)));
+      surf.ss0.surface_format = BRW_SURFACEFORMAT_B8G8R8A8_UNORM;
+      break;
+   default:
+      surf.ss0.surface_format = brw_format_for_mesa_format[irb->Base.Format];
+      assert(surf.ss0.surface_format != 0);
    }
 
-   memset(&surf, 0, sizeof(surf));
-
-   surf.ss0.surface_format = key.surface_format;
-   surf.ss0.surface_type = key.surface_type;
-   if (key.tiling == I915_TILING_NONE) {
-      surf.ss1.base_addr = (key.draw_x + key.draw_y * key.pitch) * key.cpp;
+   surf.ss0.surface_type = BRW_SURFACE_2D;
+   if (region->tiling == I915_TILING_NONE) {
+      surf.ss1.base_addr = (region->draw_x +
+			    region->draw_y * region->pitch) * region->cpp;
    } else {
       uint32_t tile_base, tile_x, tile_y;
-      uint32_t pitch = key.pitch * key.cpp;
+      uint32_t pitch = region->pitch * region->cpp;
 
-      if (key.tiling == I915_TILING_X) {
-	 tile_x = key.draw_x % (512 / key.cpp);
-	 tile_y = key.draw_y % 8;
-	 tile_base = ((key.draw_y / 8) * (8 * pitch));
-	 tile_base += (key.draw_x - tile_x) / (512 / key.cpp) * 4096;
+      if (region->tiling == I915_TILING_X) {
+	 tile_x = region->draw_x % (512 / region->cpp);
+	 tile_y = region->draw_y % 8;
+	 tile_base = ((region->draw_y / 8) * (8 * pitch));
+	 tile_base += (region->draw_x - tile_x) / (512 / region->cpp) * 4096;
       } else {
 	 /* Y */
-	 tile_x = key.draw_x % (128 / key.cpp);
-	 tile_y = key.draw_y % 32;
-	 tile_base = ((key.draw_y / 32) * (32 * pitch));
-	 tile_base += (key.draw_x - tile_x) / (128 / key.cpp) * 4096;
+	 tile_x = region->draw_x % (128 / region->cpp);
+	 tile_y = region->draw_y % 32;
+	 tile_base = ((region->draw_y / 32) * (32 * pitch));
+	 tile_base += (region->draw_x - tile_x) / (128 / region->cpp) * 4096;
       }
       assert(brw->has_surface_tile_offset || (tile_x == 0 && tile_y == 0));
       assert(tile_x % 4 == 0);
@@ -469,21 +470,27 @@ brw_update_renderbuffer_surface(struct brw_context *brw,
       surf.ss5.x_offset = tile_x / 4;
       surf.ss5.y_offset = tile_y / 2;
    }
-   if (region_bo != NULL)
-      surf.ss1.base_addr += region_bo->offset; /* reloc */
+   surf.ss1.base_addr += region->buffer->offset; /* reloc */
 
-   surf.ss2.width = key.width - 1;
-   surf.ss2.height = key.height - 1;
-   brw_set_surface_tiling(&surf, key.tiling);
-   surf.ss3.pitch = (key.pitch * key.cpp) - 1;
+   surf.ss2.width = rb->Width - 1;
+   surf.ss2.height = rb->Height - 1;
+   brw_set_surface_tiling(&surf, region->tiling);
+   surf.ss3.pitch = (region->pitch * region->cpp) - 1;
 
    if (intel->gen < 6) {
       /* _NEW_COLOR */
-      surf.ss0.color_blend = key.color_blend;
-      surf.ss0.writedisable_red =   !key.color_mask[0];
-      surf.ss0.writedisable_green = !key.color_mask[1];
-      surf.ss0.writedisable_blue =  !key.color_mask[2];
-      surf.ss0.writedisable_alpha = !key.color_mask[3];
+      surf.ss0.color_blend = (!ctx->Color._LogicOpEnabled &&
+			      (ctx->Color.BlendEnabled & (1 << unit)));
+      surf.ss0.writedisable_red =   !ctx->Color.ColorMask[unit][0];
+      surf.ss0.writedisable_green = !ctx->Color.ColorMask[unit][1];
+      surf.ss0.writedisable_blue =  !ctx->Color.ColorMask[unit][2];
+      /* As mentioned above, disable writes to the alpha component when the
+       * renderbuffer is XRGB.
+       */
+      if (ctx->DrawBuffer->Visual.alphaBits == 0)
+	 surf.ss0.writedisable_alpha = 1;
+      else
+	 surf.ss0.writedisable_alpha = !ctx->Color.ColorMask[unit][3];
    }
 
    map = brw_state_batch(brw, sizeof(surf), 32,
@@ -491,15 +498,13 @@ brw_update_renderbuffer_surface(struct brw_context *brw,
 			 &brw->wm.surf_offset[unit]);
    memcpy(map, &surf, sizeof(surf));
 
-   if (region_bo != NULL) {
-      drm_intel_bo_emit_reloc(brw->wm.surf_bo[unit],
-			      brw->wm.surf_offset[unit] +
-			      offsetof(struct brw_surface_state, ss1),
-			      region_bo,
-			      surf.ss1.base_addr - region_bo->offset,
-			      I915_GEM_DOMAIN_RENDER,
-			      I915_GEM_DOMAIN_RENDER);
-   }
+   drm_intel_bo_emit_reloc(brw->wm.surf_bo[unit],
+			   brw->wm.surf_offset[unit] +
+			   offsetof(struct brw_surface_state, ss1),
+			   region->buffer,
+			   surf.ss1.base_addr - region->buffer->offset,
+			   I915_GEM_DOMAIN_RENDER,
+			   I915_GEM_DOMAIN_RENDER);
 }
 
 static void
@@ -559,12 +564,16 @@ upload_wm_surfaces(struct brw_context *brw)
    /* Update surfaces for drawing buffers */
    if (ctx->DrawBuffer->_NumColorDrawBuffers >= 1) {
       for (i = 0; i < ctx->DrawBuffer->_NumColorDrawBuffers; i++) {
-         brw_update_renderbuffer_surface(brw,
-					 ctx->DrawBuffer->_ColorDrawBuffers[i],
-					 i);
+	 if (intel_renderbuffer(ctx->DrawBuffer->_ColorDrawBuffers[i])) {
+	    brw_update_renderbuffer_surface(brw,
+					    ctx->DrawBuffer->_ColorDrawBuffers[i],
+					    i);
+	 } else {
+	    brw_update_null_renderbuffer_surface(brw, i);
+	 }
       }
    } else {
-      brw_update_renderbuffer_surface(brw, NULL, 0);
+      brw_update_null_renderbuffer_surface(brw, 0);
    }
 
    /* Update surfaces for textures */
diff --git a/src/mesa/drivers/dri/i965/gen6_cc.c b/src/mesa/drivers/dri/i965/gen6_cc.c
index c2631a7b4df..dbcdc5b8693 100644
--- a/src/mesa/drivers/dri/i965/gen6_cc.c
+++ b/src/mesa/drivers/dri/i965/gen6_cc.c
@@ -278,7 +278,7 @@ static void upload_cc_state_pointers(struct brw_context *brw)
    struct intel_context *intel = &brw->intel;
 
    BEGIN_BATCH(4);
-   OUT_BATCH(CMD_3D_CC_STATE_POINTERS << 16 | (4 - 2));
+   OUT_BATCH(_3DSTATE_CC_STATE_POINTERS << 16 | (4 - 2));
    OUT_RELOC(brw->cc.blend_state_bo, I915_GEM_DOMAIN_INSTRUCTION, 0, 1);
    OUT_RELOC(brw->cc.depth_stencil_state_bo, I915_GEM_DOMAIN_INSTRUCTION, 0, 1);
    OUT_RELOC(brw->cc.state_bo, I915_GEM_DOMAIN_INSTRUCTION, 0, 1);
diff --git a/src/mesa/drivers/dri/i965/gen6_clip_state.c b/src/mesa/drivers/dri/i965/gen6_clip_state.c
index c7c4eb1f27d..38c98f30efb 100644
--- a/src/mesa/drivers/dri/i965/gen6_clip_state.c
+++ b/src/mesa/drivers/dri/i965/gen6_clip_state.c
@@ -43,7 +43,10 @@ upload_clip_state(struct brw_context *brw)
       depth_clamp = GEN6_CLIP_Z_TEST;
 
    if (ctx->Light.ProvokingVertex == GL_FIRST_VERTEX_CONVENTION) {
-      provoking = 0;
+      provoking =
+	 (0 << GEN6_CLIP_TRI_PROVOKE_SHIFT) |
+	 (1 << GEN6_CLIP_TRIFAN_PROVOKE_SHIFT) |
+	 (0 << GEN6_CLIP_LINE_PROVOKE_SHIFT);
    } else {
       provoking =
 	 (2 << GEN6_CLIP_TRI_PROVOKE_SHIFT) |
@@ -55,7 +58,7 @@ upload_clip_state(struct brw_context *brw)
    userclip = (1 << brw_count_bits(ctx->Transform.ClipPlanesEnabled)) - 1;
 
    BEGIN_BATCH(4);
-   OUT_BATCH(CMD_3D_CLIP_STATE << 16 | (4 - 2));
+   OUT_BATCH(_3DSTATE_CLIP << 16 | (4 - 2));
    OUT_BATCH(GEN6_CLIP_STATISTICS_ENABLE);
    OUT_BATCH(GEN6_CLIP_ENABLE |
 	     GEN6_CLIP_API_OGL |
diff --git a/src/mesa/drivers/dri/i965/gen6_gs_state.c b/src/mesa/drivers/dri/i965/gen6_gs_state.c
index 6127b9197a1..7296c7cd1b0 100644
--- a/src/mesa/drivers/dri/i965/gen6_gs_state.c
+++ b/src/mesa/drivers/dri/i965/gen6_gs_state.c
@@ -37,7 +37,7 @@ upload_gs_state(struct brw_context *brw)
 
    /* Disable all the constant buffers. */
    BEGIN_BATCH(5);
-   OUT_BATCH(CMD_3D_CONSTANT_GS_STATE << 16 | (5 - 2));
+   OUT_BATCH(_3DSTATE_CONSTANT_GS << 16 | (5 - 2));
    OUT_BATCH(0);
    OUT_BATCH(0);
    OUT_BATCH(0);
@@ -46,7 +46,7 @@ upload_gs_state(struct brw_context *brw)
 
    if (brw->gs.prog_bo) {
       BEGIN_BATCH(7);
-      OUT_BATCH(CMD_3D_GS_STATE << 16 | (7 - 2));
+      OUT_BATCH(_3DSTATE_GS << 16 | (7 - 2));
       OUT_RELOC(brw->gs.prog_bo, I915_GEM_DOMAIN_INSTRUCTION, 0, 0);
       OUT_BATCH(GEN6_GS_SPF_MODE |
 		(0 << GEN6_GS_SAMPLER_COUNT_SHIFT) |
@@ -62,7 +62,7 @@ upload_gs_state(struct brw_context *brw)
       ADVANCE_BATCH();
    } else {
       BEGIN_BATCH(7);
-      OUT_BATCH(CMD_3D_GS_STATE << 16 | (7 - 2));
+      OUT_BATCH(_3DSTATE_GS << 16 | (7 - 2));
       OUT_BATCH(0); /* prog_bo */
       OUT_BATCH((0 << GEN6_GS_SAMPLER_COUNT_SHIFT) |
 		(0 << GEN6_GS_BINDING_TABLE_ENTRY_COUNT_SHIFT));
diff --git a/src/mesa/drivers/dri/i965/gen6_sampler_state.c b/src/mesa/drivers/dri/i965/gen6_sampler_state.c
index fc5d391c3cf..f65c651bdff 100644
--- a/src/mesa/drivers/dri/i965/gen6_sampler_state.c
+++ b/src/mesa/drivers/dri/i965/gen6_sampler_state.c
@@ -36,7 +36,7 @@ upload_sampler_state_pointers(struct brw_context *brw)
    struct intel_context *intel = &brw->intel;
 
    BEGIN_BATCH(4);
-   OUT_BATCH(CMD_3D_SAMPLER_STATE_POINTERS << 16 |
+   OUT_BATCH(_3DSTATE_SAMPLER_STATE_POINTERS << 16 |
 	     VS_SAMPLER_STATE_CHANGE |
 	     GS_SAMPLER_STATE_CHANGE |
 	     PS_SAMPLER_STATE_CHANGE |
diff --git a/src/mesa/drivers/dri/i965/gen6_scissor_state.c b/src/mesa/drivers/dri/i965/gen6_scissor_state.c
index b57126c7938..12b65826ae9 100644
--- a/src/mesa/drivers/dri/i965/gen6_scissor_state.c
+++ b/src/mesa/drivers/dri/i965/gen6_scissor_state.c
@@ -92,7 +92,7 @@ static void upload_scissor_state_pointers(struct brw_context *brw)
    struct intel_context *intel = &brw->intel;
 
    BEGIN_BATCH(2);
-   OUT_BATCH(CMD_3D_SCISSOR_STATE_POINTERS << 16 | (2 - 2));
+   OUT_BATCH(_3DSTATE_SCISSOR_STATE_POINTERS << 16 | (2 - 2));
    OUT_RELOC(brw->sf.state_bo, I915_GEM_DOMAIN_INSTRUCTION, 0, 0);
    ADVANCE_BATCH();
 
diff --git a/src/mesa/drivers/dri/i965/gen6_sf_state.c b/src/mesa/drivers/dri/i965/gen6_sf_state.c
index 45c148baedd..f27782935d4 100644
--- a/src/mesa/drivers/dri/i965/gen6_sf_state.c
+++ b/src/mesa/drivers/dri/i965/gen6_sf_state.c
@@ -237,7 +237,7 @@ upload_sf_state(struct brw_context *brw)
    }
 
    BEGIN_BATCH(20);
-   OUT_BATCH(CMD_3D_SF_STATE << 16 | (20 - 2));
+   OUT_BATCH(_3DSTATE_SF << 16 | (20 - 2));
    OUT_BATCH(dw1);
    OUT_BATCH(dw2);
    OUT_BATCH(dw3);
diff --git a/src/mesa/drivers/dri/i965/gen6_urb.c b/src/mesa/drivers/dri/i965/gen6_urb.c
index de97fd3783d..fc46c4cb79e 100644
--- a/src/mesa/drivers/dri/i965/gen6_urb.c
+++ b/src/mesa/drivers/dri/i965/gen6_urb.c
@@ -60,7 +60,7 @@ upload_urb(struct brw_context *brw)
    assert(!brw->gs.prog_bo || brw->urb.vs_size < 5);
 
    BEGIN_BATCH(3);
-   OUT_BATCH(CMD_URB << 16 | (3 - 2));
+   OUT_BATCH(_3DSTATE_URB << 16 | (3 - 2));
    OUT_BATCH(((brw->urb.vs_size - 1) << GEN6_URB_VS_SIZE_SHIFT) |
 	     ((brw->urb.nr_vs_entries) << GEN6_URB_VS_ENTRIES_SHIFT));
    OUT_BATCH(((brw->urb.vs_size - 1) << GEN6_URB_GS_SIZE_SHIFT) |
diff --git a/src/mesa/drivers/dri/i965/gen6_viewport_state.c b/src/mesa/drivers/dri/i965/gen6_viewport_state.c
index d691bbebc83..cd7d209e3ea 100644
--- a/src/mesa/drivers/dri/i965/gen6_viewport_state.c
+++ b/src/mesa/drivers/dri/i965/gen6_viewport_state.c
@@ -117,7 +117,7 @@ static void upload_viewport_state_pointers(struct brw_context *brw)
    struct intel_context *intel = &brw->intel;
 
    BEGIN_BATCH(4);
-   OUT_BATCH(CMD_VIEWPORT_STATE_POINTERS << 16 | (4 - 2) |
+   OUT_BATCH(_3DSTATE_VIEWPORT_STATE_POINTERS << 16 | (4 - 2) |
 	     GEN6_CC_VIEWPORT_MODIFY |
 	     GEN6_SF_VIEWPORT_MODIFY |
 	     GEN6_CLIP_VIEWPORT_MODIFY);
diff --git a/src/mesa/drivers/dri/i965/gen6_vs_state.c b/src/mesa/drivers/dri/i965/gen6_vs_state.c
index ed132bdbd93..e68c0ac261c 100644
--- a/src/mesa/drivers/dri/i965/gen6_vs_state.c
+++ b/src/mesa/drivers/dri/i965/gen6_vs_state.c
@@ -47,7 +47,7 @@ upload_vs_state(struct brw_context *brw)
    if (brw->vs.prog_data->nr_params == 0 && !ctx->Transform.ClipPlanesEnabled) {
       /* Disable the push constant buffers. */
       BEGIN_BATCH(5);
-      OUT_BATCH(CMD_3D_CONSTANT_VS_STATE << 16 | (5 - 2));
+      OUT_BATCH(_3DSTATE_CONSTANT_VS << 16 | (5 - 2));
       OUT_BATCH(0);
       OUT_BATCH(0);
       OUT_BATCH(0);
@@ -112,7 +112,7 @@ upload_vs_state(struct brw_context *brw)
       assert(param_regs <= 32);
 
       BEGIN_BATCH(5);
-      OUT_BATCH(CMD_3D_CONSTANT_VS_STATE << 16 |
+      OUT_BATCH(_3DSTATE_CONSTANT_VS << 16 |
 		GEN6_CONSTANT_BUFFER_0_ENABLE |
 		(5 - 2));
       OUT_RELOC(constant_bo,
@@ -127,7 +127,7 @@ upload_vs_state(struct brw_context *brw)
    }
 
    BEGIN_BATCH(6);
-   OUT_BATCH(CMD_3D_VS_STATE << 16 | (6 - 2));
+   OUT_BATCH(_3DSTATE_VS << 16 | (6 - 2));
    OUT_RELOC(brw->vs.prog_bo, I915_GEM_DOMAIN_INSTRUCTION, 0, 0);
    OUT_BATCH((0 << GEN6_VS_SAMPLER_COUNT_SHIFT) |
 	     GEN6_VS_FLOATING_POINT_MODE_ALT |
diff --git a/src/mesa/drivers/dri/i965/gen6_wm_state.c b/src/mesa/drivers/dri/i965/gen6_wm_state.c
index 2ae0c093ebe..78901ecac57 100644
--- a/src/mesa/drivers/dri/i965/gen6_wm_state.c
+++ b/src/mesa/drivers/dri/i965/gen6_wm_state.c
@@ -107,7 +107,7 @@ upload_wm_state(struct brw_context *brw)
    if (brw->wm.prog_data->nr_params == 0) {
       /* Disable the push constant buffers. */
       BEGIN_BATCH(5);
-      OUT_BATCH(CMD_3D_CONSTANT_PS_STATE << 16 | (5 - 2));
+      OUT_BATCH(_3DSTATE_CONSTANT_PS << 16 | (5 - 2));
       OUT_BATCH(0);
       OUT_BATCH(0);
       OUT_BATCH(0);
@@ -115,7 +115,7 @@ upload_wm_state(struct brw_context *brw)
       ADVANCE_BATCH();
    } else {
       BEGIN_BATCH(5);
-      OUT_BATCH(CMD_3D_CONSTANT_PS_STATE << 16 |
+      OUT_BATCH(_3DSTATE_CONSTANT_PS << 16 |
 		GEN6_CONSTANT_BUFFER_0_ENABLE |
 		(5 - 2));
       OUT_RELOC(brw->wm.push_const_bo,
@@ -181,7 +181,7 @@ upload_wm_state(struct brw_context *brw)
       GEN6_WM_NUM_SF_OUTPUTS_SHIFT;
 
    BEGIN_BATCH(9);
-   OUT_BATCH(CMD_3D_WM_STATE << 16 | (9 - 2));
+   OUT_BATCH(_3DSTATE_WM << 16 | (9 - 2));
    OUT_RELOC(brw->wm.prog_bo, I915_GEM_DOMAIN_INSTRUCTION, 0, 0);
    OUT_BATCH(dw2);
    OUT_BATCH(0); /* scratch space base offset */
diff --git a/src/mesa/drivers/dri/intel/intel_batchbuffer.c b/src/mesa/drivers/dri/intel/intel_batchbuffer.c
index 20574ab5462..67ce8a4da02 100644
--- a/src/mesa/drivers/dri/intel/intel_batchbuffer.c
+++ b/src/mesa/drivers/dri/intel/intel_batchbuffer.c
@@ -271,8 +271,11 @@ intel_batchbuffer_emit_mi_flush(struct intel_batchbuffer *batch)
 
    if (intel->gen >= 6) {
       if (intel->batch->is_blit) {
-	 BEGIN_BATCH_BLT(1);
-	 OUT_BATCH(MI_FLUSH);
+	 BEGIN_BATCH_BLT(4);
+	 OUT_BATCH(MI_FLUSH_DW);
+	 OUT_BATCH(0);
+	 OUT_BATCH(0);
+	 OUT_BATCH(0);
 	 ADVANCE_BATCH();
       } else {
 	 BEGIN_BATCH(8);
diff --git a/src/mesa/drivers/dri/intel/intel_blit.c b/src/mesa/drivers/dri/intel/intel_blit.c
index a2822b11d96..6232e479cb6 100644
--- a/src/mesa/drivers/dri/intel/intel_blit.c
+++ b/src/mesa/drivers/dri/intel/intel_blit.c
@@ -38,7 +38,6 @@
 #include "intel_reg.h"
 #include "intel_regions.h"
 #include "intel_batchbuffer.h"
-#include "intel_tex.h"
 #include "intel_mipmap_tree.h"
 
 #define FILE_DEBUG_FLAG DEBUG_BLIT
@@ -207,7 +206,7 @@ intelEmitCopyBlit(struct intel_context *intel,
  * which we're clearing with triangles.
  * \param mask  bitmask of BUFFER_BIT_* values indicating buffers to clear
  */
-void
+GLbitfield
 intelClearWithBlit(struct gl_context *ctx, GLbitfield mask)
 {
    struct intel_context *intel = intel_context(ctx);
@@ -215,6 +214,7 @@ intelClearWithBlit(struct gl_context *ctx, GLbitfield mask)
    GLuint clear_depth;
    GLboolean all;
    GLint cx, cy, cw, ch;
+   GLbitfield fail_mask = 0;
    BATCH_LOCALS;
 
    /*
@@ -237,7 +237,7 @@ intelClearWithBlit(struct gl_context *ctx, GLbitfield mask)
    ch = fb->_Ymax - fb->_Ymin;
 
    if (cw == 0 || ch == 0)
-      return;
+      return 0;
 
    GLuint buf;
    all = (cw == fb->Width && ch == fb->Height);
@@ -333,9 +333,9 @@ intelClearWithBlit(struct gl_context *ctx, GLbitfield mask)
 					clear[3], clear[3]);
 	    break;
 	 default:
-	    _mesa_problem(ctx, "Unexpected renderbuffer format: %d\n",
-			  irb->Base.Format);
-	    clear_val = 0;
+	    fail_mask |= bufBit;
+	    mask &= ~bufBit;
+	    continue;
 	 }
       }
 
@@ -370,6 +370,8 @@ intelClearWithBlit(struct gl_context *ctx, GLbitfield mask)
       else
 	 mask &= ~bufBit;    /* turn off bit, for faster loop exit */
    }
+
+   return fail_mask;
 }
 
 GLboolean
diff --git a/src/mesa/drivers/dri/intel/intel_blit.h b/src/mesa/drivers/dri/intel/intel_blit.h
index ff69e4f8f8f..88322c7b493 100644
--- a/src/mesa/drivers/dri/intel/intel_blit.h
+++ b/src/mesa/drivers/dri/intel/intel_blit.h
@@ -33,7 +33,7 @@
 extern void intelCopyBuffer(const __DRIdrawable * dpriv,
                             const drm_clip_rect_t * rect);
 
-extern void intelClearWithBlit(struct gl_context * ctx, GLbitfield mask);
+extern GLbitfield intelClearWithBlit(struct gl_context * ctx, GLbitfield mask);
 
 GLboolean
 intelEmitCopyBlit(struct intel_context *intel,
diff --git a/src/mesa/drivers/dri/intel/intel_clear.c b/src/mesa/drivers/dri/intel/intel_clear.c
index fa451f0045e..82d29e76712 100644
--- a/src/mesa/drivers/dri/intel/intel_clear.c
+++ b/src/mesa/drivers/dri/intel/intel_clear.c
@@ -85,6 +85,8 @@ intelClear(struct gl_context *ctx, GLbitfield mask)
    GLbitfield blit_mask = 0;
    GLbitfield swrast_mask = 0;
    struct gl_framebuffer *fb = ctx->DrawBuffer;
+   struct intel_renderbuffer *irb;
+   int i;
 
    if (mask & (BUFFER_BIT_FRONT_LEFT | BUFFER_BIT_FRONT_RIGHT)) {
       intel->front_buffer_dirty = GL_TRUE;
@@ -93,6 +95,22 @@ intelClear(struct gl_context *ctx, GLbitfield mask)
    if (0)
       fprintf(stderr, "%s\n", __FUNCTION__);
 
+   /* Get SW clears out of the way: Anything without an intel_renderbuffer */
+   for (i = 0; i < BUFFER_COUNT; i++) {
+      if (!(mask & (1 << i)))
+	 continue;
+
+      irb = intel_get_renderbuffer(fb, i);
+      if (unlikely(!irb)) {
+	 swrast_mask |= (1 << i);
+	 mask &= ~(1 << i);
+      }
+   }
+   if (unlikely(swrast_mask)) {
+      debug_mask("swrast", swrast_mask);
+      _swrast_Clear(ctx, swrast_mask);
+   }
+
    /* HW color buffers (front, back, aux, generic FBO, etc) */
    if (colorMask == ~0) {
       /* clear all R,G,B,A */
@@ -151,44 +169,18 @@ intelClear(struct gl_context *ctx, GLbitfield mask)
       }
    }
 
-   if (intel->gen >= 6) {
-      /* Blits are in a different ringbuffer so we don't use them. */
-      tri_mask |= blit_mask;
-      blit_mask = 0;
-   }
-
-   /* SW fallback clearing */
-   swrast_mask = mask & ~tri_mask & ~blit_mask;
-
-   {
-      /* look for non-Intel renderbuffers (clear them with swrast) */
-      GLbitfield blit_or_tri = blit_mask | tri_mask;
-      while (blit_or_tri) {
-         GLuint i = _mesa_ffs(blit_or_tri) - 1;
-         GLbitfield bufBit = 1 << i;
-         if (!fb->Attachment[i].Renderbuffer->ClassID) {
-            blit_mask &= ~bufBit;
-            tri_mask &= ~bufBit;
-            swrast_mask |= bufBit;
-         }
-         blit_or_tri ^= bufBit;
-      }
-   }
+   /* Anything left, just use tris */
+   tri_mask |= mask & ~blit_mask;
 
    if (blit_mask) {
       debug_mask("blit", blit_mask);
-      intelClearWithBlit(ctx, blit_mask);
+      tri_mask |= intelClearWithBlit(ctx, blit_mask);
    }
 
    if (tri_mask) {
       debug_mask("tri", tri_mask);
       _mesa_meta_Clear(&intel->ctx, tri_mask);
    }
-
-   if (swrast_mask) {
-      debug_mask("swrast", swrast_mask);
-      _swrast_Clear(ctx, swrast_mask);
-   }
 }
 
 
diff --git a/src/mesa/drivers/dri/intel/intel_context.c b/src/mesa/drivers/dri/intel/intel_context.c
index 9c222c7b485..2a5029964be 100644
--- a/src/mesa/drivers/dri/intel/intel_context.c
+++ b/src/mesa/drivers/dri/intel/intel_context.c
@@ -565,7 +565,8 @@ intel_glFlush(struct gl_context *ctx)
 
    intel_flush(ctx);
    intel_flush_front(ctx);
-   intel->need_throttle = GL_TRUE;
+   if (intel->is_front_buffer_rendering)
+      intel->need_throttle = GL_TRUE;
 }
 
 void
@@ -682,6 +683,69 @@ intelInitContext(struct intel_context *intel,
       }
    }
 
+   memset(&ctx->TextureFormatSupported, 0,
+	  sizeof(ctx->TextureFormatSupported));
+   ctx->TextureFormatSupported[MESA_FORMAT_ARGB8888] = GL_TRUE;
+   if (intel->has_xrgb_textures)
+      ctx->TextureFormatSupported[MESA_FORMAT_XRGB8888] = GL_TRUE;
+   ctx->TextureFormatSupported[MESA_FORMAT_ARGB4444] = GL_TRUE;
+   ctx->TextureFormatSupported[MESA_FORMAT_ARGB1555] = GL_TRUE;
+   ctx->TextureFormatSupported[MESA_FORMAT_RGB565] = GL_TRUE;
+   ctx->TextureFormatSupported[MESA_FORMAT_L8] = GL_TRUE;
+   ctx->TextureFormatSupported[MESA_FORMAT_A8] = GL_TRUE;
+   ctx->TextureFormatSupported[MESA_FORMAT_I8] = GL_TRUE;
+   ctx->TextureFormatSupported[MESA_FORMAT_AL88] = GL_TRUE;
+   if (intel->gen >= 4)
+      ctx->TextureFormatSupported[MESA_FORMAT_AL1616] = GL_TRUE;
+   ctx->TextureFormatSupported[MESA_FORMAT_S8_Z24] = GL_TRUE;
+   /*
+    * This was disabled in initial FBO enabling to avoid combinations
+    * of depth+stencil that wouldn't work together.  We since decided
+    * that it was OK, since it's up to the app to come up with the
+    * combo that actually works, so this can probably be re-enabled.
+    */
+   /*
+   ctx->TextureFormatSupported[MESA_FORMAT_Z16] = GL_TRUE;
+   ctx->TextureFormatSupported[MESA_FORMAT_Z24] = GL_TRUE;
+   */
+
+   /* ctx->Extensions.MESA_ycbcr_texture */
+   ctx->TextureFormatSupported[MESA_FORMAT_YCBCR] = GL_TRUE;
+   ctx->TextureFormatSupported[MESA_FORMAT_YCBCR_REV] = GL_TRUE;
+
+   /* GL_3DFX_texture_compression_FXT1 */
+   ctx->TextureFormatSupported[MESA_FORMAT_RGB_FXT1] = GL_TRUE;
+   ctx->TextureFormatSupported[MESA_FORMAT_RGBA_FXT1] = GL_TRUE;
+
+   /* GL_EXT_texture_compression_s3tc */
+   ctx->TextureFormatSupported[MESA_FORMAT_RGB_DXT1] = GL_TRUE;
+   ctx->TextureFormatSupported[MESA_FORMAT_RGBA_DXT1] = GL_TRUE;
+   ctx->TextureFormatSupported[MESA_FORMAT_RGBA_DXT3] = GL_TRUE;
+   ctx->TextureFormatSupported[MESA_FORMAT_RGBA_DXT5] = GL_TRUE;
+
+#ifndef I915
+   /* GL_ARB_texture_rg */
+   ctx->TextureFormatSupported[MESA_FORMAT_R8] = GL_TRUE;
+   ctx->TextureFormatSupported[MESA_FORMAT_R16] = GL_TRUE;
+   ctx->TextureFormatSupported[MESA_FORMAT_RG88] = GL_TRUE;
+   ctx->TextureFormatSupported[MESA_FORMAT_RG1616] = GL_TRUE;
+
+   ctx->TextureFormatSupported[MESA_FORMAT_DUDV8] = GL_TRUE;
+   ctx->TextureFormatSupported[MESA_FORMAT_SIGNED_RGBA8888_REV] = GL_TRUE;
+
+   /* GL_EXT_texture_sRGB */
+   ctx->TextureFormatSupported[MESA_FORMAT_SARGB8] = GL_TRUE;
+   if (intel->gen >= 5 || intel->is_g4x)
+      ctx->TextureFormatSupported[MESA_FORMAT_SRGB_DXT1] = GL_TRUE;
+   ctx->TextureFormatSupported[MESA_FORMAT_SRGBA_DXT1] = GL_TRUE;
+   ctx->TextureFormatSupported[MESA_FORMAT_SRGBA_DXT3] = GL_TRUE;
+   ctx->TextureFormatSupported[MESA_FORMAT_SRGBA_DXT5] = GL_TRUE;
+   if (intel->has_luminance_srgb) {
+      ctx->TextureFormatSupported[MESA_FORMAT_SL8] = GL_TRUE;
+      ctx->TextureFormatSupported[MESA_FORMAT_SLA8] = GL_TRUE;
+   }
+#endif
+
    driParseConfigFiles(&intel->optionCache, &intelScreen->optionCache,
                        sPriv->myNum, (intel->gen >= 4) ? "i965" : "i915");
    if (intelScreen->deviceID == PCI_CHIP_I865_G)
diff --git a/src/mesa/drivers/dri/intel/intel_context.h b/src/mesa/drivers/dri/intel/intel_context.h
index 53a11ba9a7e..fd3c3ba58fc 100644
--- a/src/mesa/drivers/dri/intel/intel_context.h
+++ b/src/mesa/drivers/dri/intel/intel_context.h
@@ -149,6 +149,7 @@ struct intel_context
       void (*assert_not_dirty) (struct intel_context *intel);
 
       void (*debug_batch)(struct intel_context *intel);
+      bool (*render_target_supported)(gl_format format);
    } vtbl;
 
    struct dri_metaops meta;
diff --git a/src/mesa/drivers/dri/intel/intel_extensions.c b/src/mesa/drivers/dri/intel/intel_extensions.c
index 556a4195bdd..7e17a8c32cd 100644
--- a/src/mesa/drivers/dri/intel/intel_extensions.c
+++ b/src/mesa/drivers/dri/intel/intel_extensions.c
@@ -80,6 +80,7 @@
 static const struct dri_extension card_extensions[] = {
    { "GL_ARB_draw_elements_base_vertex",  GL_ARB_draw_elements_base_vertex_functions },
    { "GL_ARB_explicit_attrib_location",   NULL },
+   { "GL_ARB_framebuffer_object",         GL_ARB_framebuffer_object_functions},
    { "GL_ARB_half_float_pixel",           NULL },
    { "GL_ARB_map_buffer_range",           GL_ARB_map_buffer_range_functions },
    { "GL_ARB_multitexture",               NULL },
@@ -161,7 +162,6 @@ static const struct dri_extension brw_extensions[] = {
    { "GL_ARB_fragment_program",           NULL },
    { "GL_ARB_fragment_program_shadow",    NULL },
    { "GL_ARB_fragment_shader",            NULL },
-   { "GL_ARB_framebuffer_object",         GL_ARB_framebuffer_object_functions},
    { "GL_ARB_half_float_vertex",          NULL },
    { "GL_ARB_occlusion_query",            GL_ARB_occlusion_query_functions },
    { "GL_ARB_point_sprite", 		  NULL },
diff --git a/src/mesa/drivers/dri/intel/intel_extensions_es2.c b/src/mesa/drivers/dri/intel/intel_extensions_es2.c
index 71c86339c72..a58e62cd618 100644
--- a/src/mesa/drivers/dri/intel/intel_extensions_es2.c
+++ b/src/mesa/drivers/dri/intel/intel_extensions_es2.c
@@ -62,6 +62,7 @@ static const char *es2_extensions[] = {
    "GL_EXT_blend_minmax",
    "GL_EXT_blend_subtract",
    "GL_EXT_stencil_wrap",
+   "GL_NV_blend_square",
 
    /* Optional GLES2 */
    "GL_ARB_framebuffer_object",
diff --git a/src/mesa/drivers/dri/intel/intel_fbo.c b/src/mesa/drivers/dri/intel/intel_fbo.c
index c3f528c2ae5..efc726e32fa 100644
--- a/src/mesa/drivers/dri/intel/intel_fbo.c
+++ b/src/mesa/drivers/dri/intel/intel_fbo.c
@@ -115,8 +115,8 @@ intel_alloc_renderbuffer_storage(struct gl_context * ctx, struct gl_renderbuffer
        * except they're less useful because you can't texture with
        * them.
        */
-      rb->Format = intelChooseTextureFormat(ctx, internalFormat,
-					    GL_NONE, GL_NONE);
+      rb->Format = intel->ctx.Driver.ChooseTextureFormat(ctx, internalFormat,
+							 GL_NONE, GL_NONE);
       break;
    case GL_STENCIL_INDEX:
    case GL_STENCIL_INDEX1_EXT:
@@ -145,10 +145,15 @@ intel_alloc_renderbuffer_storage(struct gl_context * ctx, struct gl_renderbuffer
    DBG("Allocating %d x %d Intel RBO\n", width, height);
 
    tiling = I915_TILING_NONE;
-
-   /* Gen6 requires depth must be tiling */
-   if (intel->gen >= 6 && rb->Format == MESA_FORMAT_S8_Z24)
-       tiling = I915_TILING_Y;
+   if (intel->use_texture_tiling) {
+      GLenum base_format = _mesa_get_format_base_format(rb->Format);
+
+      if (intel->gen >= 4 && (base_format == GL_DEPTH_COMPONENT ||
+			      base_format == GL_DEPTH_STENCIL))
+	 tiling = I915_TILING_Y;
+      else
+	 tiling = I915_TILING_X;
+   }
 
    irb->region = intel_region_alloc(intel->intelScreen, tiling, cpp,
 				    width, height, GL_TRUE);
@@ -284,42 +289,7 @@ intel_create_renderbuffer(gl_format format)
 
    _mesa_init_renderbuffer(&irb->Base, 0);
    irb->Base.ClassID = INTEL_RB_CLASS;
-
-   switch (format) {
-   case MESA_FORMAT_RGB565:
-      irb->Base._BaseFormat = GL_RGB;
-      break;
-   case MESA_FORMAT_XRGB8888:
-      irb->Base._BaseFormat = GL_RGB;
-      break;
-   case MESA_FORMAT_ARGB8888:
-      irb->Base._BaseFormat = GL_RGBA;
-      break;
-   case MESA_FORMAT_Z16:
-      irb->Base._BaseFormat = GL_DEPTH_COMPONENT;
-      break;
-   case MESA_FORMAT_X8_Z24:
-      irb->Base._BaseFormat = GL_DEPTH_COMPONENT;
-      break;
-   case MESA_FORMAT_S8_Z24:
-      irb->Base._BaseFormat = GL_DEPTH_STENCIL;
-      break;
-   case MESA_FORMAT_A8:
-      irb->Base._BaseFormat = GL_ALPHA;
-      break;
-   case MESA_FORMAT_R8:
-      irb->Base._BaseFormat = GL_RED;
-      break;
-   case MESA_FORMAT_RG88:
-      irb->Base._BaseFormat = GL_RG;
-      break;
-   default:
-      _mesa_problem(NULL,
-                    "Unexpected intFormat in intel_create_renderbuffer");
-      free(irb);
-      return NULL;
-   }
-
+   irb->Base._BaseFormat = _mesa_get_format_base_format(format);
    irb->Base.Format = format;
    irb->Base.InternalFormat = irb->Base._BaseFormat;
    irb->Base.DataType = intel_mesa_format_to_rb_datatype(format);
@@ -564,6 +534,7 @@ intel_finish_render_texture(struct gl_context * ctx,
 static void
 intel_validate_framebuffer(struct gl_context *ctx, struct gl_framebuffer *fb)
 {
+   struct intel_context *intel = intel_context(ctx);
    const struct intel_renderbuffer *depthRb =
       intel_get_renderbuffer(fb, BUFFER_DEPTH);
    const struct intel_renderbuffer *stencilRb =
@@ -571,10 +542,10 @@ intel_validate_framebuffer(struct gl_context *ctx, struct gl_framebuffer *fb)
    int i;
 
    if (depthRb && stencilRb && stencilRb != depthRb) {
-      if (ctx->DrawBuffer->Attachment[BUFFER_DEPTH].Type == GL_TEXTURE &&
-	  ctx->DrawBuffer->Attachment[BUFFER_STENCIL].Type == GL_TEXTURE &&
-	  (ctx->DrawBuffer->Attachment[BUFFER_DEPTH].Texture->Name ==
-	   ctx->DrawBuffer->Attachment[BUFFER_STENCIL].Texture->Name)) {
+      if (fb->Attachment[BUFFER_DEPTH].Type == GL_TEXTURE &&
+	  fb->Attachment[BUFFER_STENCIL].Type == GL_TEXTURE &&
+	  (fb->Attachment[BUFFER_DEPTH].Texture->Name ==
+	   fb->Attachment[BUFFER_STENCIL].Texture->Name)) {
 	 /* OK */
       } else {
 	 /* we only support combined depth/stencil buffers, not separate
@@ -587,20 +558,33 @@ intel_validate_framebuffer(struct gl_context *ctx, struct gl_framebuffer *fb)
       }
    }
 
-   for (i = 0; i < ctx->Const.MaxDrawBuffers; i++) {
-      struct gl_renderbuffer *rb = ctx->DrawBuffer->_ColorDrawBuffers[i];
-      struct intel_renderbuffer *irb = intel_renderbuffer(rb);
+   for (i = 0; i < Elements(fb->Attachment); i++) {
+      struct gl_renderbuffer *rb;
+      struct intel_renderbuffer *irb;
 
-      if (rb == NULL)
+      if (fb->Attachment[i].Type == GL_NONE)
 	 continue;
 
+      /* A supported attachment will have a Renderbuffer set either
+       * from being a Renderbuffer or being a texture that got the
+       * intel_wrap_texture() treatment.
+       */
+      rb = fb->Attachment[i].Renderbuffer;
+      if (rb == NULL) {
+	 DBG("attachment without renderbuffer\n");
+	 fb->_Status = GL_FRAMEBUFFER_UNSUPPORTED_EXT;
+	 continue;
+      }
+
+      irb = intel_renderbuffer(rb);
       if (irb == NULL) {
 	 DBG("software rendering renderbuffer\n");
 	 fb->_Status = GL_FRAMEBUFFER_UNSUPPORTED_EXT;
 	 continue;
       }
 
-      if (!intel_span_supports_format(irb->Base.Format)) {
+      if (!intel_span_supports_format(irb->Base.Format) ||
+	  !intel->vtbl.render_target_supported(irb->Base.Format)) {
 	 DBG("Unsupported texture/renderbuffer format attached: %s\n",
 	     _mesa_get_format_name(irb->Base.Format));
 	 fb->_Status = GL_FRAMEBUFFER_UNSUPPORTED_EXT;
diff --git a/src/mesa/drivers/dri/intel/intel_mipmap_tree.c b/src/mesa/drivers/dri/intel/intel_mipmap_tree.c
index 9c4e5c5ee8c..2ced6ac86b2 100644
--- a/src/mesa/drivers/dri/intel/intel_mipmap_tree.c
+++ b/src/mesa/drivers/dri/intel/intel_mipmap_tree.c
@@ -56,8 +56,7 @@ static struct intel_mipmap_tree *
 intel_miptree_create_internal(struct intel_context *intel,
 			      GLenum target,
 			      GLenum internal_format,
-			      GLuint first_level,
-			      GLuint last_level,
+			      GLuint levels,
 			      GLuint width0,
 			      GLuint height0,
 			      GLuint depth0, GLuint cpp, GLuint compress_byte,
@@ -66,15 +65,14 @@ intel_miptree_create_internal(struct intel_context *intel,
    GLboolean ok;
    struct intel_mipmap_tree *mt = calloc(sizeof(*mt), 1);
 
-   DBG("%s target %s format %s level %d..%d <-- %p\n", __FUNCTION__,
+   DBG("%s target %s format %s levels %d <-- %p\n", __FUNCTION__,
        _mesa_lookup_enum_by_nr(target),
        _mesa_lookup_enum_by_nr(internal_format), 
-       first_level, last_level, mt);
+       levels, mt);
 
    mt->target = target_to_target(target);
    mt->internal_format = internal_format;
-   mt->first_level = first_level;
-   mt->last_level = last_level;
+   mt->levels = levels;
    mt->width0 = width0;
    mt->height0 = height0;
    mt->depth0 = depth0;
@@ -106,8 +104,7 @@ intel_miptree_create(struct intel_context *intel,
 		     GLenum target,
 		     GLenum base_format,
 		     GLenum internal_format,
-		     GLuint first_level,
-		     GLuint last_level,
+		     GLuint levels,
 		     GLuint width0,
 		     GLuint height0,
 		     GLuint depth0, GLuint cpp, GLuint compress_byte,
@@ -126,7 +123,7 @@ intel_miptree_create(struct intel_context *intel,
    }
 
    mt = intel_miptree_create_internal(intel, target, internal_format,
-				      first_level, last_level, width0,
+				      levels, width0,
 				      height0, depth0, cpp, compress_byte,
 				      tiling);
    /*
@@ -157,8 +154,6 @@ struct intel_mipmap_tree *
 intel_miptree_create_for_region(struct intel_context *intel,
 				GLenum target,
 				GLenum internal_format,
-				GLuint first_level,
-				GLuint last_level,
 				struct intel_region *region,
 				GLuint depth0,
 				GLuint compress_byte)
@@ -166,7 +161,7 @@ intel_miptree_create_for_region(struct intel_context *intel,
    struct intel_mipmap_tree *mt;
 
    mt = intel_miptree_create_internal(intel, target, internal_format,
-				      first_level, last_level,
+				      1,
 				      region->width, region->height, 1,
 				      region->cpp, compress_byte,
 				      I915_TILING_NONE);
diff --git a/src/mesa/drivers/dri/intel/intel_mipmap_tree.h b/src/mesa/drivers/dri/intel/intel_mipmap_tree.h
index 21db2f4d3b3..4bb90bf4ac6 100644
--- a/src/mesa/drivers/dri/intel/intel_mipmap_tree.h
+++ b/src/mesa/drivers/dri/intel/intel_mipmap_tree.h
@@ -93,8 +93,7 @@ struct intel_mipmap_tree
    GLenum target;
    GLenum internal_format;
 
-   GLuint first_level;
-   GLuint last_level;
+   GLuint levels;
 
    GLuint width0, height0, depth0; /**< Level zero image dimensions */
    GLuint cpp;
@@ -124,8 +123,7 @@ struct intel_mipmap_tree *intel_miptree_create(struct intel_context *intel,
                                                GLenum target,
                                                GLenum base_format,
                                                GLenum internal_format,
-                                               GLuint first_level,
-                                               GLuint last_level,
+                                               GLuint levels,
                                                GLuint width0,
                                                GLuint height0,
                                                GLuint depth0,
@@ -137,8 +135,6 @@ struct intel_mipmap_tree *
 intel_miptree_create_for_region(struct intel_context *intel,
 				GLenum target,
 				GLenum internal_format,
-				GLuint first_level,
-				GLuint last_level,
 				struct intel_region *region,
 				GLuint depth0,
 				GLuint compress_byte);
diff --git a/src/mesa/drivers/dri/intel/intel_reg.h b/src/mesa/drivers/dri/intel/intel_reg.h
index 955b100b212..5258699d3f9 100644
--- a/src/mesa/drivers/dri/intel/intel_reg.h
+++ b/src/mesa/drivers/dri/intel/intel_reg.h
@@ -37,6 +37,8 @@
 #define FLUSH_MAP_CACHE				(1 << 0)
 #define INHIBIT_FLUSH_RENDER_CACHE		(1 << 2)
 
+#define MI_FLUSH_DW			(CMD_MI | (0x26 << 23) | 2)
+
 /* Stalls command execution waiting for the given events to have occurred. */
 #define MI_WAIT_FOR_EVENT               (CMD_MI | (0x3 << 23))
 #define MI_WAIT_FOR_PLANE_B_FLIP        (1<<6)
diff --git a/src/mesa/drivers/dri/intel/intel_tex.c b/src/mesa/drivers/dri/intel/intel_tex.c
index 2c21ea0576e..2c3eab20fda 100644
--- a/src/mesa/drivers/dri/intel/intel_tex.c
+++ b/src/mesa/drivers/dri/intel/intel_tex.c
@@ -113,7 +113,6 @@ intelGenerateMipmap(struct gl_context *ctx, GLenum target,
 void
 intelInitTextureFuncs(struct dd_function_table *functions)
 {
-   functions->ChooseTextureFormat = intelChooseTextureFormat;
    functions->GenerateMipmap = intelGenerateMipmap;
 
    functions->NewTextureObject = intelNewTextureObject;
diff --git a/src/mesa/drivers/dri/intel/intel_tex.h b/src/mesa/drivers/dri/intel/intel_tex.h
index b638628c711..7c76bd48c60 100644
--- a/src/mesa/drivers/dri/intel/intel_tex.h
+++ b/src/mesa/drivers/dri/intel/intel_tex.h
@@ -40,8 +40,6 @@ void intelInitTextureSubImageFuncs(struct dd_function_table *functions);
 
 void intelInitTextureCopyImageFuncs(struct dd_function_table *functions);
 
-gl_format intelChooseTextureFormat(struct gl_context *ctx, GLint internalFormat,
-                                   GLenum format, GLenum type);
 GLenum intel_mesa_format_to_rb_datatype(gl_format format);
 
 void intelSetTexBuffer(__DRIcontext *pDRICtx,
@@ -67,4 +65,10 @@ void intel_tex_unmap_images(struct intel_context *intel,
 
 int intel_compressed_num_bytes(GLuint mesaFormat);
 
+struct intel_mipmap_tree *
+intel_miptree_create_for_teximage(struct intel_context *intel,
+				  struct intel_texture_object *intelObj,
+				  struct intel_texture_image *intelImage,
+				  GLboolean expect_accelerated_upload);
+
 #endif
diff --git a/src/mesa/drivers/dri/intel/intel_tex_copy.c b/src/mesa/drivers/dri/intel/intel_tex_copy.c
index c6bc3d962ab..a40011ab40c 100644
--- a/src/mesa/drivers/dri/intel/intel_tex_copy.c
+++ b/src/mesa/drivers/dri/intel/intel_tex_copy.c
@@ -35,7 +35,6 @@
 
 #include "intel_screen.h"
 #include "intel_context.h"
-#include "intel_buffers.h"
 #include "intel_mipmap_tree.h"
 #include "intel_regions.h"
 #include "intel_fbo.h"
diff --git a/src/mesa/drivers/dri/intel/intel_tex_format.c b/src/mesa/drivers/dri/intel/intel_tex_format.c
index c9763c9ae16..87745bc66d4 100644
--- a/src/mesa/drivers/dri/intel/intel_tex_format.c
+++ b/src/mesa/drivers/dri/intel/intel_tex_format.c
@@ -35,230 +35,6 @@ intel_mesa_format_to_rb_datatype(gl_format format)
    }
 }
 
-
-/**
- * Choose hardware texture format given the user's glTexImage parameters.
- *
- * It works out that this function is fine for all the supported
- * hardware.  However, there is still a need to map the formats onto
- * hardware descriptors.
- *
- * Note that the i915 can actually support many more formats than
- * these if we take the step of simply swizzling the colors
- * immediately after sampling...
- */
-gl_format
-intelChooseTextureFormat(struct gl_context * ctx, GLint internalFormat,
-                         GLenum format, GLenum type)
-{
-   struct intel_context *intel = intel_context(ctx);
-
-#if 0
-   printf("%s intFmt=0x%x format=0x%x type=0x%x\n",
-          __FUNCTION__, internalFormat, format, type);
-#endif
-
-   switch (internalFormat) {
-   case 4:
-   case GL_RGBA:
-   case GL_COMPRESSED_RGBA:
-      if (type == GL_UNSIGNED_SHORT_4_4_4_4_REV)
-	 return MESA_FORMAT_ARGB4444;
-      else if (type == GL_UNSIGNED_SHORT_1_5_5_5_REV)
-	 return MESA_FORMAT_ARGB1555;
-      else
-	 return MESA_FORMAT_ARGB8888;
-
-   case 3:
-   case GL_RGB:
-   case GL_COMPRESSED_RGB:
-      if (type == GL_UNSIGNED_SHORT_5_6_5)
-	 return MESA_FORMAT_RGB565;
-      else if (intel->has_xrgb_textures)
-	 return MESA_FORMAT_XRGB8888;
-      else
-	 return MESA_FORMAT_ARGB8888;
-
-   case GL_RGBA8:
-   case GL_RGB10_A2:
-   case GL_RGBA12:
-   case GL_RGBA16:
-      return MESA_FORMAT_ARGB8888;
-
-   case GL_RGBA4:
-   case GL_RGBA2:
-      return MESA_FORMAT_ARGB4444;
-
-   case GL_RGB5_A1:
-      return MESA_FORMAT_ARGB1555;
-
-   case GL_RGB8:
-   case GL_RGB10:
-   case GL_RGB12:
-   case GL_RGB16:
-      if (intel->has_xrgb_textures)
-	 return MESA_FORMAT_XRGB8888;
-      else
-	 return MESA_FORMAT_ARGB8888;
-
-   case GL_RGB5:
-   case GL_RGB4:
-   case GL_R3_G3_B2:
-      return MESA_FORMAT_RGB565;
-
-   case GL_ALPHA:
-   case GL_ALPHA4:
-   case GL_ALPHA8:
-   case GL_ALPHA12:
-   case GL_ALPHA16:
-   case GL_COMPRESSED_ALPHA:
-      return MESA_FORMAT_A8;
-
-   case 1:
-   case GL_LUMINANCE:
-   case GL_LUMINANCE4:
-   case GL_LUMINANCE8:
-   case GL_LUMINANCE12:
-   case GL_LUMINANCE16:
-   case GL_COMPRESSED_LUMINANCE:
-      return MESA_FORMAT_L8;
-
-   case GL_LUMINANCE12_ALPHA4:
-   case GL_LUMINANCE12_ALPHA12:
-   case GL_LUMINANCE16_ALPHA16:
-      /* i915 could implement this mode using MT_32BIT_RG1616.  However, this
-       * would require an extra swizzle instruction in the fragment shader to
-       * convert the { R, G, 1.0, 1.0 } to { R, R, R, G }.
-       */
-#ifndef I915
-      return MESA_FORMAT_AL1616;
-#else
-      /* FALLTHROUGH */
-#endif
-
-   case 2:
-   case GL_LUMINANCE_ALPHA:
-   case GL_LUMINANCE4_ALPHA4:
-   case GL_LUMINANCE6_ALPHA2:
-   case GL_LUMINANCE8_ALPHA8:
-   case GL_COMPRESSED_LUMINANCE_ALPHA:
-      return MESA_FORMAT_AL88;
-
-   case GL_INTENSITY:
-   case GL_INTENSITY4:
-   case GL_INTENSITY8:
-   case GL_INTENSITY12:
-   case GL_INTENSITY16:
-   case GL_COMPRESSED_INTENSITY:
-      return MESA_FORMAT_I8;
-
-   case GL_YCBCR_MESA:
-      if (type == GL_UNSIGNED_SHORT_8_8_MESA || type == GL_UNSIGNED_BYTE)
-         return MESA_FORMAT_YCBCR;
-      else
-         return MESA_FORMAT_YCBCR_REV;
-
-   case GL_COMPRESSED_RGB_FXT1_3DFX:
-      return MESA_FORMAT_RGB_FXT1;
-   case GL_COMPRESSED_RGBA_FXT1_3DFX:
-      return MESA_FORMAT_RGBA_FXT1;
-
-   case GL_RGB_S3TC:
-   case GL_RGB4_S3TC:
-   case GL_COMPRESSED_RGB_S3TC_DXT1_EXT:
-      return MESA_FORMAT_RGB_DXT1;
-
-   case GL_COMPRESSED_RGBA_S3TC_DXT1_EXT:
-      return MESA_FORMAT_RGBA_DXT1;
-
-   case GL_RGBA_S3TC:
-   case GL_RGBA4_S3TC:
-   case GL_COMPRESSED_RGBA_S3TC_DXT3_EXT:
-      return MESA_FORMAT_RGBA_DXT3;
-
-   case GL_COMPRESSED_RGBA_S3TC_DXT5_EXT:
-      return MESA_FORMAT_RGBA_DXT5;
-
-   case GL_DEPTH_COMPONENT:
-   case GL_DEPTH_COMPONENT16:
-   case GL_DEPTH_COMPONENT24:
-   case GL_DEPTH_COMPONENT32:
-#if 0
-      return MESA_FORMAT_Z16;
-#else
-      /* fall-through.
-       * 16bpp depth texture can't be paired with a stencil buffer so
-       * always used combined depth/stencil format.
-       */
-#endif
-   case GL_DEPTH_STENCIL_EXT:
-   case GL_DEPTH24_STENCIL8_EXT:
-      return MESA_FORMAT_S8_Z24;
-
-#ifndef I915
-   case GL_SRGB_EXT:
-   case GL_SRGB8_EXT:
-   case GL_SRGB_ALPHA_EXT:
-   case GL_SRGB8_ALPHA8_EXT:
-   case GL_COMPRESSED_SRGB_EXT:
-   case GL_COMPRESSED_SRGB_ALPHA_EXT:
-   case GL_COMPRESSED_SLUMINANCE_EXT:
-   case GL_COMPRESSED_SLUMINANCE_ALPHA_EXT:
-      return MESA_FORMAT_SARGB8;
-   case GL_SLUMINANCE_EXT:
-   case GL_SLUMINANCE8_EXT:
-      if (intel->has_luminance_srgb)
-         return MESA_FORMAT_SL8;
-      else
-         return MESA_FORMAT_SARGB8;
-   case GL_SLUMINANCE_ALPHA_EXT:
-   case GL_SLUMINANCE8_ALPHA8_EXT:
-      if (intel->has_luminance_srgb)
-         return MESA_FORMAT_SLA8;
-      else
-         return MESA_FORMAT_SARGB8;
-   case GL_COMPRESSED_SRGB_S3TC_DXT1_EXT:
-   case GL_COMPRESSED_SRGB_ALPHA_S3TC_DXT1_EXT:
-   case GL_COMPRESSED_SRGB_ALPHA_S3TC_DXT3_EXT:
-   case GL_COMPRESSED_SRGB_ALPHA_S3TC_DXT5_EXT:
-      return MESA_FORMAT_SRGB_DXT1;
-
-   /* i915 could also do this */
-   case GL_DUDV_ATI:
-   case GL_DU8DV8_ATI:
-      return MESA_FORMAT_DUDV8;
-   case GL_RGBA_SNORM:
-   case GL_RGBA8_SNORM:
-      return MESA_FORMAT_SIGNED_RGBA8888_REV;
-
-   /* i915 can do a RG16, but it can't do any of the other RED or RG formats.
-    * In addition, it only implements the broken D3D mode where undefined
-    * components are read as 1.0.  I'm not sure who thought reading
-    * { R, G, 1.0, 1.0 } from a red-green texture would be useful.
-    */
-   case GL_RED:
-   case GL_COMPRESSED_RED:
-   case GL_R8:
-      return MESA_FORMAT_R8;
-   case GL_R16:
-      return MESA_FORMAT_R16;
-   case GL_RG:
-   case GL_COMPRESSED_RG:
-   case GL_RG8:
-      return MESA_FORMAT_RG88;
-   case GL_RG16:
-      return MESA_FORMAT_RG1616;
-#endif
-
-   default:
-      fprintf(stderr, "unexpected texture format %s in %s\n",
-              _mesa_lookup_enum_by_nr(internalFormat), __FUNCTION__);
-      return MESA_FORMAT_NONE;
-   }
-
-   return MESA_FORMAT_NONE;       /* never get here */
-}
-
 int intel_compressed_num_bytes(GLuint mesaFormat)
 {
    GLuint bw, bh;
diff --git a/src/mesa/drivers/dri/intel/intel_tex_image.c b/src/mesa/drivers/dri/intel/intel_tex_image.c
index 41cdbfd2cbd..e0d4ca762f5 100644
--- a/src/mesa/drivers/dri/intel/intel_tex_image.c
+++ b/src/mesa/drivers/dri/intel/intel_tex_image.c
@@ -44,24 +44,12 @@ logbase2(int n)
    return log2;
 }
 
-
-/* Otherwise, store it in memory if (Border != 0) or (any dimension ==
- * 1).
- *    
- * Otherwise, if max_level >= level >= min_level, create tree with
- * space for textures from min_level down to max_level.
- *
- * Otherwise, create tree with space for textures from (level
- * 0)..(1x1).  Consider pruning this tree at a validation if the
- * saving is worth it.
- */
-static void
-guess_and_alloc_mipmap_tree(struct intel_context *intel,
-                            struct intel_texture_object *intelObj,
-                            struct intel_texture_image *intelImage,
-			    GLboolean expect_accelerated_upload)
+struct intel_mipmap_tree *
+intel_miptree_create_for_teximage(struct intel_context *intel,
+				  struct intel_texture_object *intelObj,
+				  struct intel_texture_image *intelImage,
+				  GLboolean expect_accelerated_upload)
 {
-   GLuint firstLevel;
    GLuint lastLevel;
    GLuint width = intelImage->base.Width;
    GLuint height = intelImage->base.Height;
@@ -72,28 +60,11 @@ guess_and_alloc_mipmap_tree(struct intel_context *intel,
    DBG("%s\n", __FUNCTION__);
 
    if (intelImage->base.Border)
-      return;
-
-   if (intelImage->level > intelObj->base.BaseLevel &&
-       (intelImage->base.Width == 1 ||
-        (intelObj->base.Target != GL_TEXTURE_1D &&
-         intelImage->base.Height == 1) ||
-        (intelObj->base.Target == GL_TEXTURE_3D &&
-         intelImage->base.Depth == 1)))
-      return;
-
-   /* If this image disrespects BaseLevel, allocate from level zero.
-    * Usually BaseLevel == 0, so it's unlikely to happen.
-    */
-   if (intelImage->level < intelObj->base.BaseLevel)
-      firstLevel = 0;
-   else
-      firstLevel = intelObj->base.BaseLevel;
-
+      return NULL;
 
    /* Figure out image dimensions at start level. 
     */
-   for (i = intelImage->level; i > firstLevel; i--) {
+   for (i = intelImage->level; i > 0; i--) {
       width <<= 1;
       if (height != 1)
          height <<= 1;
@@ -108,34 +79,29 @@ guess_and_alloc_mipmap_tree(struct intel_context *intel,
     */
    if ((intelObj->base.MinFilter == GL_NEAREST ||
         intelObj->base.MinFilter == GL_LINEAR) &&
-       intelImage->level == firstLevel &&
-       (intel->gen < 4 || firstLevel == 0)) {
-      lastLevel = firstLevel;
+       intelImage->level == 0) {
+      lastLevel = 0;
    }
    else {
-      lastLevel = firstLevel + logbase2(MAX2(MAX2(width, height), depth));
+      lastLevel = logbase2(MAX2(MAX2(width, height), depth));
    }
 
-   assert(!intelObj->mt);
    if (_mesa_is_format_compressed(intelImage->base.TexFormat))
       comp_byte = intel_compressed_num_bytes(intelImage->base.TexFormat);
 
    texelBytes = _mesa_get_format_bytes(intelImage->base.TexFormat);
 
-   intelObj->mt = intel_miptree_create(intel,
-                                       intelObj->base.Target,
-                                       intelImage->base._BaseFormat,
-                                       intelImage->base.InternalFormat,
-                                       firstLevel,
-                                       lastLevel,
-                                       width,
-                                       height,
-                                       depth,
-                                       texelBytes,
-                                       comp_byte,
-				       expect_accelerated_upload);
-
-   DBG("%s - success\n", __FUNCTION__);
+   return intel_miptree_create(intel,
+			       intelObj->base.Target,
+			       intelImage->base._BaseFormat,
+			       intelImage->base.InternalFormat,
+			       lastLevel + 1,
+			       width,
+			       height,
+			       depth,
+			       texelBytes,
+			       comp_byte,
+			       expect_accelerated_upload);
 }
 
 
@@ -343,41 +309,23 @@ intelTexImage(struct gl_context * ctx,
       texImage->Data = NULL;
    }
 
-   if (!intelObj->mt) {
-      guess_and_alloc_mipmap_tree(intel, intelObj, intelImage, pixels == NULL);
-      if (!intelObj->mt) {
-	 DBG("guess_and_alloc_mipmap_tree: failed\n");
-      }
-   }
-
-   assert(!intelImage->mt);
-
-   if (intelObj->mt &&
-       intel_miptree_match_image(intelObj->mt, &intelImage->base)) {
-
+   if (intelObj->mt && intel_miptree_match_image(intelObj->mt,
+						 &intelImage->base)) {
       intel_miptree_reference(&intelImage->mt, intelObj->mt);
-      assert(intelImage->mt);
-   } else if (intelImage->base.Border == 0) {
-      int comp_byte = 0;
-      GLuint texelBytes = _mesa_get_format_bytes(intelImage->base.TexFormat);
-      GLenum baseFormat = _mesa_get_format_base_format(intelImage->base.TexFormat);
-      if (_mesa_is_format_compressed(intelImage->base.TexFormat)) {
-	 comp_byte =
-	    intel_compressed_num_bytes(intelImage->base.TexFormat);
+   } else {
+      intel_miptree_release(intel, &intelImage->mt);
+      intelImage->mt = intel_miptree_create_for_teximage(intel, intelObj,
+							 intelImage,
+							 pixels == NULL);
+      if (!intelImage->mt) {
+	 DBG("guess_and_alloc_mipmap_tree: failed\n");
       }
 
-      /* Didn't fit in the object miptree, but it's suitable for inclusion in
-       * a miptree, so create one just for our level and store it in the image.
-       * It'll get moved into the object miptree at validate time.
+      /* Speculatively set up the object with this miptree so that the
+       * later levels can just load into the miptree we just made.
        */
-      intelImage->mt = intel_miptree_create(intel, target,
-					    baseFormat,
-					    internalFormat,
-					    level, level,
-					    width, height, depth,
-					    texelBytes,
-					    comp_byte, pixels == NULL);
-
+      if (!intelObj->mt && intelImage->mt)
+	 intel_miptree_reference(&intelObj->mt, intelImage->mt);
    }
 
    /* PBO fastpaths:
@@ -396,10 +344,7 @@ intelTexImage(struct gl_context * ctx,
        * performance (in particular when intel_region_cow() is
        * required).
        */
-      if (intelObj->mt == intelImage->mt &&
-          intelObj->mt->first_level == level &&
-          intelObj->mt->last_level == level) {
-
+      if (intelImage->mt->levels == 1) {
          if (try_pbo_zcopy(intel, intelImage, unpack,
                            internalFormat,
                            width, height, format, type, pixels)) {
@@ -711,8 +656,7 @@ intelSetTexBuffer2(__DRIcontext *pDRICtx, GLint target,
    }
 
    mt = intel_miptree_create_for_region(intel, target,
-					internalFormat,
-					0, 0, rb->region, 1, 0);
+					internalFormat, rb->region, 1, 0);
    if (mt == NULL)
        return;
 
@@ -777,7 +721,7 @@ intel_image_target_texture_2d(struct gl_context *ctx, GLenum target,
 
    mt = intel_miptree_create_for_region(intel, target,
 					image->internal_format,
-					0, 0, image->region, 1, 0);
+					image->region, 1, 0);
    if (mt == NULL)
        return;
 
diff --git a/src/mesa/drivers/dri/intel/intel_tex_layout.c b/src/mesa/drivers/dri/intel/intel_tex_layout.c
index d39733b6c5a..540ef36a415 100644
--- a/src/mesa/drivers/dri/intel/intel_tex_layout.c
+++ b/src/mesa/drivers/dri/intel/intel_tex_layout.c
@@ -86,7 +86,7 @@ void i945_miptree_layout_2d(struct intel_context *intel,
     * constraints of mipmap placement push the right edge of the
     * 2nd mipmap out past the width of its parent.
     */
-   if (mt->first_level != mt->last_level) {
+   if (mt->levels > 1) {
        GLuint mip1_width;
 
        if (mt->compressed) {
@@ -104,7 +104,7 @@ void i945_miptree_layout_2d(struct intel_context *intel,
 
    mt->total_height = 0;
 
-   for ( level = mt->first_level ; level <= mt->last_level ; level++ ) {
+   for (level = 0; level < mt->levels; level++) {
       GLuint img_height;
 
       intel_miptree_set_level_info(mt, level, nr_images, x, y, width,
@@ -123,7 +123,7 @@ void i945_miptree_layout_2d(struct intel_context *intel,
 
       /* Layout_below: step right after second mipmap.
        */
-      if (level == mt->first_level + 1) {
+      if (level == 1) {
 	 x += ALIGN(width, align_w);
       }
       else {
diff --git a/src/mesa/drivers/dri/intel/intel_tex_obj.h b/src/mesa/drivers/dri/intel/intel_tex_obj.h
index 5f60e0ea4f3..e93ef4a4727 100644
--- a/src/mesa/drivers/dri/intel/intel_tex_obj.h
+++ b/src/mesa/drivers/dri/intel/intel_tex_obj.h
@@ -32,11 +32,11 @@ struct intel_texture_object
 {
    struct gl_texture_object base;       /* The "parent" object */
 
-   /* The mipmap tree must include at least these levels once
-    * validated:
+   /* This is a mirror of base._MaxLevel, updated at validate time,
+    * except that we don't bother with the non-base levels for
+    * non-mipmapped textures.
     */
-   GLuint firstLevel;
-   GLuint lastLevel;
+   unsigned int _MaxLevel;
 
    /* Offset for firstLevel image:
     */
diff --git a/src/mesa/drivers/dri/intel/intel_tex_validate.c b/src/mesa/drivers/dri/intel/intel_tex_validate.c
index ed5c5d896b9..31c0a83ae34 100644
--- a/src/mesa/drivers/dri/intel/intel_tex_validate.c
+++ b/src/mesa/drivers/dri/intel/intel_tex_validate.c
@@ -8,72 +8,21 @@
 #define FILE_DEBUG_FLAG DEBUG_TEXTURE
 
 /**
- * Compute which mipmap levels that really need to be sent to the hardware.
- * This depends on the base image size, GL_TEXTURE_MIN_LOD,
- * GL_TEXTURE_MAX_LOD, GL_TEXTURE_BASE_LEVEL, and GL_TEXTURE_MAX_LEVEL.
+ * When validating, we only care about the texture images that could
+ * be seen, so for non-mipmapped modes we want to ignore everything
+ * but BaseLevel.
  */
 static void
-intel_calculate_first_last_level(struct intel_context *intel,
-				 struct intel_texture_object *intelObj)
+intel_update_max_level(struct intel_context *intel,
+		       struct intel_texture_object *intelObj)
 {
    struct gl_texture_object *tObj = &intelObj->base;
-   const struct gl_texture_image *const baseImage =
-      tObj->Image[0][tObj->BaseLevel];
 
-   /* These must be signed values.  MinLod and MaxLod can be negative numbers,
-    * and having firstLevel and lastLevel as signed prevents the need for
-    * extra sign checks.
-    */
-   int firstLevel;
-   int lastLevel;
-
-   /* Yes, this looks overly complicated, but it's all needed.
-    */
-   switch (tObj->Target) {
-   case GL_TEXTURE_1D:
-   case GL_TEXTURE_2D:
-   case GL_TEXTURE_3D:
-   case GL_TEXTURE_CUBE_MAP:
-      if (tObj->MinFilter == GL_NEAREST || tObj->MinFilter == GL_LINEAR) {
-         /* GL_NEAREST and GL_LINEAR only care about GL_TEXTURE_BASE_LEVEL.
-          */
-         firstLevel = lastLevel = tObj->BaseLevel;
-      }
-      else {
-	 if (intel->gen == 2) {
-	    firstLevel = tObj->BaseLevel + (GLint) (tObj->MinLod + 0.5);
-	    firstLevel = MAX2(firstLevel, tObj->BaseLevel);
-	    firstLevel = MIN2(firstLevel, tObj->BaseLevel + baseImage->MaxLog2);
-	    lastLevel = tObj->BaseLevel + (GLint) (tObj->MaxLod + 0.5);
-	    lastLevel = MAX2(lastLevel, tObj->BaseLevel);
-	    lastLevel = MIN2(lastLevel, tObj->BaseLevel + baseImage->MaxLog2);
-	    lastLevel = MIN2(lastLevel, tObj->MaxLevel);
-	    lastLevel = MAX2(firstLevel, lastLevel);       /* need at least one level */
-	 } else {
-	    /* Min/max LOD are taken into account in sampler state.  We don't
-	     * want to re-layout textures just because clamping has been applied
-	     * since it means a bunch of blitting around and probably no memory
-	     * savings (since we have to keep the other levels around anyway).
-	     */
-	    firstLevel = tObj->BaseLevel;
-	    lastLevel = MIN2(tObj->BaseLevel + baseImage->MaxLog2,
-			     tObj->MaxLevel);
-	    /* need at least one level */
-	    lastLevel = MAX2(firstLevel, lastLevel);
-	 }
-      }
-      break;
-   case GL_TEXTURE_RECTANGLE_NV:
-   case GL_TEXTURE_4D_SGIS:
-      firstLevel = lastLevel = 0;
-      break;
-   default:
-      return;
+   if (tObj->MinFilter == GL_NEAREST || tObj->MinFilter == GL_LINEAR) {
+      intelObj->_MaxLevel = tObj->BaseLevel;
+   } else {
+      intelObj->_MaxLevel = tObj->_MaxLevel;
    }
-
-   /* save these values */
-   intelObj->firstLevel = firstLevel;
-   intelObj->lastLevel = lastLevel;
 }
 
 /**
@@ -135,8 +84,8 @@ intel_finalize_mipmap_tree(struct intel_context *intel, GLuint unit)
 
    /* What levels must the tree include at a minimum?
     */
-   intel_calculate_first_last_level(intel, intelObj);
-   firstImage = intel_texture_image(tObj->Image[0][intelObj->firstLevel]);
+   intel_update_max_level(intel, intelObj);
+   firstImage = intel_texture_image(tObj->Image[0][tObj->BaseLevel]);
 
    /* Fallback case:
     */
@@ -155,8 +104,7 @@ intel_finalize_mipmap_tree(struct intel_context *intel, GLuint unit)
     */
    if (firstImage->mt &&
        firstImage->mt != intelObj->mt &&
-       firstImage->mt->first_level <= intelObj->firstLevel &&
-       firstImage->mt->last_level >= intelObj->lastLevel) {
+       firstImage->mt->levels >= intelObj->_MaxLevel) {
 
       if (intelObj->mt)
          intel_miptree_release(intel, &intelObj->mt);
@@ -183,11 +131,10 @@ intel_finalize_mipmap_tree(struct intel_context *intel, GLuint unit)
    if (intelObj->mt &&
        (intelObj->mt->target != intelObj->base.Target ||
 	intelObj->mt->internal_format != firstImage->base.InternalFormat ||
-	intelObj->mt->first_level != intelObj->firstLevel ||
-	intelObj->mt->last_level != intelObj->lastLevel ||
-	intelObj->mt->width0 != firstImage->base.Width ||
-	intelObj->mt->height0 != firstImage->base.Height ||
-	intelObj->mt->depth0 != firstImage->base.Depth ||
+	intelObj->mt->levels <= intelObj->_MaxLevel ||
+	intelObj->mt->width0 != firstImage->mt->width0 ||
+	intelObj->mt->height0 != firstImage->mt->height0 ||
+	intelObj->mt->depth0 != firstImage->mt->depth0 ||
 	intelObj->mt->cpp != cpp ||
 	intelObj->mt->compressed != _mesa_is_format_compressed(firstImage->base.TexFormat))) {
       intel_miptree_release(intel, &intelObj->mt);
@@ -197,29 +144,22 @@ intel_finalize_mipmap_tree(struct intel_context *intel, GLuint unit)
    /* May need to create a new tree:
     */
    if (!intelObj->mt) {
-      intelObj->mt = intel_miptree_create(intel,
-                                          intelObj->base.Target,
-                                          firstImage->base._BaseFormat,
-                                          firstImage->base.InternalFormat,
-                                          intelObj->firstLevel,
-                                          intelObj->lastLevel,
-                                          firstImage->base.Width,
-                                          firstImage->base.Height,
-                                          firstImage->base.Depth,
-                                          cpp,
-                                          comp_byte,
-					  GL_TRUE);
+      intelObj->mt = intel_miptree_create_for_teximage(intel, intelObj,
+						       firstImage,
+						       GL_TRUE);
    }
 
    /* Pull in any images not in the object's tree:
     */
    nr_faces = (intelObj->base.Target == GL_TEXTURE_CUBE_MAP) ? 6 : 1;
    for (face = 0; face < nr_faces; face++) {
-      for (i = intelObj->firstLevel; i <= intelObj->lastLevel; i++) {
+      for (i = tObj->BaseLevel; i <= intelObj->_MaxLevel; i++) {
          struct intel_texture_image *intelImage =
             intel_texture_image(intelObj->base.Image[face][i]);
-
-         /* Need to import images in main memory or held in other trees.
+	 /* skip too small size mipmap */
+ 	 if (intelImage == NULL)
+		 break;
+	 /* Need to import images in main memory or held in other trees.
 	  * If it's a render target, then its data isn't needed to be in
 	  * the object tree (otherwise we'd be FBO incomplete), and we need
 	  * to keep track of the image's MT as needing to be pulled in still,
@@ -289,7 +229,7 @@ intel_tex_map_images(struct intel_context *intel,
 
    DBG("%s\n", __FUNCTION__);
 
-   for (i = intelObj->firstLevel; i <= intelObj->lastLevel; i++)
+   for (i = intelObj->base.BaseLevel; i <= intelObj->_MaxLevel; i++)
       intel_tex_map_level_images(intel, intelObj, i);
 }
 
@@ -299,6 +239,6 @@ intel_tex_unmap_images(struct intel_context *intel,
 {
    int i;
 
-   for (i = intelObj->firstLevel; i <= intelObj->lastLevel; i++)
+   for (i = intelObj->base.BaseLevel; i <= intelObj->_MaxLevel; i++)
       intel_tex_unmap_level_images(intel, intelObj, i);
 }
diff --git a/src/mesa/drivers/dri/mach64/mach64_context.h b/src/mesa/drivers/dri/mach64/mach64_context.h
index 11e8f53b283..70bc0ae79de 100644
--- a/src/mesa/drivers/dri/mach64/mach64_context.h
+++ b/src/mesa/drivers/dri/mach64/mach64_context.h
@@ -295,11 +295,11 @@ extern GLboolean mach64UnbindContext( __DRIcontext *driContextPriv );
 #define LE32_OUT( x, y )	do { *(GLuint *)(x) = (y); } while (0)
 #define LE32_OUT_FLOAT( x, y )	do { *(GLfloat *)(x) = (y); } while (0)
 #else
-#ifndef __OpenBSD__
-#include <byteswap.h>
-#else
+#if defined(__OpenBSD__) || defined(__NetBSD__)
 #include <machine/endian.h>
 #define bswap_32 bswap32
+#else
+#include <byteswap.h>
 #endif
 
 #define LE32_IN( x )		bswap_32( *(GLuint *)(x) )
diff --git a/src/mesa/drivers/dri/r300/compiler/radeon_compiler_util.c b/src/mesa/drivers/dri/r300/compiler/radeon_compiler_util.c
index bf393a9fb16..2482fc68beb 100644
--- a/src/mesa/drivers/dri/r300/compiler/radeon_compiler_util.c
+++ b/src/mesa/drivers/dri/r300/compiler/radeon_compiler_util.c
@@ -223,7 +223,6 @@ unsigned int rc_inst_can_use_presub(
 {
 	struct can_use_presub_data d;
 	unsigned int num_presub_srcs;
-	unsigned int presub_src_type = rc_source_type_mask(presub_writemask);
 	const struct rc_opcode_info * info =
 					rc_get_opcode_info(inst->U.I.Opcode);
 
@@ -253,13 +252,7 @@ unsigned int rc_inst_can_use_presub(
 
 	num_presub_srcs = rc_presubtract_src_reg_count(presub_op);
 
-	if ((presub_src_type & RC_SOURCE_RGB)
-					&& d.RGBCount + num_presub_srcs > 3) {
-		return 0;
-	}
-
-	if ((presub_src_type & RC_SOURCE_ALPHA)
-					&& d.AlphaCount + num_presub_srcs > 3) {
+	if (d.RGBCount + num_presub_srcs > 3 || d.AlphaCount + num_presub_srcs > 3) {
 		return 0;
 	}
 
diff --git a/src/mesa/drivers/dri/r300/compiler/radeon_rename_regs.c b/src/mesa/drivers/dri/r300/compiler/radeon_rename_regs.c
index 88165f78953..5bd19c0b9c6 100644
--- a/src/mesa/drivers/dri/r300/compiler/radeon_rename_regs.c
+++ b/src/mesa/drivers/dri/r300/compiler/radeon_rename_regs.c
@@ -51,6 +51,14 @@ void rc_rename_regs(struct radeon_compiler *c, void *user)
 	struct rc_reader_data reader_data;
 	unsigned char * used;
 
+	/* XXX Remove this once the register allocation works with flow control. */
+	for(inst = c->Program.Instructions.Next;
+					inst != &c->Program.Instructions;
+					inst = inst->Next) {
+		if (inst->U.I.Opcode == RC_OPCODE_BGNLOOP)
+			return;
+	}
+
 	used_length = 2 * rc_recompute_ips(c);
 	used = memory_pool_malloc(&c->Pool, sizeof(unsigned char) * used_length);
 	memset(used, 0, sizeof(unsigned char) * used_length);
diff --git a/src/mesa/drivers/dri/r600/evergreen_blit.c b/src/mesa/drivers/dri/r600/evergreen_blit.c
index fc9fa9d22c7..e07da8c15b4 100644
--- a/src/mesa/drivers/dri/r600/evergreen_blit.c
+++ b/src/mesa/drivers/dri/r600/evergreen_blit.c
@@ -1406,9 +1406,95 @@ eg_set_default_state(context_t *context)
 	    num_hs_stack_entries = 85;
 	    num_ls_stack_entries = 85;
 	    break;
+    case CHIP_FAMILY_PALM:
+	    num_ps_gprs = 93;
+	    num_vs_gprs = 46;
+	    num_temp_gprs = 4;
+	    num_gs_gprs = 31;
+	    num_es_gprs = 31;
+	    num_hs_gprs = 23;
+	    num_ls_gprs = 23;
+	    num_ps_threads = 96;
+	    num_vs_threads = 16;
+	    num_gs_threads = 16;
+	    num_es_threads = 16;
+	    num_hs_threads = 16;
+	    num_ls_threads = 16;
+	    num_ps_stack_entries = 42;
+	    num_vs_stack_entries = 42;
+	    num_gs_stack_entries = 42;
+	    num_es_stack_entries = 42;
+	    num_hs_stack_entries = 42;
+	    num_ls_stack_entries = 42;
+	    break;
+    case CHIP_FAMILY_BARTS:
+	    num_ps_gprs = 93;
+	    num_vs_gprs = 46;
+	    num_temp_gprs = 4;
+	    num_gs_gprs = 31;
+	    num_es_gprs = 31;
+	    num_hs_gprs = 23;
+	    num_ls_gprs = 23;
+	    num_ps_threads = 128;
+	    num_vs_threads = 20;
+	    num_gs_threads = 20;
+	    num_es_threads = 20;
+	    num_hs_threads = 20;
+	    num_ls_threads = 20;
+	    num_ps_stack_entries = 85;
+	    num_vs_stack_entries = 85;
+	    num_gs_stack_entries = 85;
+	    num_es_stack_entries = 85;
+	    num_hs_stack_entries = 85;
+	    num_ls_stack_entries = 85;
+	    break;
+    case CHIP_FAMILY_TURKS:
+	    num_ps_gprs = 93;
+	    num_vs_gprs = 46;
+	    num_temp_gprs = 4;
+	    num_gs_gprs = 31;
+	    num_es_gprs = 31;
+	    num_hs_gprs = 23;
+	    num_ls_gprs = 23;
+	    num_ps_threads = 128;
+	    num_vs_threads = 20;
+	    num_gs_threads = 20;
+	    num_es_threads = 20;
+	    num_hs_threads = 20;
+	    num_ls_threads = 20;
+	    num_ps_stack_entries = 42;
+	    num_vs_stack_entries = 42;
+	    num_gs_stack_entries = 42;
+	    num_es_stack_entries = 42;
+	    num_hs_stack_entries = 42;
+	    num_ls_stack_entries = 42;
+	    break;
+    case CHIP_FAMILY_CAICOS:
+	    num_ps_gprs = 93;
+	    num_vs_gprs = 46;
+	    num_temp_gprs = 4;
+	    num_gs_gprs = 31;
+	    num_es_gprs = 31;
+	    num_hs_gprs = 23;
+	    num_ls_gprs = 23;
+	    num_ps_threads = 128;
+	    num_vs_threads = 10;
+	    num_gs_threads = 10;
+	    num_es_threads = 10;
+	    num_hs_threads = 10;
+	    num_ls_threads = 10;
+	    num_ps_stack_entries = 42;
+	    num_vs_stack_entries = 42;
+	    num_gs_stack_entries = 42;
+	    num_es_stack_entries = 42;
+	    num_hs_stack_entries = 42;
+	    num_ls_stack_entries = 42;
+	    break;
     }
 
-    if (context->radeon.radeonScreen->chip_family == CHIP_FAMILY_CEDAR)
+    if ((context->radeon.radeonScreen->chip_family == CHIP_FAMILY_CEDAR) ||
+	(context->radeon.radeonScreen->chip_family == CHIP_FAMILY_PALM) ||
+	(context->radeon.radeonScreen->chip_family == CHIP_FAMILY_CAICOS))
 	    CLEARbit(sq_config, EG_SQ_CONFIG__VC_ENABLE_bit);
     else
 	    SETbit(sq_config, EG_SQ_CONFIG__VC_ENABLE_bit);
diff --git a/src/mesa/drivers/dri/r600/evergreen_state.c b/src/mesa/drivers/dri/r600/evergreen_state.c
index 076a608573c..648cda0078e 100644
--- a/src/mesa/drivers/dri/r600/evergreen_state.c
+++ b/src/mesa/drivers/dri/r600/evergreen_state.c
@@ -1469,6 +1469,30 @@ static void evergreenInitSQConfig(struct gl_context * ctx)
         uMaxThreads = 192;
         uMaxStackEntries = 256;
 	    break;
+    case CHIP_FAMILY_BARTS:
+	    uSqNumCfInsts       = 2;
+        bVC_ENABLE = GL_TRUE;
+        uMaxGPRs = 256;
+        uPSThreadCount = 128;
+        uMaxThreads = 248;
+        uMaxStackEntries = 512;
+	    break;
+    case CHIP_FAMILY_TURKS:
+	    uSqNumCfInsts       = 2;
+        bVC_ENABLE = GL_TRUE;
+        uMaxGPRs = 256;
+        uPSThreadCount = 128;
+        uMaxThreads = 248;
+        uMaxStackEntries = 256;
+	    break;
+    case CHIP_FAMILY_CAICOS:
+	    uSqNumCfInsts       = 1;
+        bVC_ENABLE = GL_FALSE;
+        uMaxGPRs = 256;
+        uPSThreadCount = 128;
+        uMaxThreads = 192;
+        uMaxStackEntries = 256;
+	    break;
     default:
         uSqNumCfInsts       = 2;
         bVC_ENABLE = GL_TRUE;
diff --git a/src/mesa/drivers/dri/r600/r600_context.c b/src/mesa/drivers/dri/r600/r600_context.c
index aa1891eac32..5f5a50286d6 100644
--- a/src/mesa/drivers/dri/r600/r600_context.c
+++ b/src/mesa/drivers/dri/r600/r600_context.c
@@ -259,7 +259,7 @@ static void r600InitConstValues(struct gl_context *ctx, radeonScreenPtr screen)
     R700_CHIP_CONTEXT *r700    = (R700_CHIP_CONTEXT*)(&context->hw);
 
     if(  (context->radeon.radeonScreen->chip_family >= CHIP_FAMILY_CEDAR)
-       &&(context->radeon.radeonScreen->chip_family <= CHIP_FAMILY_PALM) )
+       &&(context->radeon.radeonScreen->chip_family <= CHIP_FAMILY_CAICOS) )
     {
         r700->bShaderUseMemConstant = GL_TRUE;
     }
diff --git a/src/mesa/drivers/dri/radeon/radeon_bocs_wrapper.h b/src/mesa/drivers/dri/radeon/radeon_bocs_wrapper.h
index 6c2648b6bd8..60f10496026 100644
--- a/src/mesa/drivers/dri/radeon/radeon_bocs_wrapper.h
+++ b/src/mesa/drivers/dri/radeon/radeon_bocs_wrapper.h
@@ -67,6 +67,9 @@ struct drm_radeon_info {
 #define DRM_RADEON_INFO 0x1
 #endif
 
+static inline void radeon_gem_get_kernel_name(struct radeon_bo *dummy, uint32_t *value)
+{
+}
 
 static inline uint32_t radeon_gem_name_bo(struct radeon_bo *dummy)
 {
diff --git a/src/mesa/drivers/dri/radeon/radeon_chipset.h b/src/mesa/drivers/dri/radeon/radeon_chipset.h
index 82789cec5ed..399052cbcbd 100644
--- a/src/mesa/drivers/dri/radeon/radeon_chipset.h
+++ b/src/mesa/drivers/dri/radeon/radeon_chipset.h
@@ -445,6 +445,45 @@
 #define PCI_CHIP_PALM_9804              0x9804
 #define PCI_CHIP_PALM_9805              0x9805
 
+#define PCI_CHIP_BARTS_6720             0x6720
+#define PCI_CHIP_BARTS_6721             0x6721
+#define PCI_CHIP_BARTS_6722             0x6722
+#define PCI_CHIP_BARTS_6723             0x6723
+#define PCI_CHIP_BARTS_6724             0x6724
+#define PCI_CHIP_BARTS_6725             0x6725
+#define PCI_CHIP_BARTS_6726             0x6726
+#define PCI_CHIP_BARTS_6727             0x6727
+#define PCI_CHIP_BARTS_6728             0x6728
+#define PCI_CHIP_BARTS_6729             0x6729
+#define PCI_CHIP_BARTS_6738             0x6738
+#define PCI_CHIP_BARTS_6739             0x6739
+
+#define PCI_CHIP_TURKS_6740             0x6740
+#define PCI_CHIP_TURKS_6741             0x6741
+#define PCI_CHIP_TURKS_6742             0x6742
+#define PCI_CHIP_TURKS_6743             0x6743
+#define PCI_CHIP_TURKS_6744             0x6744
+#define PCI_CHIP_TURKS_6745             0x6745
+#define PCI_CHIP_TURKS_6746             0x6746
+#define PCI_CHIP_TURKS_6747             0x6747
+#define PCI_CHIP_TURKS_6748             0x6748
+#define PCI_CHIP_TURKS_6749             0x6749
+#define PCI_CHIP_TURKS_6750             0x6750
+#define PCI_CHIP_TURKS_6758             0x6758
+#define PCI_CHIP_TURKS_6759             0x6759
+
+#define PCI_CHIP_CAICOS_6760            0x6760
+#define PCI_CHIP_CAICOS_6761            0x6761
+#define PCI_CHIP_CAICOS_6762            0x6762
+#define PCI_CHIP_CAICOS_6763            0x6763
+#define PCI_CHIP_CAICOS_6764            0x6764
+#define PCI_CHIP_CAICOS_6765            0x6765
+#define PCI_CHIP_CAICOS_6766            0x6766
+#define PCI_CHIP_CAICOS_6767            0x6767
+#define PCI_CHIP_CAICOS_6768            0x6768
+#define PCI_CHIP_CAICOS_6770            0x6770
+#define PCI_CHIP_CAICOS_6779            0x6779
+
 enum {
    CHIP_FAMILY_R100,
    CHIP_FAMILY_RV100,
@@ -489,6 +528,9 @@ enum {
    CHIP_FAMILY_CYPRESS,
    CHIP_FAMILY_HEMLOCK,
    CHIP_FAMILY_PALM,
+   CHIP_FAMILY_BARTS,
+   CHIP_FAMILY_TURKS,
+   CHIP_FAMILY_CAICOS,
    CHIP_FAMILY_LAST
 };
 
diff --git a/src/mesa/drivers/dri/radeon/radeon_common_context.c b/src/mesa/drivers/dri/radeon/radeon_common_context.c
index ca6ab46ca43..a1124483a6c 100644
--- a/src/mesa/drivers/dri/radeon/radeon_common_context.c
+++ b/src/mesa/drivers/dri/radeon/radeon_common_context.c
@@ -100,6 +100,9 @@ static const char* get_chip_family_name(int chip_family)
 	case CHIP_FAMILY_CYPRESS: return "CYPRESS";
 	case CHIP_FAMILY_HEMLOCK: return "HEMLOCK";
 	case CHIP_FAMILY_PALM: return "PALM";
+	case CHIP_FAMILY_BARTS: return "BARTS";
+	case CHIP_FAMILY_TURKS: return "TURKS";
+	case CHIP_FAMILY_CAICOS: return "CAICOS";
 	default: return "unknown";
 	}
 }
diff --git a/src/mesa/drivers/dri/radeon/radeon_screen.c b/src/mesa/drivers/dri/radeon/radeon_screen.c
index 94e56c2ade6..a35fcfe9d7f 100644
--- a/src/mesa/drivers/dri/radeon/radeon_screen.c
+++ b/src/mesa/drivers/dri/radeon/radeon_screen.c
@@ -1163,6 +1163,54 @@ static int radeon_set_screen_flags(radeonScreenPtr screen, int device_id)
        screen->chip_flags = RADEON_CHIPSET_TCL;
        break;
 
+   case PCI_CHIP_BARTS_6720:
+   case PCI_CHIP_BARTS_6721:
+   case PCI_CHIP_BARTS_6722:
+   case PCI_CHIP_BARTS_6723:
+   case PCI_CHIP_BARTS_6724:
+   case PCI_CHIP_BARTS_6725:
+   case PCI_CHIP_BARTS_6726:
+   case PCI_CHIP_BARTS_6727:
+   case PCI_CHIP_BARTS_6728:
+   case PCI_CHIP_BARTS_6729:
+   case PCI_CHIP_BARTS_6738:
+   case PCI_CHIP_BARTS_6739:
+       screen->chip_family = CHIP_FAMILY_BARTS;
+       screen->chip_flags = RADEON_CHIPSET_TCL;
+       break;
+
+   case PCI_CHIP_TURKS_6740:
+   case PCI_CHIP_TURKS_6741:
+   case PCI_CHIP_TURKS_6742:
+   case PCI_CHIP_TURKS_6743:
+   case PCI_CHIP_TURKS_6744:
+   case PCI_CHIP_TURKS_6745:
+   case PCI_CHIP_TURKS_6746:
+   case PCI_CHIP_TURKS_6747:
+   case PCI_CHIP_TURKS_6748:
+   case PCI_CHIP_TURKS_6749:
+   case PCI_CHIP_TURKS_6750:
+   case PCI_CHIP_TURKS_6758:
+   case PCI_CHIP_TURKS_6759:
+       screen->chip_family = CHIP_FAMILY_TURKS;
+       screen->chip_flags = RADEON_CHIPSET_TCL;
+       break;
+
+   case PCI_CHIP_CAICOS_6760:
+   case PCI_CHIP_CAICOS_6761:
+   case PCI_CHIP_CAICOS_6762:
+   case PCI_CHIP_CAICOS_6763:
+   case PCI_CHIP_CAICOS_6764:
+   case PCI_CHIP_CAICOS_6765:
+   case PCI_CHIP_CAICOS_6766:
+   case PCI_CHIP_CAICOS_6767:
+   case PCI_CHIP_CAICOS_6768:
+   case PCI_CHIP_CAICOS_6770:
+   case PCI_CHIP_CAICOS_6779:
+       screen->chip_family = CHIP_FAMILY_CAICOS;
+       screen->chip_flags = RADEON_CHIPSET_TCL;
+       break;
+
    default:
       fprintf(stderr, "unknown chip id 0x%x, can't guess.\n",
 	      device_id);
diff --git a/src/mesa/drivers/osmesa/osmesa.c b/src/mesa/drivers/osmesa/osmesa.c
index 10ba6b79314..e1f0c69bacf 100644
--- a/src/mesa/drivers/osmesa/osmesa.c
+++ b/src/mesa/drivers/osmesa/osmesa.c
@@ -1356,7 +1356,7 @@ OSMesaMakeCurrent( OSMesaContext osmesa, void *buffer, GLenum type,
 
 
    /* this updates the visual's red/green/blue/alphaBits fields */
-   _mesa_update_framebuffer_visual(osmesa->gl_buffer);
+   _mesa_update_framebuffer_visual(&osmesa->mesa, osmesa->gl_buffer);
 
    /* update the framebuffer size */
    _mesa_resize_framebuffer(&osmesa->mesa, osmesa->gl_buffer, width, height);
diff --git a/src/mesa/main/accum.c b/src/mesa/main/accum.c
index 9026110f3ef..940f398f023 100644
--- a/src/mesa/main/accum.c
+++ b/src/mesa/main/accum.c
@@ -27,6 +27,7 @@
 #include "context.h"
 #include "imports.h"
 #include "macros.h"
+#include "mfeatures.h"
 #include "state.h"
 #include "mtypes.h"
 #include "main/dispatch.h"
diff --git a/src/mesa/main/api_arrayelt.c b/src/mesa/main/api_arrayelt.c
index c22e18c9fbc..b429e2627e3 100644
--- a/src/mesa/main/api_arrayelt.c
+++ b/src/mesa/main/api_arrayelt.c
@@ -40,6 +40,8 @@
 #include "context.h"
 #include "imports.h"
 #include "macros.h"
+#include "mfeatures.h"
+#include "mtypes.h"
 #include "main/dispatch.h"
 
 typedef void (GLAPIENTRY *array_func)( const void * );
diff --git a/src/mesa/main/api_arrayelt.h b/src/mesa/main/api_arrayelt.h
index 610e522a943..b90d713682e 100644
--- a/src/mesa/main/api_arrayelt.h
+++ b/src/mesa/main/api_arrayelt.h
@@ -28,6 +28,7 @@
 #define API_ARRAYELT_H
 
 
+#include "main/mfeatures.h"
 #include "main/mtypes.h"
 
 #if FEATURE_arrayelt
diff --git a/src/mesa/main/api_noop.c b/src/mesa/main/api_noop.c
index 08b4b4a3b67..7d3d3327279 100644
--- a/src/mesa/main/api_noop.c
+++ b/src/mesa/main/api_noop.c
@@ -30,6 +30,7 @@
 #include "context.h"
 #include "light.h"
 #include "macros.h"
+#include "mfeatures.h"
 #include "dlist.h"
 #include "eval.h"
 #include "main/dispatch.h"
diff --git a/src/mesa/main/api_noop.h b/src/mesa/main/api_noop.h
index e7fd49bafbb..aca7f088db6 100644
--- a/src/mesa/main/api_noop.h
+++ b/src/mesa/main/api_noop.h
@@ -25,6 +25,7 @@
 #ifndef _API_NOOP_H
 #define _API_NOOP_H
 
+#include "main/mfeatures.h"
 #include "main/mtypes.h"
 
 #if FEATURE_beginend
diff --git a/src/mesa/main/arrayobj.c b/src/mesa/main/arrayobj.c
index 0d64b7de8dd..9e9728c2c62 100644
--- a/src/mesa/main/arrayobj.c
+++ b/src/mesa/main/arrayobj.c
@@ -43,11 +43,13 @@
 #include "hash.h"
 #include "imports.h"
 #include "context.h"
+#include "mfeatures.h"
 #if FEATURE_ARB_vertex_buffer_object
 #include "bufferobj.h"
 #endif
 #include "arrayobj.h"
 #include "macros.h"
+#include "mtypes.h"
 #include "main/dispatch.h"
 
 
diff --git a/src/mesa/main/atifragshader.c b/src/mesa/main/atifragshader.c
index ae2feb3229a..44097e1a266 100644
--- a/src/mesa/main/atifragshader.c
+++ b/src/mesa/main/atifragshader.c
@@ -26,6 +26,7 @@
 #include "main/hash.h"
 #include "main/imports.h"
 #include "main/macros.h"
+#include "main/mfeatures.h"
 #include "main/enums.h"
 #include "main/mtypes.h"
 #include "main/dispatch.h"
@@ -316,6 +317,7 @@ _mesa_DeleteFragmentShaderATI(GLuint id)
       if (prog) {
 	 prog->RefCount--;
 	 if (prog->RefCount <= 0) {
+	    assert(prog != &DummyShader);
 	    free(prog);
 	 }
       }
diff --git a/src/mesa/main/attrib.c b/src/mesa/main/attrib.c
index fb6fbe5b6db..adfec3b0d5a 100644
--- a/src/mesa/main/attrib.c
+++ b/src/mesa/main/attrib.c
@@ -43,6 +43,7 @@
 #include "lines.h"
 #include "macros.h"
 #include "matrix.h"
+#include "mfeatures.h"
 #include "multisample.h"
 #include "points.h"
 #include "polygon.h"
diff --git a/src/mesa/main/bufferobj.c b/src/mesa/main/bufferobj.c
index 4d62f54a95a..1f9a5212c04 100644
--- a/src/mesa/main/bufferobj.c
+++ b/src/mesa/main/bufferobj.c
@@ -39,6 +39,8 @@
 #include "context.h"
 #include "bufferobj.h"
 #include "fbobject.h"
+#include "mfeatures.h"
+#include "mtypes.h"
 #include "texobj.h"
 
 
diff --git a/src/mesa/main/bufferobj.h b/src/mesa/main/bufferobj.h
index 4b97e34767e..09ccab31742 100644
--- a/src/mesa/main/bufferobj.h
+++ b/src/mesa/main/bufferobj.h
@@ -29,6 +29,7 @@
 #define BUFFEROBJ_H
 
 
+#include "mfeatures.h"
 #include "mtypes.h"
 
 
diff --git a/src/mesa/main/buffers.c b/src/mesa/main/buffers.c
index 86446311fe3..5c37f3d1a86 100644
--- a/src/mesa/main/buffers.c
+++ b/src/mesa/main/buffers.c
@@ -35,6 +35,7 @@
 #include "colormac.h"
 #include "context.h"
 #include "enums.h"
+#include "mtypes.h"
 
 
 #define BAD_MASK ~0u
diff --git a/src/mesa/main/clear.c b/src/mesa/main/clear.c
index c4c1d96e49c..43a9ccc116a 100644
--- a/src/mesa/main/clear.c
+++ b/src/mesa/main/clear.c
@@ -36,6 +36,7 @@
 #include "colormac.h"
 #include "enums.h"
 #include "macros.h"
+#include "mtypes.h"
 #include "state.h"
 
 
diff --git a/src/mesa/main/colormac.h b/src/mesa/main/colormac.h
index 245fb658bb3..a328dcd32af 100644
--- a/src/mesa/main/colormac.h
+++ b/src/mesa/main/colormac.h
@@ -198,12 +198,23 @@ do {						\
    ((((B) & 0xf8) >> 1) | (((G) & 0xc0) >> 6) | (((G) & 0x38) << 10) | (((R) & 0xf8) << 5) |	\
     ((A) ? 0x80 : 0))
 
+#define PACK_COLOR_2101010_UB( A, B, G, R )					\
+   (((B) << 22) | ((G) << 12) | ((R) << 2) |	\
+    (((A) & 0xc0) << 24))
+
+#define PACK_COLOR_2101010_US( A, B, G, R )					\
+   ((((B) >> 6) << 20) | (((G) >> 6) << 10) | ((R) >> 6) |	\
+    (((A) >> 14) << 30))
+
 #define PACK_COLOR_4444( R, G, B, A )					\
    ((((R) & 0xf0) << 8) | (((G) & 0xf0) << 4) | ((B) & 0xf0) | ((A) >> 4))
 
 #define PACK_COLOR_4444_REV( R, G, B, A )				\
    ((((B) & 0xf0) << 8) | (((A) & 0xf0) << 4) | ((R) & 0xf0) | ((G) >> 4))
 
+#define PACK_COLOR_44( L, A )						\
+   (((L) & 0xf0) | (((A) & 0xf0) >> 4))
+
 #define PACK_COLOR_88( L, A )						\
    (((L) << 8) | (A))
 
diff --git a/src/mesa/main/colortab.c b/src/mesa/main/colortab.c
index caa95835bc2..b0ba31c732f 100644
--- a/src/mesa/main/colortab.c
+++ b/src/mesa/main/colortab.c
@@ -29,6 +29,8 @@
 #include "context.h"
 #include "image.h"
 #include "macros.h"
+#include "mfeatures.h"
+#include "mtypes.h"
 #include "pack.h"
 #include "state.h"
 #include "teximage.h"
diff --git a/src/mesa/main/condrender.c b/src/mesa/main/condrender.c
index 25b3dd678dc..352e2e2b165 100644
--- a/src/mesa/main/condrender.c
+++ b/src/mesa/main/condrender.c
@@ -34,6 +34,7 @@
 #include "glheader.h"
 #include "condrender.h"
 #include "enums.h"
+#include "mtypes.h"
 #include "queryobj.h"
 
 
diff --git a/src/mesa/main/context.c b/src/mesa/main/context.c
index f42a566c302..2c57c8d4cbb 100644
--- a/src/mesa/main/context.c
+++ b/src/mesa/main/context.c
@@ -955,6 +955,14 @@ _mesa_initialize_context_for_api(struct gl_context *ctx,
       ctx->FragmentProgram._MaintainTexEnvProgram = GL_TRUE;
    }
 
+   /* Mesa core handles all the formats that mesa core knows about.
+    * Drivers will want to override this list with just the formats
+    * they can handle, and confirm that appropriate fallbacks exist in
+    * _mesa_choose_tex_format().
+    */
+   memset(&ctx->TextureFormatSupported, GL_TRUE,
+	  sizeof(ctx->TextureFormatSupported));
+
    switch (ctx->API) {
    case API_OPENGL:
 #if FEATURE_dlist
diff --git a/src/mesa/main/convolve.c b/src/mesa/main/convolve.c
index 0231bcf6d3f..5d286eb316b 100644
--- a/src/mesa/main/convolve.c
+++ b/src/mesa/main/convolve.c
@@ -36,6 +36,7 @@
 #include "colormac.h"
 #include "convolve.h"
 #include "macros.h"
+#include "mfeatures.h"
 #include "mtypes.h"
 #include "main/dispatch.h"
 
diff --git a/src/mesa/main/cpuinfo.c b/src/mesa/main/cpuinfo.c
index 35de69b180f..6363512961a 100644
--- a/src/mesa/main/cpuinfo.c
+++ b/src/mesa/main/cpuinfo.c
@@ -23,7 +23,7 @@
  */
 
 
-#include "main/imports.h"
+#include "main/compiler.h"
 #include "main/cpuinfo.h"
 
 
diff --git a/src/mesa/main/dlist.h b/src/mesa/main/dlist.h
index 24241a4bd4f..160ed62f557 100644
--- a/src/mesa/main/dlist.h
+++ b/src/mesa/main/dlist.h
@@ -33,6 +33,7 @@
 #define DLIST_H
 
 
+#include "main/mfeatures.h"
 #include "main/mtypes.h"
 
 
diff --git a/src/mesa/main/drawpix.c b/src/mesa/main/drawpix.c
index 757aae6e703..6fda3c5665c 100644
--- a/src/mesa/main/drawpix.c
+++ b/src/mesa/main/drawpix.c
@@ -30,6 +30,7 @@
 #include "enums.h"
 #include "feedback.h"
 #include "framebuffer.h"
+#include "mfeatures.h"
 #include "readpix.h"
 #include "state.h"
 #include "dispatch.h"
diff --git a/src/mesa/main/drawtex.c b/src/mesa/main/drawtex.c
index b9afc9974e0..2089cdfcef9 100644
--- a/src/mesa/main/drawtex.c
+++ b/src/mesa/main/drawtex.c
@@ -24,6 +24,8 @@
 #include "main/drawtex.h"
 #include "main/state.h"
 #include "main/imports.h"
+#include "main/mfeatures.h"
+#include "main/mtypes.h"
 
 
 #if FEATURE_OES_draw_texture
diff --git a/src/mesa/main/enable.c b/src/mesa/main/enable.c
index a038a95c55a..d047586eb35 100644
--- a/src/mesa/main/enable.c
+++ b/src/mesa/main/enable.c
@@ -33,6 +33,7 @@
 #include "enable.h"
 #include "light.h"
 #include "simple_list.h"
+#include "mfeatures.h"
 #include "mtypes.h"
 #include "enums.h"
 #include "api_arrayelt.h"
diff --git a/src/mesa/main/eval.c b/src/mesa/main/eval.c
index c607e6a26af..9ab55072f00 100644
--- a/src/mesa/main/eval.c
+++ b/src/mesa/main/eval.c
@@ -43,6 +43,7 @@
 #include "context.h"
 #include "eval.h"
 #include "macros.h"
+#include "mfeatures.h"
 #include "mtypes.h"
 #include "main/dispatch.h"
 
diff --git a/src/mesa/main/eval.h b/src/mesa/main/eval.h
index bd908f00cdd..bfeeebde54d 100644
--- a/src/mesa/main/eval.h
+++ b/src/mesa/main/eval.h
@@ -37,6 +37,7 @@
 #define EVAL_H
 
 
+#include "main/mfeatures.h"
 #include "main/mtypes.h"
 
 
diff --git a/src/mesa/main/extensions.c b/src/mesa/main/extensions.c
index fd5b4e915cd..42626152e8e 100644
--- a/src/mesa/main/extensions.c
+++ b/src/mesa/main/extensions.c
@@ -28,6 +28,7 @@
 #include "imports.h"
 #include "context.h"
 #include "extensions.h"
+#include "mfeatures.h"
 #include "mtypes.h"
 
 
diff --git a/src/mesa/main/fbobject.c b/src/mesa/main/fbobject.c
index 975063d0d78..3e7fd9cf0ec 100644
--- a/src/mesa/main/fbobject.c
+++ b/src/mesa/main/fbobject.c
@@ -40,6 +40,8 @@
 #include "framebuffer.h"
 #include "hash.h"
 #include "macros.h"
+#include "mfeatures.h"
+#include "mtypes.h"
 #include "renderbuffer.h"
 #include "state.h"
 #include "teximage.h"
@@ -402,8 +404,8 @@ fbo_incomplete(const char *msg, int index)
 /**
  * Is the given base format a legal format for a color renderbuffer?
  */
-static GLboolean
-is_legal_color_format(const struct gl_context *ctx, GLenum baseFormat)
+GLboolean
+_mesa_is_legal_color_format(const struct gl_context *ctx, GLenum baseFormat)
 {
    switch (baseFormat) {
    case GL_RGB:
@@ -488,7 +490,7 @@ test_attachment_completeness(const struct gl_context *ctx, GLenum format,
       baseFormat = _mesa_get_format_base_format(texImage->TexFormat);
 
       if (format == GL_COLOR) {
-         if (!is_legal_color_format(ctx, baseFormat)) {
+         if (!_mesa_is_legal_color_format(ctx, baseFormat)) {
             att_incomplete("bad format");
             att->Complete = GL_FALSE;
             return;
@@ -542,8 +544,7 @@ test_attachment_completeness(const struct gl_context *ctx, GLenum format,
          return;
       }
       if (format == GL_COLOR) {
-         if (baseFormat != GL_RGB &&
-             baseFormat != GL_RGBA) {
+         if (!_mesa_is_legal_color_format(ctx, baseFormat)) {
             att_incomplete("bad renderbuffer color format");
             att->Complete = GL_FALSE;
             return;
@@ -669,7 +670,7 @@ _mesa_test_framebuffer_completeness(struct gl_context *ctx,
          f = texImg->_BaseFormat;
          mesaFormat = texImg->TexFormat;
          numImages++;
-         if (!is_legal_color_format(ctx, f) &&
+         if (!_mesa_is_legal_color_format(ctx, f) &&
              !is_legal_depth_format(ctx, f)) {
             fb->_Status = GL_FRAMEBUFFER_INCOMPLETE_FORMATS_EXT;
             fbo_incomplete("texture attachment incomplete", -1);
@@ -793,7 +794,7 @@ _mesa_test_framebuffer_completeness(struct gl_context *ctx,
       fb->Height = minHeight;
 
       /* finally, update the visual info for the framebuffer */
-      _mesa_update_framebuffer_visual(fb);
+      _mesa_update_framebuffer_visual(ctx, fb);
    }
 }
 
@@ -1172,8 +1173,17 @@ get_component_bits(GLenum pname, GLenum baseFormat, gl_format format)
    switch (pname) {
    case GL_RENDERBUFFER_RED_SIZE_EXT:
    case GL_FRAMEBUFFER_ATTACHMENT_RED_SIZE:
+      if (baseFormat == GL_RGB || baseFormat == GL_RGBA ||
+	  baseFormat == GL_RG || baseFormat == GL_RED)
+         return _mesa_get_format_bits(format, pname);
+      else
+         return 0;
    case GL_RENDERBUFFER_GREEN_SIZE_EXT:
    case GL_FRAMEBUFFER_ATTACHMENT_GREEN_SIZE:
+      if (baseFormat == GL_RGB || baseFormat == GL_RGBA || baseFormat == GL_RG)
+         return _mesa_get_format_bits(format, pname);
+      else
+         return 0;
    case GL_RENDERBUFFER_BLUE_SIZE_EXT:
    case GL_FRAMEBUFFER_ATTACHMENT_BLUE_SIZE:
       if (baseFormat == GL_RGB || baseFormat == GL_RGBA)
@@ -1182,7 +1192,8 @@ get_component_bits(GLenum pname, GLenum baseFormat, gl_format format)
          return 0;
    case GL_RENDERBUFFER_ALPHA_SIZE_EXT:
    case GL_FRAMEBUFFER_ATTACHMENT_ALPHA_SIZE:
-      if (baseFormat == GL_RGBA || baseFormat == GL_ALPHA)
+      if (baseFormat == GL_RGBA || baseFormat == GL_ALPHA ||
+	  baseFormat == GL_LUMINANCE_ALPHA)
          return _mesa_get_format_bits(format, pname);
       else
          return 0;
@@ -1940,7 +1951,7 @@ _mesa_FramebufferRenderbufferEXT(GLenum target, GLenum attachment,
    /* Some subsequent GL commands may depend on the framebuffer's visual
     * after the binding is updated.  Update visual info now.
     */
-   _mesa_update_framebuffer_visual(fb);
+   _mesa_update_framebuffer_visual(ctx, fb);
 }
 
 
diff --git a/src/mesa/main/fbobject.h b/src/mesa/main/fbobject.h
index 2aace2ebd4b..8763f99c4a7 100644
--- a/src/mesa/main/fbobject.h
+++ b/src/mesa/main/fbobject.h
@@ -71,6 +71,9 @@ _mesa_framebuffer_renderbuffer(struct gl_context *ctx, struct gl_framebuffer *fb
 extern void
 _mesa_test_framebuffer_completeness(struct gl_context *ctx, struct gl_framebuffer *fb);
 
+extern GLboolean
+_mesa_is_legal_color_format(const struct gl_context *ctx, GLenum baseFormat);
+
 extern GLenum
 _mesa_base_fbo_format(struct gl_context *ctx, GLenum internalFormat);
 
diff --git a/src/mesa/main/feedback.c b/src/mesa/main/feedback.c
index ffdecaecc29..fcb089f1f31 100644
--- a/src/mesa/main/feedback.c
+++ b/src/mesa/main/feedback.c
@@ -35,6 +35,7 @@
 #include "enums.h"
 #include "feedback.h"
 #include "macros.h"
+#include "mfeatures.h"
 #include "mtypes.h"
 #include "main/dispatch.h"
 
diff --git a/src/mesa/main/feedback.h b/src/mesa/main/feedback.h
index f9fbbce70b9..0039a0b4cb0 100644
--- a/src/mesa/main/feedback.h
+++ b/src/mesa/main/feedback.h
@@ -27,6 +27,7 @@
 #define FEEDBACK_H
 
 
+#include "main/mfeatures.h"
 #include "main/mtypes.h"
 
 
diff --git a/src/mesa/main/ffvertex_prog.c b/src/mesa/main/ffvertex_prog.c
index 0f2c313c819..b8e49a3757f 100644
--- a/src/mesa/main/ffvertex_prog.c
+++ b/src/mesa/main/ffvertex_prog.c
@@ -36,6 +36,7 @@
 #include "main/glheader.h"
 #include "main/mtypes.h"
 #include "main/macros.h"
+#include "main/mfeatures.h"
 #include "main/enums.h"
 #include "main/ffvertex_prog.h"
 #include "program/program.h"
diff --git a/src/mesa/main/formats.c b/src/mesa/main/formats.c
index 42f70ca232b..4eb2c354f4c 100644
--- a/src/mesa/main/formats.c
+++ b/src/mesa/main/formats.c
@@ -40,9 +40,10 @@ struct gl_format_info
    const char *StrName;
 
    /**
-    * Base format is one of GL_RGB, GL_RGBA, GL_ALPHA, GL_LUMINANCE,
-    * GL_LUMINANCE_ALPHA, GL_INTENSITY, GL_YCBCR_MESA, GL_COLOR_INDEX,
-    * GL_DEPTH_COMPONENT, GL_STENCIL_INDEX, GL_DEPTH_STENCIL.
+    * Base format is one of GL_RED, GL_RG, GL_RGB, GL_RGBA, GL_ALPHA,
+    * GL_LUMINANCE, GL_LUMINANCE_ALPHA, GL_INTENSITY, GL_YCBCR_MESA,
+    * GL_COLOR_INDEX, GL_DEPTH_COMPONENT, GL_STENCIL_INDEX,
+    * GL_DEPTH_STENCIL, GL_DUDV_ATI.
     */
    GLenum BaseFormat;
 
@@ -222,6 +223,15 @@ static struct gl_format_info format_info[MESA_FORMAT_COUNT] =
       1, 1, 2                      /* BlockWidth/Height,Bytes */
    },
    {
+      MESA_FORMAT_AL44,            /* Name */
+      "MESA_FORMAT_AL44",          /* StrName */
+      GL_LUMINANCE_ALPHA,          /* BaseFormat */
+      GL_UNSIGNED_NORMALIZED,      /* DataType */
+      0, 0, 0, 4,                  /* Red/Green/Blue/AlphaBits */
+      4, 0, 0, 0, 0,               /* Lum/Int/Index/Depth/StencilBits */
+      1, 1, 1                      /* BlockWidth/Height,Bytes */
+   },
+   {
       MESA_FORMAT_AL88,            /* Name */
       "MESA_FORMAT_AL88",          /* StrName */
       GL_LUMINANCE_ALPHA,          /* BaseFormat */
@@ -276,6 +286,15 @@ static struct gl_format_info format_info[MESA_FORMAT_COUNT] =
       1, 1, 1                      /* BlockWidth/Height,Bytes */
    },
    {
+      MESA_FORMAT_A16,             /* Name */
+      "MESA_FORMAT_A16",           /* StrName */
+      GL_ALPHA,                    /* BaseFormat */
+      GL_UNSIGNED_NORMALIZED,      /* DataType */
+      0, 0, 0, 16,                 /* Red/Green/Blue/AlphaBits */
+      0, 0, 0, 0, 0,               /* Lum/Int/Index/Depth/StencilBits */
+      1, 1, 2                      /* BlockWidth/Height,Bytes */
+   },
+   {
       MESA_FORMAT_L8,              /* Name */
       "MESA_FORMAT_L8",            /* StrName */
       GL_LUMINANCE,                /* BaseFormat */
@@ -285,6 +304,15 @@ static struct gl_format_info format_info[MESA_FORMAT_COUNT] =
       1, 1, 1                      /* BlockWidth/Height,Bytes */
    },
    {
+      MESA_FORMAT_L16,             /* Name */
+      "MESA_FORMAT_L16",           /* StrName */
+      GL_LUMINANCE,                /* BaseFormat */
+      GL_UNSIGNED_NORMALIZED,      /* DataType */
+      0, 0, 0, 0,                  /* Red/Green/Blue/AlphaBits */
+      16, 0, 0, 0, 0,              /* Lum/Int/Index/Depth/StencilBits */
+      1, 1, 2                      /* BlockWidth/Height,Bytes */
+   },
+   {
       MESA_FORMAT_I8,              /* Name */
       "MESA_FORMAT_I8",            /* StrName */
       GL_INTENSITY,                /* BaseFormat */
@@ -294,6 +322,15 @@ static struct gl_format_info format_info[MESA_FORMAT_COUNT] =
       1, 1, 1                      /* BlockWidth/Height,Bytes */
    },
    {
+      MESA_FORMAT_I16,             /* Name */
+      "MESA_FORMAT_I16",           /* StrName */
+      GL_INTENSITY,                /* BaseFormat */
+      GL_UNSIGNED_NORMALIZED,      /* DataType */
+      0, 0, 0, 0,                  /* Red/Green/Blue/AlphaBits */
+      0, 16, 0, 0, 0,              /* Lum/Int/Index/Depth/StencilBits */
+      1, 1, 2                      /* BlockWidth/Height,Bytes */
+   },
+   {
       MESA_FORMAT_CI8,             /* Name */
       "MESA_FORMAT_CI8",           /* StrName */
       GL_COLOR_INDEX,              /* BaseFormat */
@@ -375,6 +412,15 @@ static struct gl_format_info format_info[MESA_FORMAT_COUNT] =
       1, 1, 4
    },
    {
+      MESA_FORMAT_ARGB2101010,
+      "MESA_FORMAT_ARGB2101010",
+      GL_RGBA,
+      GL_UNSIGNED_NORMALIZED,
+      10, 10, 10, 2,
+      0, 0, 0, 0, 0,
+      1, 1, 4
+   },
+   {
       MESA_FORMAT_Z24_S8,          /* Name */
       "MESA_FORMAT_Z24_S8",        /* StrName */
       GL_DEPTH_STENCIL,            /* BaseFormat */
@@ -1251,11 +1297,17 @@ _mesa_format_to_type_and_comps(gl_format format,
       *comps = 4;
       return;
 
+   case MESA_FORMAT_ARGB2101010:
+      *datatype = GL_UNSIGNED_INT_2_10_10_10_REV;
+      *comps = 4;
+      return;
+
    case MESA_FORMAT_RGBA5551:
       *datatype = GL_UNSIGNED_SHORT_5_5_5_1;
       *comps = 4;
       return;
 
+   case MESA_FORMAT_AL44: /* XXX this isn't plain GL_UNSIGNED_BYTE */
    case MESA_FORMAT_AL88:
    case MESA_FORMAT_AL88_REV:
    case MESA_FORMAT_RG88:
@@ -1273,6 +1325,9 @@ _mesa_format_to_type_and_comps(gl_format format,
       return;
 
    case MESA_FORMAT_R16:
+   case MESA_FORMAT_A16:
+   case MESA_FORMAT_L16:
+   case MESA_FORMAT_I16:
       *datatype = GL_UNSIGNED_SHORT;
       *comps = 1;
       return;
@@ -1477,10 +1532,13 @@ _mesa_format_to_type_and_comps(gl_format format,
       *comps = 4;
       return;
 
-   case MESA_FORMAT_NONE:
    case MESA_FORMAT_COUNT:
+      assert(0);
+      return;
+
+   case MESA_FORMAT_NONE:
    /* For debug builds, warn if any formats are not handled */
-#ifndef DEBUG
+#ifdef DEBUG
    default:
 #endif
       _mesa_problem(NULL, "bad format %s in _mesa_format_to_type_and_comps",
diff --git a/src/mesa/main/formats.h b/src/mesa/main/formats.h
index 997229bf9f4..b8e76664f85 100644
--- a/src/mesa/main/formats.h
+++ b/src/mesa/main/formats.h
@@ -65,14 +65,18 @@ typedef enum
    MESA_FORMAT_RGBA5551,        /*                     RRRR RGGG GGBB BBBA */
    MESA_FORMAT_ARGB1555,	/*                     ARRR RRGG GGGB BBBB */
    MESA_FORMAT_ARGB1555_REV,	/*                     GGGB BBBB ARRR RRGG */
+   MESA_FORMAT_AL44,		/*                               AAAA LLLL */
    MESA_FORMAT_AL88,		/*                     AAAA AAAA LLLL LLLL */
    MESA_FORMAT_AL88_REV,	/*                     LLLL LLLL AAAA AAAA */
    MESA_FORMAT_AL1616,          /* AAAA AAAA AAAA AAAA LLLL LLLL LLLL LLLL */
    MESA_FORMAT_AL1616_REV,      /* LLLL LLLL LLLL LLLL AAAA AAAA AAAA AAAA */
    MESA_FORMAT_RGB332,		/*                               RRRG GGBB */
    MESA_FORMAT_A8,		/*                               AAAA AAAA */
+   MESA_FORMAT_A16,             /*                     AAAA AAAA AAAA AAAA */
    MESA_FORMAT_L8,		/*                               LLLL LLLL */
+   MESA_FORMAT_L16,             /*                     LLLL LLLL LLLL LLLL */
    MESA_FORMAT_I8,		/*                               IIII IIII */
+   MESA_FORMAT_I16,             /*                     IIII IIII IIII IIII */
    MESA_FORMAT_CI8,		/*                               CCCC CCCC */
    MESA_FORMAT_YCBCR,		/*                     YYYY YYYY UorV UorV */
    MESA_FORMAT_YCBCR_REV,	/*                     UorV UorV YYYY YYYY */
@@ -82,6 +86,7 @@ typedef enum
    MESA_FORMAT_R16,		/*                     RRRR RRRR RRRR RRRR */
    MESA_FORMAT_RG1616,		/* RRRR RRRR RRRR RRRR GGGG GGGG GGGG GGGG */
    MESA_FORMAT_RG1616_REV,	/* GGGG GGGG GGGG GGGG RRRR RRRR RRRR RRRR */
+   MESA_FORMAT_ARGB2101010,     /* AARR RRRR RRRR GGGG GGGG GGBB BBBB BBBB */
    MESA_FORMAT_Z24_S8,          /* ZZZZ ZZZZ ZZZZ ZZZZ ZZZZ ZZZZ SSSS SSSS */
    MESA_FORMAT_S8_Z24,          /* SSSS SSSS ZZZZ ZZZZ ZZZZ ZZZZ ZZZZ ZZZZ */
    MESA_FORMAT_Z16,             /*                     ZZZZ ZZZZ ZZZZ ZZZZ */
diff --git a/src/mesa/main/framebuffer.c b/src/mesa/main/framebuffer.c
index 5530c51c892..63da71c95b4 100644
--- a/src/mesa/main/framebuffer.c
+++ b/src/mesa/main/framebuffer.c
@@ -522,7 +522,8 @@ _mesa_update_draw_buffer_bounds(struct gl_context *ctx)
  * integer Z values.
  */
 void
-_mesa_update_framebuffer_visual(struct gl_framebuffer *fb)
+_mesa_update_framebuffer_visual(struct gl_context *ctx,
+				struct gl_framebuffer *fb)
 {
    GLuint i;
 
@@ -542,9 +543,8 @@ _mesa_update_framebuffer_visual(struct gl_framebuffer *fb)
          const struct gl_renderbuffer *rb = fb->Attachment[i].Renderbuffer;
          const GLenum baseFormat = _mesa_get_format_base_format(rb->Format);
          const gl_format fmt = rb->Format;
-         
-         if (baseFormat == GL_RGBA || baseFormat == GL_RGB ||
-	     baseFormat == GL_ALPHA) {
+
+         if (_mesa_is_legal_color_format(ctx, baseFormat)) {
             fb->Visual.redBits = _mesa_get_format_bits(fmt, GL_RED_BITS);
             fb->Visual.greenBits = _mesa_get_format_bits(fmt, GL_GREEN_BITS);
             fb->Visual.blueBits = _mesa_get_format_bits(fmt, GL_BLUE_BITS);
diff --git a/src/mesa/main/framebuffer.h b/src/mesa/main/framebuffer.h
index 20e3ff56b55..c3bd638c9de 100644
--- a/src/mesa/main/framebuffer.h
+++ b/src/mesa/main/framebuffer.h
@@ -70,7 +70,8 @@ extern void
 _mesa_update_draw_buffer_bounds(struct gl_context *ctx);
 
 extern void
-_mesa_update_framebuffer_visual(struct gl_framebuffer *fb);
+_mesa_update_framebuffer_visual(struct gl_context *ctx,
+				struct gl_framebuffer *fb);
 
 extern void
 _mesa_update_depth_buffer(struct gl_context *ctx, struct gl_framebuffer *fb,
diff --git a/src/mesa/main/get.c b/src/mesa/main/get.c
index 5ae35b868e3..5d26f74bbc2 100644
--- a/src/mesa/main/get.c
+++ b/src/mesa/main/get.c
@@ -29,6 +29,7 @@
 #include "extensions.h"
 #include "get.h"
 #include "macros.h"
+#include "mfeatures.h"
 #include "mtypes.h"
 #include "state.h"
 #include "texcompress.h"
diff --git a/src/mesa/main/getstring.c b/src/mesa/main/getstring.c
index bfa283f6a30..f8866f68581 100644
--- a/src/mesa/main/getstring.c
+++ b/src/mesa/main/getstring.c
@@ -29,6 +29,8 @@
 #include "get.h"
 #include "enums.h"
 #include "extensions.h"
+#include "mfeatures.h"
+#include "mtypes.h"
 
 
 /**
diff --git a/src/mesa/main/hint.c b/src/mesa/main/hint.c
index 878f10d4a43..bdbd7519d37 100644
--- a/src/mesa/main/hint.c
+++ b/src/mesa/main/hint.c
@@ -29,6 +29,7 @@
 #include "context.h"
 #include "hint.h"
 #include "imports.h"
+#include "mtypes.h"
 
 
 
diff --git a/src/mesa/main/histogram.c b/src/mesa/main/histogram.c
index b98e3219c53..75bb9259416 100644
--- a/src/mesa/main/histogram.c
+++ b/src/mesa/main/histogram.c
@@ -28,6 +28,7 @@
 #include "colormac.h"
 #include "histogram.h"
 #include "macros.h"
+#include "mfeatures.h"
 #include "main/dispatch.h"
 
 
diff --git a/src/mesa/main/image.c b/src/mesa/main/image.c
index f9f2ed73077..909c18e7e60 100644
--- a/src/mesa/main/image.c
+++ b/src/mesa/main/image.c
@@ -35,6 +35,8 @@
 #include "image.h"
 #include "imports.h"
 #include "macros.h"
+#include "mfeatures.h"
+#include "mtypes.h"
 
 
 /**
diff --git a/src/mesa/main/imports.c b/src/mesa/main/imports.c
index cefbf4d8c98..bd1fd7cfbf8 100644
--- a/src/mesa/main/imports.c
+++ b/src/mesa/main/imports.c
@@ -46,6 +46,7 @@
 
 #include "imports.h"
 #include "context.h"
+#include "mtypes.h"
 #include "version.h"
 
 #ifdef _GNU_SOURCE
diff --git a/src/mesa/main/matrix.c b/src/mesa/main/matrix.c
index 105d4a327fb..76c7e1c9d1b 100644
--- a/src/mesa/main/matrix.c
+++ b/src/mesa/main/matrix.c
@@ -40,6 +40,7 @@
 #include "context.h"
 #include "enums.h"
 #include "macros.h"
+#include "mfeatures.h"
 #include "matrix.h"
 #include "mtypes.h"
 #include "math/m_matrix.h"
diff --git a/src/mesa/main/mipmap.c b/src/mesa/main/mipmap.c
index ad63b178df8..e073e17039f 100644
--- a/src/mesa/main/mipmap.c
+++ b/src/mesa/main/mipmap.c
@@ -30,6 +30,7 @@
 #include "imports.h"
 #include "formats.h"
 #include "mipmap.h"
+#include "mtypes.h"
 #include "teximage.h"
 #include "texstore.h"
 #include "image.h"
diff --git a/src/mesa/main/mtypes.h b/src/mesa/main/mtypes.h
index 1c549a8e247..4fc25a4148f 100644
--- a/src/mesa/main/mtypes.h
+++ b/src/mesa/main/mtypes.h
@@ -40,6 +40,7 @@
 #include "glapi/glapi.h"
 #include "math/m_matrix.h"	/* GLmatrix */
 #include "main/simple_list.h"	/* struct simple_node */
+#include "main/formats.h"       /* MESA_FORMAT_COUNT */
 
 
 /**
@@ -3240,6 +3241,8 @@ struct gl_context
    /** software compression/decompression supported or not */
    GLboolean Mesa_DXTn;
 
+   GLboolean TextureFormatSupported[MESA_FORMAT_COUNT];
+
    /** 
     * Use dp4 (rather than mul/mad) instructions for position
     * transformation?
diff --git a/src/mesa/main/multisample.c b/src/mesa/main/multisample.c
index 5487d45f56b..67fdb8c9db9 100644
--- a/src/mesa/main/multisample.c
+++ b/src/mesa/main/multisample.c
@@ -27,6 +27,7 @@
 #include "main/context.h"
 #include "main/macros.h"
 #include "main/multisample.h"
+#include "main/mtypes.h"
 
 
 /**
diff --git a/src/mesa/main/nvprogram.c b/src/mesa/main/nvprogram.c
index 36cea3bb9a6..dd198b8141a 100644
--- a/src/mesa/main/nvprogram.c
+++ b/src/mesa/main/nvprogram.c
@@ -42,6 +42,7 @@
 #include "main/hash.h"
 #include "main/imports.h"
 #include "main/macros.h"
+#include "main/mtypes.h"
 #include "main/nvprogram.h"
 #include "program/arbprogparse.h"
 #include "program/nvfragparse.h"
diff --git a/src/mesa/main/pack.c b/src/mesa/main/pack.c
index 6d524e64908..512835cb803 100644
--- a/src/mesa/main/pack.c
+++ b/src/mesa/main/pack.c
@@ -34,6 +34,7 @@
 #include "enums.h"
 #include "image.h"
 #include "imports.h"
+#include "mtypes.h"
 #include "pack.h"
 #include "pixeltransfer.h"
 #include "imports.h"
diff --git a/src/mesa/main/pixel.c b/src/mesa/main/pixel.c
index 5f824b34294..0254980b89d 100644
--- a/src/mesa/main/pixel.c
+++ b/src/mesa/main/pixel.c
@@ -33,6 +33,7 @@
 #include "colormac.h"
 #include "context.h"
 #include "macros.h"
+#include "mfeatures.h"
 #include "pixel.h"
 #include "mtypes.h"
 #include "main/dispatch.h"
diff --git a/src/mesa/main/pixelstore.c b/src/mesa/main/pixelstore.c
index b16d27a4ea5..9d78eec7fea 100644
--- a/src/mesa/main/pixelstore.c
+++ b/src/mesa/main/pixelstore.c
@@ -32,6 +32,7 @@
 #include "bufferobj.h"
 #include "context.h"
 #include "pixelstore.h"
+#include "mfeatures.h"
 #include "mtypes.h"
 
 
diff --git a/src/mesa/main/pixeltransfer.c b/src/mesa/main/pixeltransfer.c
index 711181fd89e..c1832972f5b 100644
--- a/src/mesa/main/pixeltransfer.c
+++ b/src/mesa/main/pixeltransfer.c
@@ -33,6 +33,7 @@
 #include "colormac.h"
 #include "pixeltransfer.h"
 #include "imports.h"
+#include "mtypes.h"
 
 
 /*
diff --git a/src/mesa/main/queryobj.c b/src/mesa/main/queryobj.c
index 66fb5980589..ef6460b2f97 100644
--- a/src/mesa/main/queryobj.c
+++ b/src/mesa/main/queryobj.c
@@ -28,6 +28,7 @@
 #include "hash.h"
 #include "imports.h"
 #include "queryobj.h"
+#include "mfeatures.h"
 #include "mtypes.h"
 #include "main/dispatch.h"
 
diff --git a/src/mesa/main/queryobj.h b/src/mesa/main/queryobj.h
index e289625731a..e4ad96ba5a3 100644
--- a/src/mesa/main/queryobj.h
+++ b/src/mesa/main/queryobj.h
@@ -27,6 +27,7 @@
 #define QUERYOBJ_H
 
 
+#include "main/mfeatures.h"
 #include "main/mtypes.h"
 #include "main/hash.h"
 
diff --git a/src/mesa/main/rastpos.c b/src/mesa/main/rastpos.c
index 6f52f07dfab..a1f6f423416 100644
--- a/src/mesa/main/rastpos.c
+++ b/src/mesa/main/rastpos.c
@@ -32,6 +32,8 @@
 #include "context.h"
 #include "feedback.h"
 #include "macros.h"
+#include "mfeatures.h"
+#include "mtypes.h"
 #include "rastpos.h"
 #include "state.h"
 #include "main/dispatch.h"
diff --git a/src/mesa/main/readpix.c b/src/mesa/main/readpix.c
index a5612e34ecf..e5b85ca32b4 100644
--- a/src/mesa/main/readpix.c
+++ b/src/mesa/main/readpix.c
@@ -30,6 +30,7 @@
 #include "framebuffer.h"
 #include "formats.h"
 #include "image.h"
+#include "mtypes.h"
 #include "state.h"
 
 
diff --git a/src/mesa/main/renderbuffer.c b/src/mesa/main/renderbuffer.c
index 3d1a8f85923..23c662bab7f 100644
--- a/src/mesa/main/renderbuffer.c
+++ b/src/mesa/main/renderbuffer.c
@@ -1481,7 +1481,7 @@ _mesa_add_color_renderbuffers(struct gl_context *ctx, struct gl_framebuffer *fb,
                               GLboolean frontLeft, GLboolean backLeft,
                               GLboolean frontRight, GLboolean backRight)
 {
-   GLuint b;
+   gl_buffer_index b;
 
    if (rgbBits > 16 || alphaBits > 16) {
       _mesa_problem(ctx,
@@ -1545,7 +1545,7 @@ _mesa_add_alpha_renderbuffers(struct gl_context *ctx, struct gl_framebuffer *fb,
                               GLboolean frontLeft, GLboolean backLeft,
                               GLboolean frontRight, GLboolean backRight)
 {
-   GLuint b;
+   gl_buffer_index b;
 
    /* for window system framebuffers only! */
    assert(fb->Name == 0);
@@ -1883,10 +1883,11 @@ _mesa_add_soft_renderbuffers(struct gl_framebuffer *fb,
 
 /**
  * Attach a renderbuffer to a framebuffer.
+ * \param bufferName  one of the BUFFER_x tokens
  */
 void
 _mesa_add_renderbuffer(struct gl_framebuffer *fb,
-                       GLuint bufferName, struct gl_renderbuffer *rb)
+                       gl_buffer_index bufferName, struct gl_renderbuffer *rb)
 {
    assert(fb);
    assert(rb);
@@ -1916,9 +1917,11 @@ _mesa_add_renderbuffer(struct gl_framebuffer *fb,
 
 /**
  * Remove the named renderbuffer from the given framebuffer.
+ * \param bufferName  one of the BUFFER_x tokens
  */
 void
-_mesa_remove_renderbuffer(struct gl_framebuffer *fb, GLuint bufferName)
+_mesa_remove_renderbuffer(struct gl_framebuffer *fb,
+                          gl_buffer_index bufferName)
 {
    struct gl_renderbuffer *rb;
 
diff --git a/src/mesa/main/renderbuffer.h b/src/mesa/main/renderbuffer.h
index 62e812f93a2..4702a8e952a 100644
--- a/src/mesa/main/renderbuffer.h
+++ b/src/mesa/main/renderbuffer.h
@@ -27,6 +27,7 @@
 #define RENDERBUFFER_H
 
 #include "glheader.h"
+#include "mtypes.h"
 
 struct gl_context;
 struct gl_framebuffer;
@@ -95,10 +96,11 @@ _mesa_add_soft_renderbuffers(struct gl_framebuffer *fb,
 
 extern void
 _mesa_add_renderbuffer(struct gl_framebuffer *fb,
-                       GLuint bufferName, struct gl_renderbuffer *rb);
+                       gl_buffer_index bufferName, struct gl_renderbuffer *rb);
 
 extern void
-_mesa_remove_renderbuffer(struct gl_framebuffer *fb, GLuint bufferName);
+_mesa_remove_renderbuffer(struct gl_framebuffer *fb,
+                          gl_buffer_index bufferName);
 
 extern void
 _mesa_reference_renderbuffer(struct gl_renderbuffer **ptr,
diff --git a/src/mesa/main/scissor.c b/src/mesa/main/scissor.c
index 4cf0bc2528c..df665994a95 100644
--- a/src/mesa/main/scissor.c
+++ b/src/mesa/main/scissor.c
@@ -25,6 +25,7 @@
 
 #include "main/glheader.h"
 #include "main/context.h"
+#include "main/mtypes.h"
 #include "main/scissor.h"
 
 
diff --git a/src/mesa/main/shaderapi.c b/src/mesa/main/shaderapi.c
index 96df58d35c2..a311148eaa6 100644
--- a/src/mesa/main/shaderapi.c
+++ b/src/mesa/main/shaderapi.c
@@ -41,6 +41,8 @@
 #include "main/dispatch.h"
 #include "main/enums.h"
 #include "main/hash.h"
+#include "main/mfeatures.h"
+#include "main/mtypes.h"
 #include "main/shaderapi.h"
 #include "main/shaderobj.h"
 #include "program/program.h"
diff --git a/src/mesa/main/shaderobj.c b/src/mesa/main/shaderobj.c
index 216bbce0032..647fd31cab4 100644
--- a/src/mesa/main/shaderobj.c
+++ b/src/mesa/main/shaderobj.c
@@ -32,6 +32,8 @@
 #include "main/glheader.h"
 #include "main/context.h"
 #include "main/hash.h"
+#include "main/mfeatures.h"
+#include "main/mtypes.h"
 #include "main/shaderobj.h"
 #include "program/program.h"
 #include "program/prog_parameter.h"
diff --git a/src/mesa/main/shared.c b/src/mesa/main/shared.c
index 3abee0178eb..ce9fc4de327 100644
--- a/src/mesa/main/shared.c
+++ b/src/mesa/main/shared.c
@@ -30,6 +30,7 @@
 
 
 #include "imports.h"
+#include "mfeatures.h"
 #include "mtypes.h"
 #include "hash.h"
 #if FEATURE_ATI_fragment_shader
diff --git a/src/mesa/main/syncobj.c b/src/mesa/main/syncobj.c
index 2c8bcbeaf7c..e1a5c6c0f92 100644
--- a/src/mesa/main/syncobj.c
+++ b/src/mesa/main/syncobj.c
@@ -59,8 +59,10 @@
 #include "imports.h"
 #include "context.h"
 #include "macros.h"
+#include "mfeatures.h"
 #include "get.h"
 #include "dispatch.h"
+#include "mtypes.h"
 
 #if FEATURE_ARB_sync
 #include "syncobj.h"
diff --git a/src/mesa/main/texcompress.c b/src/mesa/main/texcompress.c
index e3d2a786b3e..7a0b522a2d8 100644
--- a/src/mesa/main/texcompress.c
+++ b/src/mesa/main/texcompress.c
@@ -34,6 +34,8 @@
 #include "imports.h"
 #include "colormac.h"
 #include "formats.h"
+#include "mfeatures.h"
+#include "mtypes.h"
 #include "texcompress.h"
 
 
diff --git a/src/mesa/main/texcompress_fxt1.c b/src/mesa/main/texcompress_fxt1.c
index 33f46d4c8bb..bb7fb567f25 100644
--- a/src/mesa/main/texcompress_fxt1.c
+++ b/src/mesa/main/texcompress_fxt1.c
@@ -34,6 +34,7 @@
 #include "colormac.h"
 #include "image.h"
 #include "macros.h"
+#include "mfeatures.h"
 #include "mipmap.h"
 #include "texcompress.h"
 #include "texcompress_fxt1.h"
diff --git a/src/mesa/main/texcompress_fxt1.h b/src/mesa/main/texcompress_fxt1.h
index 38048b26ccb..b991f4c67ec 100644
--- a/src/mesa/main/texcompress_fxt1.h
+++ b/src/mesa/main/texcompress_fxt1.h
@@ -26,6 +26,7 @@
 #define TEXCOMPRESS_FXT1_H
 
 #include "glheader.h"
+#include "mfeatures.h"
 #include "texstore.h"
 
 struct gl_texture_image;
diff --git a/src/mesa/main/texcompress_s3tc.c b/src/mesa/main/texcompress_s3tc.c
index 0e893a59fa3..25257ecc3d3 100644
--- a/src/mesa/main/texcompress_s3tc.c
+++ b/src/mesa/main/texcompress_s3tc.c
@@ -39,6 +39,8 @@
 #include "dlopen.h"
 #include "image.h"
 #include "macros.h"
+#include "mfeatures.h"
+#include "mtypes.h"
 #include "texcompress.h"
 #include "texcompress_s3tc.h"
 #include "texstore.h"
diff --git a/src/mesa/main/texenv.c b/src/mesa/main/texenv.c
index 508dbf4887d..194bcbea983 100644
--- a/src/mesa/main/texenv.c
+++ b/src/mesa/main/texenv.c
@@ -34,6 +34,7 @@
 #include "main/context.h"
 #include "main/enums.h"
 #include "main/macros.h"
+#include "main/mtypes.h"
 #include "main/texenv.h"
 #include "main/texstate.h"
 
diff --git a/src/mesa/main/texenvprogram.c b/src/mesa/main/texenvprogram.c
index a8bffe416d9..a3fa3fe826c 100644
--- a/src/mesa/main/texenvprogram.c
+++ b/src/mesa/main/texenvprogram.c
@@ -28,6 +28,7 @@
 
 #include "glheader.h"
 #include "imports.h"
+#include "mtypes.h"
 #include "program/program.h"
 #include "program/prog_parameter.h"
 #include "program/prog_cache.h"
diff --git a/src/mesa/main/texfetch.c b/src/mesa/main/texfetch.c
index 372ef2654a7..77bbc91795f 100644
--- a/src/mesa/main/texfetch.c
+++ b/src/mesa/main/texfetch.c
@@ -237,6 +237,13 @@ texfetch_funcs[MESA_FORMAT_COUNT] =
       store_texel_argb1555_rev
    },
    {
+      MESA_FORMAT_AL44,
+      fetch_texel_1d_f_al44,
+      fetch_texel_2d_f_al44,
+      fetch_texel_3d_f_al44,
+      store_texel_al44
+   },
+   {
       MESA_FORMAT_AL88,
       fetch_texel_1d_f_al88,
       fetch_texel_2d_f_al88,
@@ -279,6 +286,13 @@ texfetch_funcs[MESA_FORMAT_COUNT] =
       store_texel_a8
    },
    {
+      MESA_FORMAT_A16,
+      fetch_texel_1d_f_a16,
+      fetch_texel_2d_f_a16,
+      fetch_texel_3d_f_a16,
+      store_texel_a16
+   },
+   {
       MESA_FORMAT_L8,
       fetch_texel_1d_f_l8,
       fetch_texel_2d_f_l8,
@@ -286,6 +300,13 @@ texfetch_funcs[MESA_FORMAT_COUNT] =
       store_texel_l8
    },
    {
+      MESA_FORMAT_L16,
+      fetch_texel_1d_f_l16,
+      fetch_texel_2d_f_l16,
+      fetch_texel_3d_f_l16,
+      store_texel_l16
+   },
+   {
       MESA_FORMAT_I8,
       fetch_texel_1d_f_i8,
       fetch_texel_2d_f_i8,
@@ -293,6 +314,13 @@ texfetch_funcs[MESA_FORMAT_COUNT] =
       store_texel_i8
    },
    {
+      MESA_FORMAT_I16,
+      fetch_texel_1d_f_i16,
+      fetch_texel_2d_f_i16,
+      fetch_texel_3d_f_i16,
+      store_texel_i16
+   },
+   {
       MESA_FORMAT_CI8,
       fetch_texel_1d_f_ci8,
       fetch_texel_2d_f_ci8,
@@ -356,6 +384,13 @@ texfetch_funcs[MESA_FORMAT_COUNT] =
       store_texel_rg1616_rev,
    },
    {
+      MESA_FORMAT_ARGB2101010,
+      fetch_texel_1d_f_argb2101010,
+      fetch_texel_2d_f_argb2101010,
+      fetch_texel_3d_f_argb2101010,
+      store_texel_argb2101010
+   },
+   {
       MESA_FORMAT_Z24_S8,
       fetch_texel_1d_f_z24_s8,
       fetch_texel_2d_f_z24_s8,
diff --git a/src/mesa/main/texfetch_tmp.h b/src/mesa/main/texfetch_tmp.h
index 2f583ed5223..36dede57f00 100644
--- a/src/mesa/main/texfetch_tmp.h
+++ b/src/mesa/main/texfetch_tmp.h
@@ -135,7 +135,7 @@ static void store_texel_rgba_f32(struct gl_texture_image *texImage,
                                  GLint i, GLint j, GLint k, const void *texel)
 {
    const GLfloat *depth = (const GLfloat *) texel;
-   GLfloat *dst = TEXEL_ADDR(GLfloat, texImage, i, j, k, 1);
+   GLfloat *dst = TEXEL_ADDR(GLfloat, texImage, i, j, k, 4);
    dst[0] = depth[RCOMP];
    dst[1] = depth[GCOMP];
    dst[2] = depth[BCOMP];
@@ -163,9 +163,12 @@ static void FETCH(f_rgba_f16)( const struct gl_texture_image *texImage,
 static void store_texel_rgba_f16(struct gl_texture_image *texImage,
                                  GLint i, GLint j, GLint k, const void *texel)
 {
-   const GLfloat *depth = (const GLfloat *) texel;
-   GLhalfARB *dst = TEXEL_ADDR(GLhalfARB, texImage, i, j, k, 1);
-   dst[0] = _mesa_float_to_half(*depth);
+   const GLfloat *src = (const GLfloat *) texel;
+   GLhalfARB *dst = TEXEL_ADDR(GLhalfARB, texImage, i, j, k, 4);
+   dst[0] = _mesa_float_to_half(src[RCOMP]);
+   dst[1] = _mesa_float_to_half(src[GCOMP]);
+   dst[2] = _mesa_float_to_half(src[BCOMP]);
+   dst[3] = _mesa_float_to_half(src[ACOMP]);
 }
 #endif
 
@@ -188,9 +191,11 @@ static void FETCH(f_rgb_f32)( const struct gl_texture_image *texImage,
 static void store_texel_rgb_f32(struct gl_texture_image *texImage,
                                  GLint i, GLint j, GLint k, const void *texel)
 {
-   const GLfloat *depth = (const GLfloat *) texel;
-   GLfloat *dst = TEXEL_ADDR(GLfloat, texImage, i, j, k, 1);
-   dst[0] = *depth;
+   const GLfloat *src = (const GLfloat *) texel;
+   GLfloat *dst = TEXEL_ADDR(GLfloat, texImage, i, j, k, 3);
+   dst[0] = src[RCOMP];
+   dst[1] = src[GCOMP];
+   dst[2] = src[BCOMP];
 }
 #endif
 
@@ -214,9 +219,11 @@ static void FETCH(f_rgb_f16)( const struct gl_texture_image *texImage,
 static void store_texel_rgb_f16(struct gl_texture_image *texImage,
                                 GLint i, GLint j, GLint k, const void *texel)
 {
-   const GLfloat *depth = (const GLfloat *) texel;
-   GLhalfARB *dst = TEXEL_ADDR(GLhalfARB, texImage, i, j, k, 1);
-   dst[0] = _mesa_float_to_half(*depth);
+   const GLfloat *src = (const GLfloat *) texel;
+   GLhalfARB *dst = TEXEL_ADDR(GLhalfARB, texImage, i, j, k, 3);
+   dst[0] = _mesa_float_to_half(src[RCOMP]);
+   dst[1] = _mesa_float_to_half(src[GCOMP]);
+   dst[2] = _mesa_float_to_half(src[BCOMP]);
 }
 #endif
 
@@ -810,6 +817,31 @@ static void store_texel_argb1555_rev(struct gl_texture_image *texImage,
 #endif
 
 
+/* MESA_FORMAT_ARGB2101010 ***************************************************/
+
+/* Fetch texel from 1D, 2D or 3D argb2101010 texture, return 4 GLchans */
+static void FETCH(f_argb2101010)( const struct gl_texture_image *texImage,
+                                  GLint i, GLint j, GLint k, GLfloat *texel )
+{
+   const GLuint *src = TEXEL_ADDR(GLuint, texImage, i, j, k, 1);
+   const GLuint s = *src;
+   texel[RCOMP] = ((s >> 20) & 0x3ff) * (1.0F / 1023.0F);
+   texel[GCOMP] = ((s >> 10) & 0x3ff) * (1.0F / 1023.0F);
+   texel[BCOMP] = ((s >>  0) & 0x3ff) * (1.0F / 1023.0F);
+   texel[ACOMP] = ((s >> 30) & 0x03) * (1.0F / 3.0F);
+}
+
+#if DIM == 3
+static void store_texel_argb2101010(struct gl_texture_image *texImage,
+                                    GLint i, GLint j, GLint k, const void *texel)
+{
+   const GLubyte *rgba = (const GLubyte *) texel;
+   GLuint *dst = TEXEL_ADDR(GLuint, texImage, i, j, k, 1);
+   *dst = PACK_COLOR_2101010_UB(rgba[ACOMP], rgba[RCOMP], rgba[GCOMP], rgba[BCOMP]);
+}
+#endif
+
+
 /* MESA_FORMAT_RG88 **********************************************************/
 
 /* Fetch texel from 1D, 2D or 3D rg88 texture, return 4 GLchans */
@@ -858,6 +890,30 @@ static void store_texel_rg88_rev(struct gl_texture_image *texImage,
 #endif
 
 
+/* MESA_FORMAT_AL44 **********************************************************/
+
+/* Fetch texel from 1D, 2D or 3D al44 texture, return 4 GLchans */
+static void FETCH(f_al44)( const struct gl_texture_image *texImage,
+                           GLint i, GLint j, GLint k, GLfloat *texel )
+{
+   const GLubyte s = *TEXEL_ADDR(GLubyte, texImage, i, j, k, 1);
+   texel[RCOMP] =
+   texel[GCOMP] =
+   texel[BCOMP] = (s & 0xf) * (1.0F / 15.0F);
+   texel[ACOMP] = ((s >> 4) & 0xf) * (1.0F / 15.0F);
+}
+
+#if DIM == 3
+static void store_texel_al44(struct gl_texture_image *texImage,
+                             GLint i, GLint j, GLint k, const void *texel)
+{
+   const GLubyte *rgba = (const GLubyte *) texel;
+   GLubyte *dst = TEXEL_ADDR(GLubyte, texImage, i, j, k, 1);
+   *dst = PACK_COLOR_44(rgba[ACOMP], rgba[RCOMP]);
+}
+#endif
+
+
 /* MESA_FORMAT_AL88 **********************************************************/
 
 /* Fetch texel from 1D, 2D or 3D al88 texture, return 4 GLchans */
@@ -1099,6 +1155,30 @@ static void store_texel_a8(struct gl_texture_image *texImage,
 #endif
 
 
+/* MESA_FORMAT_A16 ************************************************************/
+
+/* Fetch texel from 1D, 2D or 3D a8 texture, return 4 GLchans */
+static void FETCH(f_a16)( const struct gl_texture_image *texImage,
+                          GLint i, GLint j, GLint k, GLfloat *texel )
+{
+   const GLushort *src = TEXEL_ADDR(GLushort, texImage, i, j, k, 1);
+   texel[RCOMP] =
+   texel[GCOMP] =
+   texel[BCOMP] = 0.0F;
+   texel[ACOMP] = USHORT_TO_FLOAT( src[0] );
+}
+
+#if DIM == 3
+static void store_texel_a16(struct gl_texture_image *texImage,
+                            GLint i, GLint j, GLint k, const void *texel)
+{
+   const GLushort *rgba = (const GLushort *) texel;
+   GLushort *dst = TEXEL_ADDR(GLushort, texImage, i, j, k, 1);
+   *dst = rgba[ACOMP];
+}
+#endif
+
+
 /* MESA_FORMAT_L8 ************************************************************/
 
 /* Fetch texel from 1D, 2D or 3D l8 texture, return 4 GLchans */
@@ -1123,6 +1203,30 @@ static void store_texel_l8(struct gl_texture_image *texImage,
 #endif
 
 
+/* MESA_FORMAT_L16 ***********************************************************/
+
+/* Fetch texel from 1D, 2D or 3D l16 texture, return 4 GLchans */
+static void FETCH(f_l16)( const struct gl_texture_image *texImage,
+                          GLint i, GLint j, GLint k, GLfloat *texel )
+{
+   const GLushort *src = TEXEL_ADDR(GLushort, texImage, i, j, k, 1);
+   texel[RCOMP] =
+   texel[GCOMP] =
+   texel[BCOMP] = USHORT_TO_FLOAT( src[0] );
+   texel[ACOMP] = 1.0F;
+}
+
+#if DIM == 3
+static void store_texel_l16(struct gl_texture_image *texImage,
+                            GLint i, GLint j, GLint k, const void *texel)
+{
+   const GLushort *rgba = (const GLushort *) texel;
+   GLushort *dst = TEXEL_ADDR(GLushort, texImage, i, j, k, 1);
+   *dst = rgba[RCOMP];
+}
+#endif
+
+
 /* MESA_FORMAT_I8 ************************************************************/
 
 /* Fetch texel from 1D, 2D or 3D i8 texture, return 4 GLchans */
@@ -1147,6 +1251,30 @@ static void store_texel_i8(struct gl_texture_image *texImage,
 #endif
 
 
+/* MESA_FORMAT_I16 ***********************************************************/
+
+/* Fetch texel from 1D, 2D or 3D i16 texture, return 4 GLchans */
+static void FETCH(f_i16)( const struct gl_texture_image *texImage,
+                          GLint i, GLint j, GLint k, GLfloat *texel )
+{
+   const GLushort *src = TEXEL_ADDR(GLushort, texImage, i, j, k, 1);
+   texel[RCOMP] =
+   texel[GCOMP] =
+   texel[BCOMP] =
+   texel[ACOMP] = USHORT_TO_FLOAT( src[0] );
+}
+
+#if DIM == 3
+static void store_texel_i16(struct gl_texture_image *texImage,
+                            GLint i, GLint j, GLint k, const void *texel)
+{
+   const GLushort *rgba = (const GLushort *) texel;
+   GLushort *dst = TEXEL_ADDR(GLushort, texImage, i, j, k, 1);
+   *dst = rgba[RCOMP];
+}
+#endif
+
+
 /* MESA_FORMAT_CI8 ***********************************************************/
 
 /* Fetch CI texel from 1D, 2D or 3D ci8 texture, lookup the index in a
diff --git a/src/mesa/main/texformat.c b/src/mesa/main/texformat.c
index 894c0130b47..146b2b340e7 100644
--- a/src/mesa/main/texformat.c
+++ b/src/mesa/main/texformat.c
@@ -34,9 +34,15 @@
 
 
 #include "context.h"
+#include "mfeatures.h"
+#include "mtypes.h"
 #include "texcompress.h"
 #include "texformat.h"
 
+#define RETURN_IF_SUPPORTED(f) do {		\
+   if (ctx->TextureFormatSupported[f])		\
+      return f;					\
+} while (0)
 
 /**
  * Choose an appropriate texture format given the format, type and
@@ -64,75 +70,132 @@ _mesa_choose_tex_format( struct gl_context *ctx, GLint internalFormat,
       /* shallow RGBA formats */
       case 4:
       case GL_RGBA:
+	 if (type == GL_UNSIGNED_SHORT_4_4_4_4_REV) {
+	    RETURN_IF_SUPPORTED(MESA_FORMAT_ARGB4444);
+	 } else if (type == GL_UNSIGNED_SHORT_1_5_5_5_REV) {
+	    RETURN_IF_SUPPORTED(MESA_FORMAT_ARGB1555);
+	 }
+	 RETURN_IF_SUPPORTED(MESA_FORMAT_RGBA8888);
+	 RETURN_IF_SUPPORTED(MESA_FORMAT_ARGB8888);
+	 break;
+
       case GL_RGBA8:
-         return MESA_FORMAT_RGBA8888;
+	 RETURN_IF_SUPPORTED(MESA_FORMAT_RGBA8888);
+	 RETURN_IF_SUPPORTED(MESA_FORMAT_ARGB8888);
+	 break;
       case GL_RGB5_A1:
-         return MESA_FORMAT_ARGB1555;
+	 RETURN_IF_SUPPORTED(MESA_FORMAT_ARGB1555);
+	 break;
       case GL_RGBA2:
-         return MESA_FORMAT_ARGB4444_REV; /* just to test another format*/
+	 RETURN_IF_SUPPORTED(MESA_FORMAT_ARGB4444_REV); /* just to test another format*/
+	 RETURN_IF_SUPPORTED(MESA_FORMAT_ARGB4444);
+	 break;
       case GL_RGBA4:
-         return MESA_FORMAT_ARGB4444;
+	 RETURN_IF_SUPPORTED(MESA_FORMAT_ARGB4444);
+	 break;
 
       /* deep RGBA formats */
       case GL_RGB10_A2:
+	 RETURN_IF_SUPPORTED(MESA_FORMAT_ARGB2101010);
+	 RETURN_IF_SUPPORTED(MESA_FORMAT_ARGB8888);
+	 break;
       case GL_RGBA12:
       case GL_RGBA16:
-         return MESA_FORMAT_RGBA_16;
+	 RETURN_IF_SUPPORTED(MESA_FORMAT_RGBA_16);
+	 RETURN_IF_SUPPORTED(MESA_FORMAT_RGBA_16);
+	 RETURN_IF_SUPPORTED(MESA_FORMAT_RGBA8888);
+	 RETURN_IF_SUPPORTED(MESA_FORMAT_ARGB8888);
+	 break;
 
       /* shallow RGB formats */
       case 3:
       case GL_RGB:
       case GL_RGB8:
-         return MESA_FORMAT_RGB888;
+	 RETURN_IF_SUPPORTED(MESA_FORMAT_RGB888);
+	 RETURN_IF_SUPPORTED(MESA_FORMAT_XRGB8888);
+	 RETURN_IF_SUPPORTED(MESA_FORMAT_ARGB8888);
+	 break;
       case GL_R3_G3_B2:
-         return MESA_FORMAT_RGB332;
+	 RETURN_IF_SUPPORTED(MESA_FORMAT_RGB332);
+	 RETURN_IF_SUPPORTED(MESA_FORMAT_RGB888);
+	 RETURN_IF_SUPPORTED(MESA_FORMAT_XRGB8888);
+	 RETURN_IF_SUPPORTED(MESA_FORMAT_ARGB8888);
+	 break;
       case GL_RGB4:
-         return MESA_FORMAT_RGB565_REV; /* just to test another format */
+	 RETURN_IF_SUPPORTED(MESA_FORMAT_RGB565_REV); /* just to test another format */
+	 RETURN_IF_SUPPORTED(MESA_FORMAT_RGB565);
+	 break;
       case GL_RGB5:
-         return MESA_FORMAT_RGB565;
+	 RETURN_IF_SUPPORTED(MESA_FORMAT_RGB565);
+	 break;
 
       /* deep RGB formats */
       case GL_RGB10:
       case GL_RGB12:
       case GL_RGB16:
-         return MESA_FORMAT_RGBA_16;
+	 RETURN_IF_SUPPORTED(MESA_FORMAT_RGBA_16);
+	 RETURN_IF_SUPPORTED(MESA_FORMAT_XRGB8888);
+	 RETURN_IF_SUPPORTED(MESA_FORMAT_ARGB8888);
+	 break;
 
       /* Alpha formats */
       case GL_ALPHA:
       case GL_ALPHA4:
+      case GL_ALPHA8:
+	 RETURN_IF_SUPPORTED(MESA_FORMAT_A8);
+	 break;
+
       case GL_ALPHA12:
       case GL_ALPHA16:
-      case GL_ALPHA8:
-         return MESA_FORMAT_A8;
+	 RETURN_IF_SUPPORTED(MESA_FORMAT_A16);
+	 RETURN_IF_SUPPORTED(MESA_FORMAT_A8);
+	 break;
 
       /* Luminance formats */
       case 1:
       case GL_LUMINANCE:
       case GL_LUMINANCE4:
+      case GL_LUMINANCE8:
+	 RETURN_IF_SUPPORTED(MESA_FORMAT_L8);
+	 break;
+
       case GL_LUMINANCE12:
       case GL_LUMINANCE16:
-      case GL_LUMINANCE8:
-         return MESA_FORMAT_L8;
+	 RETURN_IF_SUPPORTED(MESA_FORMAT_L16);
+	 RETURN_IF_SUPPORTED(MESA_FORMAT_L8);
+	 break;
 
       /* Luminance/Alpha formats */
+      case GL_LUMINANCE4_ALPHA4:
+	 RETURN_IF_SUPPORTED(MESA_FORMAT_AL44);
+	 RETURN_IF_SUPPORTED(MESA_FORMAT_AL88);
+	 break;
+
       case 2:
       case GL_LUMINANCE_ALPHA:
-      case GL_LUMINANCE4_ALPHA4:
       case GL_LUMINANCE6_ALPHA2:
       case GL_LUMINANCE8_ALPHA8:
-         return MESA_FORMAT_AL88;
+	 RETURN_IF_SUPPORTED(MESA_FORMAT_AL88);
+	 break;
 
       case GL_LUMINANCE12_ALPHA4:
       case GL_LUMINANCE12_ALPHA12:
       case GL_LUMINANCE16_ALPHA16:
-         return MESA_FORMAT_AL1616;
+	 RETURN_IF_SUPPORTED(MESA_FORMAT_AL1616);
+	 RETURN_IF_SUPPORTED(MESA_FORMAT_AL88);
+	 break;
 
       case GL_INTENSITY:
       case GL_INTENSITY4:
+      case GL_INTENSITY8:
+	 RETURN_IF_SUPPORTED(MESA_FORMAT_I8);
+	 break;
+
       case GL_INTENSITY12:
       case GL_INTENSITY16:
-      case GL_INTENSITY8:
-         return MESA_FORMAT_I8;
+	 RETURN_IF_SUPPORTED(MESA_FORMAT_I16);
+	 RETURN_IF_SUPPORTED(MESA_FORMAT_I8);
+	 break;
 
       case GL_COLOR_INDEX:
       case GL_COLOR_INDEX1_EXT:
@@ -141,7 +204,8 @@ _mesa_choose_tex_format( struct gl_context *ctx, GLint internalFormat,
       case GL_COLOR_INDEX12_EXT:
       case GL_COLOR_INDEX16_EXT:
       case GL_COLOR_INDEX8_EXT:
-         return MESA_FORMAT_CI8;
+	 RETURN_IF_SUPPORTED(MESA_FORMAT_CI8);
+	 break;
 
       default:
          ; /* fallthrough */
@@ -152,9 +216,12 @@ _mesa_choose_tex_format( struct gl_context *ctx, GLint internalFormat,
          case GL_DEPTH_COMPONENT:
          case GL_DEPTH_COMPONENT24:
          case GL_DEPTH_COMPONENT32:
-            return MESA_FORMAT_Z32;
+	    RETURN_IF_SUPPORTED(MESA_FORMAT_Z32);
+	    RETURN_IF_SUPPORTED(MESA_FORMAT_S8_Z24);
+	    break;
          case GL_DEPTH_COMPONENT16:
-            return MESA_FORMAT_Z16;
+	    RETURN_IF_SUPPORTED(MESA_FORMAT_Z16);
+	    RETURN_IF_SUPPORTED(MESA_FORMAT_S8_Z24);
          default:
             ; /* fallthrough */
       }
@@ -162,27 +229,36 @@ _mesa_choose_tex_format( struct gl_context *ctx, GLint internalFormat,
 
    switch (internalFormat) {
       case GL_COMPRESSED_ALPHA_ARB:
-         return MESA_FORMAT_A8;
+	 RETURN_IF_SUPPORTED(MESA_FORMAT_A8);
+	 break;
       case GL_COMPRESSED_LUMINANCE_ARB:
-         return MESA_FORMAT_L8;
+	 RETURN_IF_SUPPORTED(MESA_FORMAT_L8);
+	 break;
       case GL_COMPRESSED_LUMINANCE_ALPHA_ARB:
-         return MESA_FORMAT_AL88;
+	 RETURN_IF_SUPPORTED(MESA_FORMAT_AL88);
+	 break;
       case GL_COMPRESSED_INTENSITY_ARB:
-         return MESA_FORMAT_I8;
+	 RETURN_IF_SUPPORTED(MESA_FORMAT_I8);
+	 break;
       case GL_COMPRESSED_RGB_ARB:
          if (ctx->Extensions.EXT_texture_compression_s3tc ||
              ctx->Extensions.S3_s3tc)
-            return MESA_FORMAT_RGB_DXT1;
+	    RETURN_IF_SUPPORTED(MESA_FORMAT_RGB_DXT1);
          if (ctx->Extensions.TDFX_texture_compression_FXT1)
-            return MESA_FORMAT_RGB_FXT1;
-         return MESA_FORMAT_RGB888;
+	    RETURN_IF_SUPPORTED(MESA_FORMAT_RGB_FXT1);
+	 RETURN_IF_SUPPORTED(MESA_FORMAT_RGB888);
+	 RETURN_IF_SUPPORTED(MESA_FORMAT_XRGB8888);
+	 RETURN_IF_SUPPORTED(MESA_FORMAT_ARGB8888);
+	 break;
       case GL_COMPRESSED_RGBA_ARB:
          if (ctx->Extensions.EXT_texture_compression_s3tc ||
              ctx->Extensions.S3_s3tc)
-            return MESA_FORMAT_RGBA_DXT3; /* Not rgba_dxt1, see spec */
+	    RETURN_IF_SUPPORTED(MESA_FORMAT_RGBA_DXT3); /* Not rgba_dxt1, see spec */
          if (ctx->Extensions.TDFX_texture_compression_FXT1)
-            return MESA_FORMAT_RGBA_FXT1;
-         return MESA_FORMAT_RGBA8888;
+	    RETURN_IF_SUPPORTED(MESA_FORMAT_RGBA_FXT1);
+	 RETURN_IF_SUPPORTED(MESA_FORMAT_RGBA8888);
+	 RETURN_IF_SUPPORTED(MESA_FORMAT_ARGB8888);
+	 break;
       default:
          ; /* fallthrough */
    }
@@ -190,9 +266,9 @@ _mesa_choose_tex_format( struct gl_context *ctx, GLint internalFormat,
    if (ctx->Extensions.MESA_ycbcr_texture) {
       if (internalFormat == GL_YCBCR_MESA) {
          if (type == GL_UNSIGNED_SHORT_8_8_MESA)
-            return MESA_FORMAT_YCBCR;
+	    RETURN_IF_SUPPORTED(MESA_FORMAT_YCBCR);
          else
-            return MESA_FORMAT_YCBCR_REV;
+	    RETURN_IF_SUPPORTED(MESA_FORMAT_YCBCR_REV);
       }
    }
 
@@ -200,9 +276,11 @@ _mesa_choose_tex_format( struct gl_context *ctx, GLint internalFormat,
    if (ctx->Extensions.TDFX_texture_compression_FXT1) {
       switch (internalFormat) {
          case GL_COMPRESSED_RGB_FXT1_3DFX:
-            return MESA_FORMAT_RGB_FXT1;
+	    RETURN_IF_SUPPORTED(MESA_FORMAT_RGB_FXT1);
+	 break;
          case GL_COMPRESSED_RGBA_FXT1_3DFX:
-            return MESA_FORMAT_RGBA_FXT1;
+	    RETURN_IF_SUPPORTED(MESA_FORMAT_RGBA_FXT1);
+	 break;
          default:
             ; /* fallthrough */
       }
@@ -213,13 +291,17 @@ _mesa_choose_tex_format( struct gl_context *ctx, GLint internalFormat,
    if (ctx->Extensions.EXT_texture_compression_s3tc) {
       switch (internalFormat) {
          case GL_COMPRESSED_RGB_S3TC_DXT1_EXT:
-            return MESA_FORMAT_RGB_DXT1;
+	    RETURN_IF_SUPPORTED(MESA_FORMAT_RGB_DXT1);
+	    break;
          case GL_COMPRESSED_RGBA_S3TC_DXT1_EXT:
-            return MESA_FORMAT_RGBA_DXT1;
+	    RETURN_IF_SUPPORTED(MESA_FORMAT_RGBA_DXT1);
+	    break;
          case GL_COMPRESSED_RGBA_S3TC_DXT3_EXT:
-            return MESA_FORMAT_RGBA_DXT3;
+	    RETURN_IF_SUPPORTED(MESA_FORMAT_RGBA_DXT3);
+	    break;
          case GL_COMPRESSED_RGBA_S3TC_DXT5_EXT:
-            return MESA_FORMAT_RGBA_DXT5;
+	    RETURN_IF_SUPPORTED(MESA_FORMAT_RGBA_DXT5);
+	    break;
          default:
             ; /* fallthrough */
       }
@@ -229,10 +311,12 @@ _mesa_choose_tex_format( struct gl_context *ctx, GLint internalFormat,
       switch (internalFormat) {
          case GL_RGB_S3TC:
          case GL_RGB4_S3TC:
-            return MESA_FORMAT_RGB_DXT1;
+	    RETURN_IF_SUPPORTED(MESA_FORMAT_RGB_DXT1);
+	    break;
          case GL_RGBA_S3TC:
          case GL_RGBA4_S3TC:
-            return MESA_FORMAT_RGBA_DXT3;
+	    RETURN_IF_SUPPORTED(MESA_FORMAT_RGBA_DXT3);
+	    break;
          default:
             ; /* fallthrough */
       }
@@ -242,29 +326,41 @@ _mesa_choose_tex_format( struct gl_context *ctx, GLint internalFormat,
    if (ctx->Extensions.ARB_texture_float) {
       switch (internalFormat) {
          case GL_ALPHA16F_ARB:
-            return MESA_FORMAT_ALPHA_FLOAT16;
+	    RETURN_IF_SUPPORTED(MESA_FORMAT_ALPHA_FLOAT16);
+	    break;
          case GL_ALPHA32F_ARB:
-            return MESA_FORMAT_ALPHA_FLOAT32;
+	    RETURN_IF_SUPPORTED(MESA_FORMAT_ALPHA_FLOAT32);
+	    break;
          case GL_LUMINANCE16F_ARB:
-            return MESA_FORMAT_LUMINANCE_FLOAT16;
+	    RETURN_IF_SUPPORTED(MESA_FORMAT_LUMINANCE_FLOAT16);
+	    break;
          case GL_LUMINANCE32F_ARB:
-            return MESA_FORMAT_LUMINANCE_FLOAT32;
+	    RETURN_IF_SUPPORTED(MESA_FORMAT_LUMINANCE_FLOAT32);
+	    break;
          case GL_LUMINANCE_ALPHA16F_ARB:
-            return MESA_FORMAT_LUMINANCE_ALPHA_FLOAT16;
+	    RETURN_IF_SUPPORTED(MESA_FORMAT_LUMINANCE_ALPHA_FLOAT16);
+	    break;
          case GL_LUMINANCE_ALPHA32F_ARB:
-            return MESA_FORMAT_LUMINANCE_ALPHA_FLOAT32;
+	    RETURN_IF_SUPPORTED(MESA_FORMAT_LUMINANCE_ALPHA_FLOAT32);
+	    break;
          case GL_INTENSITY16F_ARB:
-            return MESA_FORMAT_INTENSITY_FLOAT16;
+	    RETURN_IF_SUPPORTED(MESA_FORMAT_INTENSITY_FLOAT16);
+	    break;
          case GL_INTENSITY32F_ARB:
-            return MESA_FORMAT_INTENSITY_FLOAT32;
+	    RETURN_IF_SUPPORTED(MESA_FORMAT_INTENSITY_FLOAT32);
+	    break;
          case GL_RGB16F_ARB:
-            return MESA_FORMAT_RGB_FLOAT16;
+	    RETURN_IF_SUPPORTED(MESA_FORMAT_RGB_FLOAT16);
+	    break;
          case GL_RGB32F_ARB:
-            return MESA_FORMAT_RGB_FLOAT32;
+	    RETURN_IF_SUPPORTED(MESA_FORMAT_RGB_FLOAT32);
+	    break;
          case GL_RGBA16F_ARB:
-            return MESA_FORMAT_RGBA_FLOAT16;
+	    RETURN_IF_SUPPORTED(MESA_FORMAT_RGBA_FLOAT16);
+	    break;
          case GL_RGBA32F_ARB:
-            return MESA_FORMAT_RGBA_FLOAT32;
+	    RETURN_IF_SUPPORTED(MESA_FORMAT_RGBA_FLOAT32);
+	    break;
          default:
             ; /* fallthrough */
       }
@@ -274,7 +370,9 @@ _mesa_choose_tex_format( struct gl_context *ctx, GLint internalFormat,
       switch (internalFormat) {
          case GL_DEPTH_STENCIL_EXT:
          case GL_DEPTH24_STENCIL8_EXT:
-            return MESA_FORMAT_Z24_S8;
+	    RETURN_IF_SUPPORTED(MESA_FORMAT_Z24_S8);
+	    RETURN_IF_SUPPORTED(MESA_FORMAT_S8_Z24);
+	    break;
          default:
             ; /* fallthrough */
       }
@@ -284,7 +382,8 @@ _mesa_choose_tex_format( struct gl_context *ctx, GLint internalFormat,
       switch (internalFormat) {
          case GL_DUDV_ATI:
          case GL_DU8DV8_ATI:
-            return MESA_FORMAT_DUDV8;
+	    RETURN_IF_SUPPORTED(MESA_FORMAT_DUDV8);
+	    break;
          default:
             ; /* fallthrough */
       }
@@ -294,7 +393,9 @@ _mesa_choose_tex_format( struct gl_context *ctx, GLint internalFormat,
       switch (internalFormat) {
          case GL_RGBA_SNORM:
          case GL_RGBA8_SNORM:
-            return MESA_FORMAT_SIGNED_RGBA8888;
+	    RETURN_IF_SUPPORTED(MESA_FORMAT_SIGNED_RGBA8888);
+	    RETURN_IF_SUPPORTED(MESA_FORMAT_SIGNED_RGBA8888_REV);
+	    break;
          default:
             ; /* fallthrough */
       }
@@ -304,24 +405,32 @@ _mesa_choose_tex_format( struct gl_context *ctx, GLint internalFormat,
       switch (internalFormat) {
          case GL_RED_SNORM:
          case GL_R8_SNORM:
-            return MESA_FORMAT_SIGNED_R8;
+	    RETURN_IF_SUPPORTED(MESA_FORMAT_SIGNED_R8);
+	    break;
          case GL_RG_SNORM:
          case GL_RG8_SNORM:
-            return MESA_FORMAT_SIGNED_RG88;
+	    RETURN_IF_SUPPORTED(MESA_FORMAT_SIGNED_RG88);
+	    break;
          case GL_RGB_SNORM:
          case GL_RGB8_SNORM:
-            return MESA_FORMAT_SIGNED_RGBX8888;
+	    RETURN_IF_SUPPORTED(MESA_FORMAT_SIGNED_RGBX8888);
+	    break;
          case GL_RGBA_SNORM:
          case GL_RGBA8_SNORM:
-            return MESA_FORMAT_SIGNED_RGBA8888;
+	    RETURN_IF_SUPPORTED(MESA_FORMAT_SIGNED_RGBA8888);
+	    break;
          case GL_R16_SNORM:
-            return MESA_FORMAT_SIGNED_R_16;
+	    RETURN_IF_SUPPORTED(MESA_FORMAT_SIGNED_R_16);
+	    break;
          case GL_RG16_SNORM:
-            return MESA_FORMAT_SIGNED_RG_16;
+	    RETURN_IF_SUPPORTED(MESA_FORMAT_SIGNED_RG_16);
+	    break;
          case GL_RGB16_SNORM:
-            return MESA_FORMAT_SIGNED_RGB_16;
+	    RETURN_IF_SUPPORTED(MESA_FORMAT_SIGNED_RGB_16);
+	    break;
          case GL_RGBA16_SNORM:
-            return MESA_FORMAT_SIGNED_RGBA_16;
+	    RETURN_IF_SUPPORTED(MESA_FORMAT_SIGNED_RGBA_16);
+	    break;
          default:
             ; /* fall-through */
       }
@@ -332,48 +441,68 @@ _mesa_choose_tex_format( struct gl_context *ctx, GLint internalFormat,
       switch (internalFormat) {
          case GL_SRGB_EXT:
          case GL_SRGB8_EXT:
-            return MESA_FORMAT_SRGB8;
+	    RETURN_IF_SUPPORTED(MESA_FORMAT_SRGB8);
+	    RETURN_IF_SUPPORTED(MESA_FORMAT_SARGB8);
+	    break;
          case GL_SRGB_ALPHA_EXT:
          case GL_SRGB8_ALPHA8_EXT:
-            return MESA_FORMAT_SRGBA8;
+	    RETURN_IF_SUPPORTED(MESA_FORMAT_SRGBA8);
+	    RETURN_IF_SUPPORTED(MESA_FORMAT_SARGB8);
+	    break;
          case GL_SLUMINANCE_EXT:
          case GL_SLUMINANCE8_EXT:
-            return MESA_FORMAT_SL8;
+	    RETURN_IF_SUPPORTED(MESA_FORMAT_SL8);
+	    RETURN_IF_SUPPORTED(MESA_FORMAT_SARGB8);
+	    break;
          case GL_SLUMINANCE_ALPHA_EXT:
          case GL_SLUMINANCE8_ALPHA8_EXT:
-            return MESA_FORMAT_SLA8;
+	    RETURN_IF_SUPPORTED(MESA_FORMAT_SLA8);
+	    RETURN_IF_SUPPORTED(MESA_FORMAT_SARGB8);
+	    break;
          case GL_COMPRESSED_SLUMINANCE_EXT:
-            return MESA_FORMAT_SL8;
+	    RETURN_IF_SUPPORTED(MESA_FORMAT_SL8);
+	    RETURN_IF_SUPPORTED(MESA_FORMAT_SARGB8);
+	    break;
          case GL_COMPRESSED_SLUMINANCE_ALPHA_EXT:
-            return MESA_FORMAT_SLA8;
+	    RETURN_IF_SUPPORTED(MESA_FORMAT_SLA8);
+	    RETURN_IF_SUPPORTED(MESA_FORMAT_SARGB8);
+	    break;
          case GL_COMPRESSED_SRGB_EXT:
 #if FEATURE_texture_s3tc
             if (ctx->Extensions.EXT_texture_compression_s3tc)
-               return MESA_FORMAT_SRGB_DXT1;
+	       RETURN_IF_SUPPORTED(MESA_FORMAT_SRGB_DXT1);
 #endif
-            return MESA_FORMAT_SRGB8;
+	    RETURN_IF_SUPPORTED(MESA_FORMAT_SRGB8);
+	    RETURN_IF_SUPPORTED(MESA_FORMAT_SARGB8);
+	    break;
          case GL_COMPRESSED_SRGB_ALPHA_EXT:
 #if FEATURE_texture_s3tc
             if (ctx->Extensions.EXT_texture_compression_s3tc)
-               return MESA_FORMAT_SRGBA_DXT3; /* Not srgba_dxt1, see spec */
+	       RETURN_IF_SUPPORTED(MESA_FORMAT_SRGBA_DXT3); /* Not srgba_dxt1, see spec */
 #endif
-            return MESA_FORMAT_SRGBA8;
+	    RETURN_IF_SUPPORTED(MESA_FORMAT_SRGBA8);
+	    RETURN_IF_SUPPORTED(MESA_FORMAT_SARGB8);
+	    break;
 #if FEATURE_texture_s3tc
          case GL_COMPRESSED_SRGB_S3TC_DXT1_EXT:
             if (ctx->Extensions.EXT_texture_compression_s3tc)
-               return MESA_FORMAT_SRGB_DXT1;
+	       RETURN_IF_SUPPORTED(MESA_FORMAT_SRGB_DXT1);
+	    RETURN_IF_SUPPORTED(MESA_FORMAT_SARGB8);
             break;
          case GL_COMPRESSED_SRGB_ALPHA_S3TC_DXT1_EXT:
             if (ctx->Extensions.EXT_texture_compression_s3tc)
-               return MESA_FORMAT_SRGBA_DXT1;
+	       RETURN_IF_SUPPORTED(MESA_FORMAT_SRGBA_DXT1);
+	    RETURN_IF_SUPPORTED(MESA_FORMAT_SARGB8);
             break;
          case GL_COMPRESSED_SRGB_ALPHA_S3TC_DXT3_EXT:
             if (ctx->Extensions.EXT_texture_compression_s3tc)
-               return MESA_FORMAT_SRGBA_DXT3;
+	       RETURN_IF_SUPPORTED(MESA_FORMAT_SRGBA_DXT3);
+	    RETURN_IF_SUPPORTED(MESA_FORMAT_SARGB8);
             break;
          case GL_COMPRESSED_SRGB_ALPHA_S3TC_DXT5_EXT:
             if (ctx->Extensions.EXT_texture_compression_s3tc)
-               return MESA_FORMAT_SRGBA_DXT5;
+	       RETURN_IF_SUPPORTED(MESA_FORMAT_SRGBA_DXT5);
+	    RETURN_IF_SUPPORTED(MESA_FORMAT_SARGB8);
             break;
 #endif
          default:
@@ -390,42 +519,48 @@ _mesa_choose_tex_format( struct gl_context *ctx, GLint internalFormat,
       case GL_INTENSITY32UI_EXT:
       case GL_LUMINANCE32UI_EXT:
       case GL_LUMINANCE_ALPHA32UI_EXT:
-         return MESA_FORMAT_RGBA_UINT32;
+	 RETURN_IF_SUPPORTED(MESA_FORMAT_RGBA_UINT32);
+	 break;
       case GL_RGBA16UI_EXT:
       case GL_RGB16UI_EXT:
       case GL_ALPHA16UI_EXT:
       case GL_INTENSITY16UI_EXT:
       case GL_LUMINANCE16UI_EXT:
       case GL_LUMINANCE_ALPHA16UI_EXT:
-         return MESA_FORMAT_RGBA_UINT16;
+	 RETURN_IF_SUPPORTED(MESA_FORMAT_RGBA_UINT16);
+	 break;
       case GL_RGBA8UI_EXT:
       case GL_RGB8UI_EXT:
       case GL_ALPHA8UI_EXT:
       case GL_INTENSITY8UI_EXT:
       case GL_LUMINANCE8UI_EXT:
       case GL_LUMINANCE_ALPHA8UI_EXT:
-         return MESA_FORMAT_RGBA_UINT8;
+	 RETURN_IF_SUPPORTED(MESA_FORMAT_RGBA_UINT8);
+	 break;
       case GL_RGBA32I_EXT:
       case GL_RGB32I_EXT:
       case GL_ALPHA32I_EXT:
       case GL_INTENSITY32I_EXT:
       case GL_LUMINANCE32I_EXT:
       case GL_LUMINANCE_ALPHA32I_EXT:
-         return MESA_FORMAT_RGBA_INT32;
+	 RETURN_IF_SUPPORTED(MESA_FORMAT_RGBA_INT32);
+	 break;
       case GL_RGBA16I_EXT:
       case GL_RGB16I_EXT:
       case GL_ALPHA16I_EXT:
       case GL_INTENSITY16I_EXT:
       case GL_LUMINANCE16I_EXT:
       case GL_LUMINANCE_ALPHA16I_EXT:
-         return MESA_FORMAT_RGBA_INT16;
+	 RETURN_IF_SUPPORTED(MESA_FORMAT_RGBA_INT16);
+	 break;
       case GL_RGBA8I_EXT:
       case GL_RGB8I_EXT:
       case GL_ALPHA8I_EXT:
       case GL_INTENSITY8I_EXT:
       case GL_LUMINANCE8I_EXT:
       case GL_LUMINANCE_ALPHA8I_EXT:
-         return MESA_FORMAT_RGBA_INT8;
+	 RETURN_IF_SUPPORTED(MESA_FORMAT_RGBA_INT8);
+	 break;
       }
    }
 
@@ -434,18 +569,22 @@ _mesa_choose_tex_format( struct gl_context *ctx, GLint internalFormat,
       case GL_R8:
       case GL_RED:
       case GL_COMPRESSED_RED:
-	 return MESA_FORMAT_R8;
+	 RETURN_IF_SUPPORTED(MESA_FORMAT_R8);
+	 break;
 
       case GL_R16:
-         return MESA_FORMAT_R16;
+	 RETURN_IF_SUPPORTED(MESA_FORMAT_R16);
+	 break;
 
       case GL_RG:
       case GL_RG8:
       case GL_COMPRESSED_RG:
-	 return MESA_FORMAT_RG88;
+	 RETURN_IF_SUPPORTED(MESA_FORMAT_RG88);
+	 break;
 
       case GL_RG16:
-	 return MESA_FORMAT_RG1616;
+	 RETURN_IF_SUPPORTED(MESA_FORMAT_RG1616);
+	 break;
 
       default:
          ; /* fallthrough */
diff --git a/src/mesa/main/texgen.c b/src/mesa/main/texgen.c
index 108ea4cd42c..0ace0b9364d 100644
--- a/src/mesa/main/texgen.c
+++ b/src/mesa/main/texgen.c
@@ -34,6 +34,7 @@
 #include "main/context.h"
 #include "main/enums.h"
 #include "main/macros.h"
+#include "main/mfeatures.h"
 #include "main/texgen.h"
 #include "main/texstate.h"
 #include "math/m_matrix.h"
diff --git a/src/mesa/main/texgen.h b/src/mesa/main/texgen.h
index 9ed80238363..336ffce9245 100644
--- a/src/mesa/main/texgen.h
+++ b/src/mesa/main/texgen.h
@@ -29,6 +29,7 @@
 
 #include "compiler.h"
 #include "glheader.h"
+#include "mfeatures.h"
 
 struct _glapi_table;
 
diff --git a/src/mesa/main/texgetimage.c b/src/mesa/main/texgetimage.c
index c94f88e16e6..da610798e12 100644
--- a/src/mesa/main/texgetimage.c
+++ b/src/mesa/main/texgetimage.c
@@ -35,6 +35,8 @@
 #include "context.h"
 #include "formats.h"
 #include "image.h"
+#include "mfeatures.h"
+#include "mtypes.h"
 #include "pack.h"
 #include "texgetimage.h"
 #include "teximage.h"
@@ -432,11 +434,21 @@ get_tex_memcpy(struct gl_context *ctx, GLenum format, GLenum type, GLvoid *pixel
                type == GL_UNSIGNED_BYTE) {
          memCopy = GL_TRUE;
       }
+      else if (texImage->TexFormat == MESA_FORMAT_L16 &&
+               format == GL_LUMINANCE &&
+               type == GL_UNSIGNED_SHORT) {
+         memCopy = GL_TRUE;
+      }
       else if (texImage->TexFormat == MESA_FORMAT_A8 &&
                format == GL_ALPHA &&
                type == GL_UNSIGNED_BYTE) {
          memCopy = GL_TRUE;
       }
+      else if (texImage->TexFormat == MESA_FORMAT_A16 &&
+               format == GL_ALPHA &&
+               type == GL_UNSIGNED_SHORT) {
+         memCopy = GL_TRUE;
+      }
    }
 
    if (memCopy) {
diff --git a/src/mesa/main/teximage.c b/src/mesa/main/teximage.c
index c5ae63052a7..1e8626c7346 100644
--- a/src/mesa/main/teximage.c
+++ b/src/mesa/main/teximage.c
@@ -40,6 +40,7 @@
 #include "image.h"
 #include "imports.h"
 #include "macros.h"
+#include "mfeatures.h"
 #include "state.h"
 #include "texcompress.h"
 #include "texfetch.h"
@@ -340,11 +341,11 @@ _mesa_base_tex_format( struct gl_context *ctx, GLint internalFormat )
          return GL_RGBA;
       case GL_SLUMINANCE_ALPHA_EXT:
       case GL_SLUMINANCE8_ALPHA8_EXT:
-      case GL_COMPRESSED_SLUMINANCE_EXT:
       case GL_COMPRESSED_SLUMINANCE_ALPHA_EXT:
          return GL_LUMINANCE_ALPHA;
       case GL_SLUMINANCE_EXT:
       case GL_SLUMINANCE8_EXT:
+      case GL_COMPRESSED_SLUMINANCE_EXT:
          return GL_LUMINANCE;
       default:
          ; /* fallthrough */
diff --git a/src/mesa/main/texobj.c b/src/mesa/main/texobj.c
index 5bc5639dbf7..f61e0237add 100644
--- a/src/mesa/main/texobj.c
+++ b/src/mesa/main/texobj.c
@@ -470,6 +470,12 @@ _mesa_test_texobj_completeness( const struct gl_context *ctx,
 
    ASSERT(maxLevels > 0);
 
+   if (t->MaxLevel < t->BaseLevel) {
+      incomplete(t, "MAX_LEVEL (%d) < BASE_LEVEL (%d)",
+		 t->MaxLevel, t->BaseLevel);
+      return;
+   }
+
    t->_MaxLevel = baseLevel + maxLog2;
    t->_MaxLevel = MIN2(t->_MaxLevel, t->MaxLevel);
    t->_MaxLevel = MIN2(t->_MaxLevel, maxLevels - 1);
diff --git a/src/mesa/main/texpal.c b/src/mesa/main/texpal.c
index a25e7aa4ff8..b2112f957a2 100644
--- a/src/mesa/main/texpal.c
+++ b/src/mesa/main/texpal.c
@@ -18,6 +18,7 @@
 #include "glheader.h"
 #include "compiler.h" /* for ASSERT */
 #include "context.h"
+#include "mfeatures.h"
 #include "mtypes.h"
 #include "imports.h"
 #include "pixelstore.h"
diff --git a/src/mesa/main/texparam.c b/src/mesa/main/texparam.c
index d5c83de97f7..d2b8b5ca4ad 100644
--- a/src/mesa/main/texparam.c
+++ b/src/mesa/main/texparam.c
@@ -35,6 +35,8 @@
 #include "main/context.h"
 #include "main/formats.h"
 #include "main/macros.h"
+#include "main/mfeatures.h"
+#include "main/mtypes.h"
 #include "main/texcompress.h"
 #include "main/texparam.h"
 #include "main/teximage.h"
diff --git a/src/mesa/main/texstore.c b/src/mesa/main/texstore.c
index 89677c519e5..7dd4a1fa650 100644
--- a/src/mesa/main/texstore.c
+++ b/src/mesa/main/texstore.c
@@ -58,6 +58,8 @@
 #include "image.h"
 #include "macros.h"
 #include "mipmap.h"
+#include "mfeatures.h"
+#include "mtypes.h"
 #include "pack.h"
 #include "imports.h"
 #include "pack.h"
@@ -2038,6 +2040,132 @@ _mesa_texstore_argb1555(TEXSTORE_PARAMS)
 }
 
 
+static GLboolean
+_mesa_texstore_argb2101010(TEXSTORE_PARAMS)
+{
+   const GLuint texelBytes = _mesa_get_format_bytes(dstFormat);
+   const GLenum baseFormat = _mesa_get_format_base_format(dstFormat);
+
+   ASSERT(dstFormat == MESA_FORMAT_ARGB2101010);
+   ASSERT(texelBytes == 4);
+
+   if (!ctx->_ImageTransferState &&
+       !srcPacking->SwapBytes &&
+       dstFormat == MESA_FORMAT_ARGB2101010 &&
+       srcFormat == GL_BGRA &&
+       srcType == GL_UNSIGNED_INT_2_10_10_10_REV &&
+       baseInternalFormat == GL_RGBA) {
+      /* simple memcpy path */
+      memcpy_texture(ctx, dims,
+                     dstFormat, dstAddr, dstXoffset, dstYoffset, dstZoffset,
+                     dstRowStride,
+                     dstImageOffsets,
+                     srcWidth, srcHeight, srcDepth, srcFormat, srcType,
+                     srcAddr, srcPacking);
+   }
+   else {
+      /* general path */
+      const GLfloat *tempImage = make_temp_float_image(ctx, dims,
+                                                 baseInternalFormat,
+                                                 baseFormat,
+                                                 srcWidth, srcHeight, srcDepth,
+                                                 srcFormat, srcType, srcAddr,
+                                                 srcPacking,
+                                                 ctx->_ImageTransferState);
+      const GLfloat *src = tempImage;
+      GLint img, row, col;
+      if (!tempImage)
+         return GL_FALSE;
+      for (img = 0; img < srcDepth; img++) {
+         GLubyte *dstRow = (GLubyte *) dstAddr
+            + dstImageOffsets[dstZoffset + img] * texelBytes
+            + dstYoffset * dstRowStride
+            + dstXoffset * texelBytes;
+         if (baseInternalFormat == GL_RGBA) {
+            for (row = 0; row < srcHeight; row++) {
+               GLuint *dstUI = (GLuint *) dstRow;
+               for (col = 0; col < srcWidth; col++) {
+                  GLushort a,r,g,b;
+
+                  UNCLAMPED_FLOAT_TO_USHORT(a, src[ACOMP]);
+                  UNCLAMPED_FLOAT_TO_USHORT(r, src[RCOMP]);
+                  UNCLAMPED_FLOAT_TO_USHORT(g, src[GCOMP]);
+                  UNCLAMPED_FLOAT_TO_USHORT(b, src[BCOMP]);
+                  dstUI[col] = PACK_COLOR_2101010_US(a, r, g, b);
+                  src += 4;
+               }
+               dstRow += dstRowStride;
+            }
+         } else if (baseInternalFormat == GL_RGB) {
+            for (row = 0; row < srcHeight; row++) {
+               GLuint *dstUI = (GLuint *) dstRow;
+               for (col = 0; col < srcWidth; col++) {
+                  GLushort r,g,b;
+
+                  UNCLAMPED_FLOAT_TO_USHORT(r, src[RCOMP]);
+                  UNCLAMPED_FLOAT_TO_USHORT(g, src[GCOMP]);
+                  UNCLAMPED_FLOAT_TO_USHORT(b, src[BCOMP]);
+                  dstUI[col] = PACK_COLOR_2101010_US(0xffff, r, g, b);
+                  src += 4;
+               }
+               dstRow += dstRowStride;
+            }
+         } else {
+            ASSERT(0);
+         }
+      }
+      free((void *) tempImage);
+   }
+   return GL_TRUE;
+}
+
+
+/**
+ * Do texstore for 2-channel, 4-bit/channel, unsigned normalized formats.
+ */
+static GLboolean
+_mesa_texstore_unorm44(TEXSTORE_PARAMS)
+{
+   const GLuint texelBytes = _mesa_get_format_bytes(dstFormat);
+   const GLenum baseFormat = _mesa_get_format_base_format(dstFormat);
+
+   ASSERT(dstFormat == MESA_FORMAT_AL44);
+   ASSERT(texelBytes == 1);
+
+   {
+      /* general path */
+      const GLchan *tempImage = _mesa_make_temp_chan_image(ctx, dims,
+                                                 baseInternalFormat,
+                                                 baseFormat,
+                                                 srcWidth, srcHeight, srcDepth,
+                                                 srcFormat, srcType, srcAddr,
+                                                 srcPacking);
+      const GLchan *src = tempImage;
+      GLint img, row, col;
+      if (!tempImage)
+         return GL_FALSE;
+      for (img = 0; img < srcDepth; img++) {
+         GLubyte *dstRow = (GLubyte *) dstAddr
+            + dstImageOffsets[dstZoffset + img] * texelBytes
+            + dstYoffset * dstRowStride
+            + dstXoffset * texelBytes;
+         for (row = 0; row < srcHeight; row++) {
+            GLubyte *dstUS = (GLubyte *) dstRow;
+            for (col = 0; col < srcWidth; col++) {
+               /* src[0] is luminance, src[1] is alpha */
+               dstUS[col] = PACK_COLOR_44( CHAN_TO_UBYTE(src[1]),
+                                           CHAN_TO_UBYTE(src[0]) );
+               src += 2;
+            }
+            dstRow += dstRowStride;
+         }
+      }
+      free((void *) tempImage);
+   }
+   return GL_TRUE;
+}
+
+
 /**
  * Do texstore for 2-channel, 8-bit/channel, unsigned normalized formats.
  */
@@ -2237,21 +2365,23 @@ _mesa_texstore_unorm1616(TEXSTORE_PARAMS)
 }
 
 
+/* Texstore for R16, A16, L16, I16. */
 static GLboolean
-_mesa_texstore_r16(TEXSTORE_PARAMS)
+_mesa_texstore_unorm16(TEXSTORE_PARAMS)
 {
    const GLboolean littleEndian = _mesa_little_endian();
    const GLuint texelBytes = _mesa_get_format_bytes(dstFormat);
    const GLenum baseFormat = _mesa_get_format_base_format(dstFormat);
 
-   ASSERT(dstFormat == MESA_FORMAT_R16);
+   ASSERT(dstFormat == MESA_FORMAT_R16 ||
+          dstFormat == MESA_FORMAT_A16 ||
+          dstFormat == MESA_FORMAT_L16 ||
+          dstFormat == MESA_FORMAT_I16);
    ASSERT(texelBytes == 2);
 
    if (!ctx->_ImageTransferState &&
        !srcPacking->SwapBytes &&
-       dstFormat == MESA_FORMAT_R16 &&
-       baseInternalFormat == GL_RED &&
-       srcFormat == GL_RED &&
+       baseInternalFormat == srcFormat &&
        srcType == GL_UNSIGNED_SHORT &&
        littleEndian) {
       /* simple memcpy path */
@@ -3921,23 +4051,28 @@ texstore_funcs[MESA_FORMAT_COUNT] =
    { MESA_FORMAT_RGBA5551, _mesa_texstore_rgba5551 },
    { MESA_FORMAT_ARGB1555, _mesa_texstore_argb1555 },
    { MESA_FORMAT_ARGB1555_REV, _mesa_texstore_argb1555 },
+   { MESA_FORMAT_AL44, _mesa_texstore_unorm44 },
    { MESA_FORMAT_AL88, _mesa_texstore_unorm88 },
    { MESA_FORMAT_AL88_REV, _mesa_texstore_unorm88 },
    { MESA_FORMAT_AL1616, _mesa_texstore_unorm1616 },
    { MESA_FORMAT_AL1616_REV, _mesa_texstore_unorm1616 },
    { MESA_FORMAT_RGB332, _mesa_texstore_rgb332 },
    { MESA_FORMAT_A8, _mesa_texstore_a8 },
+   { MESA_FORMAT_A16, _mesa_texstore_unorm16 },
    { MESA_FORMAT_L8, _mesa_texstore_a8 },
+   { MESA_FORMAT_L16, _mesa_texstore_unorm16 },
    { MESA_FORMAT_I8, _mesa_texstore_a8 },
+   { MESA_FORMAT_I16, _mesa_texstore_unorm16 },
    { MESA_FORMAT_CI8, _mesa_texstore_ci8 },
    { MESA_FORMAT_YCBCR, _mesa_texstore_ycbcr },
    { MESA_FORMAT_YCBCR_REV, _mesa_texstore_ycbcr },
    { MESA_FORMAT_R8, _mesa_texstore_a8 },
    { MESA_FORMAT_RG88, _mesa_texstore_unorm88 },
    { MESA_FORMAT_RG88_REV, _mesa_texstore_unorm88 },
-   { MESA_FORMAT_R16, _mesa_texstore_r16 },
+   { MESA_FORMAT_R16, _mesa_texstore_unorm16 },
    { MESA_FORMAT_RG1616, _mesa_texstore_unorm1616 },
    { MESA_FORMAT_RG1616_REV, _mesa_texstore_unorm1616 },
+   { MESA_FORMAT_ARGB2101010, _mesa_texstore_argb2101010 },
    { MESA_FORMAT_Z24_S8, _mesa_texstore_z24_s8 },
    { MESA_FORMAT_S8_Z24, _mesa_texstore_s8_z24 },
    { MESA_FORMAT_Z16, _mesa_texstore_z16 },
diff --git a/src/mesa/main/transformfeedback.c b/src/mesa/main/transformfeedback.c
index d297b5ed712..fbd70d56ca9 100644
--- a/src/mesa/main/transformfeedback.c
+++ b/src/mesa/main/transformfeedback.c
@@ -34,6 +34,8 @@
 #include "bufferobj.h"
 #include "context.h"
 #include "hash.h"
+#include "mfeatures.h"
+#include "mtypes.h"
 #include "transformfeedback.h"
 #include "shaderapi.h"
 #include "shaderobj.h"
diff --git a/src/mesa/main/uniforms.c b/src/mesa/main/uniforms.c
index d61856d0ebc..aee2e6b4e9b 100644
--- a/src/mesa/main/uniforms.c
+++ b/src/mesa/main/uniforms.c
@@ -39,6 +39,8 @@
 #include "main/glheader.h"
 #include "main/context.h"
 #include "main/dispatch.h"
+#include "main/mfeatures.h"
+#include "main/mtypes.h"
 #include "main/shaderapi.h"
 #include "main/shaderobj.h"
 #include "main/uniforms.h"
@@ -511,7 +513,7 @@ get_uniform_rows_cols(const struct gl_program_parameter *p,
          *cols = p->Size;
       }
       else {
-         *rows = p->Size / 4 + 1;
+         *rows = (p->Size + 3) / 4;
          if (p->Size % 4 == 0)
             *cols = 4;
          else
diff --git a/src/mesa/main/varray.c b/src/mesa/main/varray.c
index 32bf95e3ed1..bcde65adc70 100644
--- a/src/mesa/main/varray.c
+++ b/src/mesa/main/varray.c
@@ -33,6 +33,7 @@
 #include "hash.h"
 #include "image.h"
 #include "macros.h"
+#include "mfeatures.h"
 #include "mtypes.h"
 #include "varray.h"
 #include "arrayobj.h"
diff --git a/src/mesa/main/version.c b/src/mesa/main/version.c
index 69a28da84c6..a10b86e761e 100644
--- a/src/mesa/main/version.c
+++ b/src/mesa/main/version.c
@@ -22,7 +22,8 @@
  */
 
 
-#include "context.h"
+#include "imports.h"
+#include "mtypes.h"
 #include "version.h"
 
 
diff --git a/src/mesa/main/version.h b/src/mesa/main/version.h
index 1b535080991..2e6335846e3 100644
--- a/src/mesa/main/version.h
+++ b/src/mesa/main/version.h
@@ -1,6 +1,6 @@
 /*
  * Mesa 3-D graphics library
- * Version:  7.10
+ * Version:  7.11
  *
  * Copyright (C) 1999-2008  Brian Paul   All Rights Reserved.
  * Copyright (C) 2009  VMware, Inc.  All Rights Reserved.
@@ -33,9 +33,9 @@ struct gl_context;
 
 /* Mesa version */
 #define MESA_MAJOR 7
-#define MESA_MINOR 10
+#define MESA_MINOR 11
 #define MESA_PATCH 0
-#define MESA_VERSION_STRING "7.10-devel"
+#define MESA_VERSION_STRING "7.11-devel"
 
 /* To make version comparison easy */
 #define MESA_VERSION(a,b,c) (((a) << 16) + ((b) << 8) + (c))
diff --git a/src/mesa/main/viewport.c b/src/mesa/main/viewport.c
index 4747022d0b4..d19ae3b6c12 100644
--- a/src/mesa/main/viewport.c
+++ b/src/mesa/main/viewport.c
@@ -31,6 +31,7 @@
 
 #include "context.h"
 #include "macros.h"
+#include "mtypes.h"
 #include "viewport.h"
 
 
diff --git a/src/mesa/main/vtxfmt.c b/src/mesa/main/vtxfmt.c
index 8a72641862b..692ce080c9f 100644
--- a/src/mesa/main/vtxfmt.c
+++ b/src/mesa/main/vtxfmt.c
@@ -30,6 +30,7 @@
 #include "api_arrayelt.h"
 #include "context.h"
 #include "imports.h"
+#include "mfeatures.h"
 #include "mtypes.h"
 #include "vtxfmt.h"
 #include "eval.h"
diff --git a/src/mesa/main/vtxfmt.h b/src/mesa/main/vtxfmt.h
index 147385cee96..8bbc54964da 100644
--- a/src/mesa/main/vtxfmt.h
+++ b/src/mesa/main/vtxfmt.h
@@ -34,6 +34,7 @@
 #define _VTXFMT_H_
 
 #include "compiler.h"
+#include "mfeatures.h"
 #include "mtypes.h"
 
 #if FEATURE_beginend
diff --git a/src/mesa/program/ir_to_mesa.cpp b/src/mesa/program/ir_to_mesa.cpp
index 490c4cab7ab..c601ef6eaaa 100644
--- a/src/mesa/program/ir_to_mesa.cpp
+++ b/src/mesa/program/ir_to_mesa.cpp
@@ -2867,7 +2867,7 @@ _mesa_ir_link_shader(struct gl_context *ctx, struct gl_shader_program *prog)
 
 	 if (options->EmitNoIfs) {
 	    progress = lower_discard(ir) || progress;
-	    progress = do_if_to_cond_assign(ir) || progress;
+	    progress = lower_if_to_cond_assign(ir) || progress;
 	 }
 
 	 if (options->EmitNoNoise)
diff --git a/src/mesa/state_tracker/st_atom.c b/src/mesa/state_tracker/st_atom.c
index e29ab46ef99..bf160fe1080 100644
--- a/src/mesa/state_tracker/st_atom.c
+++ b/src/mesa/state_tracker/st_atom.c
@@ -147,7 +147,11 @@ void st_validate_state( struct st_context *st )
 
    /*printf("%s %x/%x\n", __FUNCTION__, state->mesa, state->st);*/
 
+#ifdef NDEBUG
+   if (0) {
+#else
    if (1) {
+#endif
       /* Debug version which enforces various sanity checks on the
        * state flags which are generated and checked to help ensure
        * state atoms are ordered correctly in the list.
diff --git a/src/mesa/state_tracker/st_atom_constbuf.c b/src/mesa/state_tracker/st_atom_constbuf.c
index f1d08a3e166..05667a74305 100644
--- a/src/mesa/state_tracker/st_atom_constbuf.c
+++ b/src/mesa/state_tracker/st_atom_constbuf.c
@@ -56,7 +56,6 @@ void st_upload_constants( struct st_context *st,
                           unsigned shader_type)
 {
    struct pipe_context *pipe = st->pipe;
-   struct pipe_resource **cbuf = &st->state.constants[shader_type];
 
    assert(shader_type == PIPE_SHADER_VERTEX ||
           shader_type == PIPE_SHADER_FRAGMENT ||
@@ -64,6 +63,7 @@ void st_upload_constants( struct st_context *st,
 
    /* update constants */
    if (params && params->NumParameters) {
+      struct pipe_resource *cbuf;
       const uint paramBytes = params->NumParameters * sizeof(GLfloat) * 4;
 
       /* Update the constants which come from fixed-function state, such as
@@ -75,11 +75,12 @@ void st_upload_constants( struct st_context *st,
 
       /* We always need to get a new buffer, to keep the drivers simple and
        * avoid gratuitous rendering synchronization.
+       * Let's use a user buffer to avoid an unnecessary copy.
        */
-      pipe_resource_reference(cbuf, NULL );
-      *cbuf = pipe_buffer_create(pipe->screen,
-				 PIPE_BIND_CONSTANT_BUFFER,
-				 paramBytes );
+      cbuf = pipe_user_buffer_create(pipe->screen,
+                                     params->ParameterValues,
+                                     paramBytes,
+                                     PIPE_BIND_CONSTANT_BUFFER);
 
       if (ST_DEBUG & DEBUG_CONSTANTS) {
 	 debug_printf("%s(shader=%d, numParams=%d, stateFlags=0x%x)\n", 
@@ -88,17 +89,15 @@ void st_upload_constants( struct st_context *st,
          _mesa_print_parameter_list(params);
       }
 
-      /* load Mesa constants into the constant buffer */
-      pipe_buffer_write(st->pipe, *cbuf,
-				    0, paramBytes,
-				    params->ParameterValues);
+      st->pipe->set_constant_buffer(st->pipe, shader_type, 0, cbuf);
+      pipe_resource_reference(&cbuf, NULL);
 
-      st->pipe->set_constant_buffer(st->pipe, shader_type, 0, *cbuf);
+      st->state.constants[shader_type].ptr = params->ParameterValues;
+      st->state.constants[shader_type].size = paramBytes;
    }
-   else if (*cbuf) {
-      st->constants.tracked_state[shader_type].dirty.mesa = 0x0;
-
-      pipe_resource_reference(cbuf, NULL);
+   else if (st->state.constants[shader_type].ptr) {
+      st->state.constants[shader_type].ptr = NULL;
+      st->state.constants[shader_type].size = 0;
       st->pipe->set_constant_buffer(st->pipe, shader_type, 0, NULL);
    }
 }
diff --git a/src/mesa/state_tracker/st_cb_blit.c b/src/mesa/state_tracker/st_cb_blit.c
index 06cee520b37..340773b51cc 100644
--- a/src/mesa/state_tracker/st_cb_blit.c
+++ b/src/mesa/state_tracker/st_cb_blit.c
@@ -40,7 +40,6 @@
 #include "st_cb_fbo.h"
 
 #include "util/u_blit.h"
-#include "util/u_inlines.h"
 
 
 void
diff --git a/src/mesa/state_tracker/st_cb_clear.c b/src/mesa/state_tracker/st_cb_clear.c
index bd1dd78b23c..6571bf237fb 100644
--- a/src/mesa/state_tracker/st_cb_clear.c
+++ b/src/mesa/state_tracker/st_cb_clear.c
@@ -470,13 +470,9 @@ st_Clear(struct gl_context *ctx, GLbitfield mask)
          if (mask & (1 << b)) {
             struct gl_renderbuffer *rb
                = ctx->DrawBuffer->Attachment[b].Renderbuffer;
-            struct st_renderbuffer *strb;
+            struct st_renderbuffer *strb = st_renderbuffer(rb);
 
-            assert(rb);
-
-            strb = st_renderbuffer(rb);
-
-            if (!strb->surface)
+            if (!strb || !strb->surface)
                continue;
 
             if (check_clear_color_with_quad( ctx, rb ))
diff --git a/src/mesa/state_tracker/st_cb_drawpixels.c b/src/mesa/state_tracker/st_cb_drawpixels.c
index 1fc47688e4a..ea6d021c010 100644
--- a/src/mesa/state_tracker/st_cb_drawpixels.c
+++ b/src/mesa/state_tracker/st_cb_drawpixels.c
@@ -34,6 +34,7 @@
 #include "main/image.h"
 #include "main/bufferobj.h"
 #include "main/macros.h"
+#include "main/mtypes.h"
 #include "main/pack.h"
 #include "main/texformat.h"
 #include "main/texstore.h"
diff --git a/src/mesa/state_tracker/st_cb_fbo.c b/src/mesa/state_tracker/st_cb_fbo.c
index cd718a31a14..d566d999b9a 100644
--- a/src/mesa/state_tracker/st_cb_fbo.c
+++ b/src/mesa/state_tracker/st_cb_fbo.c
@@ -61,7 +61,8 @@
  * during window resize.
  */
 static GLboolean
-st_renderbuffer_alloc_storage(struct gl_context * ctx, struct gl_renderbuffer *rb,
+st_renderbuffer_alloc_storage(struct gl_context * ctx,
+                              struct gl_renderbuffer *rb,
                               GLenum internalFormat,
                               GLuint width, GLuint height)
 {
@@ -75,7 +76,8 @@ st_renderbuffer_alloc_storage(struct gl_context * ctx, struct gl_renderbuffer *r
    if (strb->format != PIPE_FORMAT_NONE)
       format = strb->format;
    else
-      format = st_choose_renderbuffer_format(screen, internalFormat, rb->NumSamples);
+      format = st_choose_renderbuffer_format(screen, internalFormat,
+                                             rb->NumSamples);
       
    /* init renderbuffer fields */
    strb->Base.Width  = width;
diff --git a/src/mesa/state_tracker/st_cb_texture.c b/src/mesa/state_tracker/st_cb_texture.c
index 866426a7549..14d33f7b490 100644
--- a/src/mesa/state_tracker/st_cb_texture.c
+++ b/src/mesa/state_tracker/st_cb_texture.c
@@ -1528,6 +1528,9 @@ st_copy_texsubimage(struct gl_context *ctx,
    if (texBaseFormat == GL_DEPTH_COMPONENT ||
        texBaseFormat == GL_DEPTH_STENCIL) {
       strb = st_renderbuffer(fb->_DepthBuffer);
+      if (strb->Base.Wrapped) {
+         strb = st_renderbuffer(strb->Base.Wrapped);
+      }
    }
    else {
       /* texBaseFormat == GL_RGB, GL_RGBA, GL_ALPHA, etc */
diff --git a/src/mesa/state_tracker/st_context.c b/src/mesa/state_tracker/st_context.c
index 21bb91f47a8..c7f3949bf9e 100644
--- a/src/mesa/state_tracker/st_context.c
+++ b/src/mesa/state_tracker/st_context.c
@@ -207,12 +207,6 @@ static void st_destroy_context_priv( struct st_context *st )
       pipe_sampler_view_reference(&st->state.sampler_views[i], NULL);
    }
 
-   for (i = 0; i < Elements(st->state.constants); i++) {
-      if (st->state.constants[i]) {
-         pipe_resource_reference(&st->state.constants[i], NULL);
-      }
-   }
-
    if (st->default_texture) {
       st->ctx->Driver.DeleteTexture(st->ctx, st->default_texture);
       st->default_texture = NULL;
@@ -245,7 +239,6 @@ void st_destroy_context( struct st_context *st )
 
    for (i = 0; i < PIPE_SHADER_TYPES; i++) {
       pipe->set_constant_buffer(pipe, i, 0, NULL);
-      pipe_resource_reference(&st->state.constants[i], NULL);
    }
 
    _mesa_delete_program_cache(st->ctx, st->pixel_xfer.cache);
diff --git a/src/mesa/state_tracker/st_context.h b/src/mesa/state_tracker/st_context.h
index 1fd80053c1b..64a8f790e22 100644
--- a/src/mesa/state_tracker/st_context.h
+++ b/src/mesa/state_tracker/st_context.h
@@ -92,7 +92,10 @@ struct st_context
       struct pipe_sampler_state             samplers[PIPE_MAX_SAMPLERS];
       struct pipe_sampler_state             *sampler_list[PIPE_MAX_SAMPLERS];
       struct pipe_clip_state clip;
-      struct pipe_resource *constants[PIPE_SHADER_TYPES];
+      struct {
+         void *ptr;
+         unsigned size;
+      } constants[PIPE_SHADER_TYPES];
       struct pipe_framebuffer_state framebuffer;
       struct pipe_sampler_view *sampler_views[PIPE_MAX_SAMPLERS];
       struct pipe_scissor_state scissor;
@@ -105,15 +108,6 @@ struct st_context
       GLuint poly_stipple[32];  /**< In OpenGL's bottom-to-top order */
    } state;
 
-   struct {
-      struct st_tracked_state tracked_state[PIPE_SHADER_TYPES];
-   } constants;
-
-   /* XXX unused: */
-   struct {
-      struct gl_fragment_program *fragment_program;
-   } cb;
-
    char vendor[100];
    char renderer[100];
 
diff --git a/src/mesa/state_tracker/st_draw_feedback.c b/src/mesa/state_tracker/st_draw_feedback.c
index da67c713710..05917992d6e 100644
--- a/src/mesa/state_tracker/st_draw_feedback.c
+++ b/src/mesa/state_tracker/st_draw_feedback.c
@@ -109,9 +109,7 @@ st_feedback_draw_vbo(struct gl_context *ctx,
    struct pipe_index_buffer ibuffer;
    struct pipe_transfer *vb_transfer[PIPE_MAX_ATTRIBS];
    struct pipe_transfer *ib_transfer = NULL;
-   struct pipe_transfer *cb_transfer;
    GLuint attr, i;
-   ubyte *mapped_constants;
    const void *mapped_indices = NULL;
 
    assert(draw);
@@ -242,14 +240,10 @@ st_feedback_draw_vbo(struct gl_context *ctx,
       draw_set_mapped_index_buffer(draw, mapped_indices);
    }
 
-   /* map constant buffers */
-   mapped_constants = pipe_buffer_map(pipe,
-                                      st->state.constants[PIPE_SHADER_VERTEX],
-                                      PIPE_TRANSFER_READ,
-				      &cb_transfer);
+   /* set the constant buffer */
    draw_set_mapped_constant_buffer(st->draw, PIPE_SHADER_VERTEX, 0,
-                                   mapped_constants,
-                                   st->state.constants[PIPE_SHADER_VERTEX]->width0);
+                                   st->state.constants[PIPE_SHADER_VERTEX].ptr,
+                                   st->state.constants[PIPE_SHADER_VERTEX].size);
 
 
    /* draw here */
@@ -258,9 +252,6 @@ st_feedback_draw_vbo(struct gl_context *ctx,
    }
 
 
-   /* unmap constant buffers */
-   pipe_buffer_unmap(pipe, cb_transfer);
-
    /*
     * unmap vertex/index buffers
     */
diff --git a/src/mesa/state_tracker/st_extensions.c b/src/mesa/state_tracker/st_extensions.c
index 62c9ce7273d..84c0e8080bd 100644
--- a/src/mesa/state_tracker/st_extensions.c
+++ b/src/mesa/state_tracker/st_extensions.c
@@ -224,6 +224,7 @@ void st_init_extensions(struct st_context *st)
    ctx->Extensions.ARB_draw_elements_base_vertex = GL_TRUE;
    ctx->Extensions.ARB_fragment_coord_conventions = GL_TRUE;
    ctx->Extensions.ARB_fragment_program = GL_TRUE;
+   ctx->Extensions.ARB_half_float_pixel = GL_TRUE;
    ctx->Extensions.ARB_map_buffer_range = GL_TRUE;
    ctx->Extensions.ARB_multisample = GL_TRUE;
    ctx->Extensions.ARB_texture_border_clamp = GL_TRUE; /* XXX temp */
diff --git a/src/mesa/state_tracker/st_format.c b/src/mesa/state_tracker/st_format.c
index 955d821a657..f6a44a8bf05 100644
--- a/src/mesa/state_tracker/st_format.c
+++ b/src/mesa/state_tracker/st_format.c
@@ -146,14 +146,28 @@ st_mesa_format_to_pipe_format(gl_format mesaFormat)
       return PIPE_FORMAT_B4G4R4A4_UNORM;
    case MESA_FORMAT_RGB565:
       return PIPE_FORMAT_B5G6R5_UNORM;
+   case MESA_FORMAT_RGB332:
+      return PIPE_FORMAT_B2G3R3_UNORM;
+   case MESA_FORMAT_ARGB2101010:
+      return PIPE_FORMAT_B10G10R10A2_UNORM;
+   case MESA_FORMAT_AL44:
+      return PIPE_FORMAT_L4A4_UNORM;
    case MESA_FORMAT_AL88:
       return PIPE_FORMAT_L8A8_UNORM;
+   case MESA_FORMAT_AL1616:
+      return PIPE_FORMAT_L16A16_UNORM;
    case MESA_FORMAT_A8:
       return PIPE_FORMAT_A8_UNORM;
+   case MESA_FORMAT_A16:
+      return PIPE_FORMAT_A16_UNORM;
    case MESA_FORMAT_L8:
       return PIPE_FORMAT_L8_UNORM;
+   case MESA_FORMAT_L16:
+      return PIPE_FORMAT_L16_UNORM;
    case MESA_FORMAT_I8:
       return PIPE_FORMAT_I8_UNORM;
+   case MESA_FORMAT_I16:
+      return PIPE_FORMAT_I16_UNORM;
    case MESA_FORMAT_Z16:
       return PIPE_FORMAT_Z16_UNORM;
    case MESA_FORMAT_Z32:
@@ -261,14 +275,28 @@ st_pipe_format_to_mesa_format(enum pipe_format format)
       return MESA_FORMAT_ARGB4444;
    case PIPE_FORMAT_B5G6R5_UNORM:
       return MESA_FORMAT_RGB565;
+   case PIPE_FORMAT_B2G3R3_UNORM:
+      return MESA_FORMAT_RGB332;
+   case PIPE_FORMAT_B10G10R10A2_UNORM:
+      return MESA_FORMAT_ARGB2101010;
+   case PIPE_FORMAT_L4A4_UNORM:
+      return MESA_FORMAT_AL44;
    case PIPE_FORMAT_L8A8_UNORM:
       return MESA_FORMAT_AL88;
+   case PIPE_FORMAT_L16A16_UNORM:
+      return MESA_FORMAT_AL1616;
    case PIPE_FORMAT_A8_UNORM:
       return MESA_FORMAT_A8;
+   case PIPE_FORMAT_A16_UNORM:
+      return MESA_FORMAT_A16;
    case PIPE_FORMAT_L8_UNORM:
       return MESA_FORMAT_L8;
+   case PIPE_FORMAT_L16_UNORM:
+      return MESA_FORMAT_L16;
    case PIPE_FORMAT_I8_UNORM:
       return MESA_FORMAT_I8;
+   case PIPE_FORMAT_I16_UNORM:
+      return MESA_FORMAT_I16;
    case PIPE_FORMAT_S8_USCALED:
       return MESA_FORMAT_S8;
 
@@ -469,17 +497,27 @@ st_choose_format(struct pipe_screen *screen, GLenum internalFormat,
    unsigned geom_flags = 0; /* we don't care about POT vs. NPOT here, yet */
 
    switch (internalFormat) {
+   case GL_RGB10:
+   case GL_RGB10_A2:
+      if (screen->is_format_supported( screen, PIPE_FORMAT_B10G10R10A2_UNORM,
+                                       target, sample_count, bindings,
+                                       geom_flags ))
+         return PIPE_FORMAT_B10G10R10A2_UNORM;
+      /* Pass through. */
    case 4:
    case GL_RGBA:
    case GL_RGBA8:
-   case GL_RGB10_A2:
       return default_rgba_format( screen, target, sample_count, bindings,
                                   geom_flags );
 
    case 3:
    case GL_RGB:
+   case GL_RGB8:
       return default_rgb_format( screen, target, sample_count, bindings,
                                  geom_flags );
+
+   case GL_RGB12:
+   case GL_RGB16:
    case GL_RGBA12:
    case GL_RGBA16:
       if (screen->is_format_supported( screen, PIPE_FORMAT_R16G16B16A16_UNORM,
@@ -506,16 +544,14 @@ st_choose_format(struct pipe_screen *screen, GLenum internalFormat,
       return default_rgba_format( screen, target, sample_count, bindings,
                                   geom_flags );
 
-   case GL_RGB8:
-   case GL_RGB10:
-   case GL_RGB12:
-   case GL_RGB16:
-      return default_rgb_format( screen, target, sample_count, bindings,
-                                 geom_flags );
-
+   case GL_R3_G3_B2:
+      if (screen->is_format_supported( screen, PIPE_FORMAT_B2G3R3_UNORM,
+                                       target, sample_count, bindings,
+                                       geom_flags ))
+         return PIPE_FORMAT_B2G3R3_UNORM;
+      /* Pass through. */
    case GL_RGB5:
    case GL_RGB4:
-   case GL_R3_G3_B2:
       if (screen->is_format_supported( screen, PIPE_FORMAT_B5G6R5_UNORM,
                                        target, sample_count, bindings,
                                        geom_flags ))
@@ -527,11 +563,15 @@ st_choose_format(struct pipe_screen *screen, GLenum internalFormat,
       return default_rgba_format( screen, target, sample_count, bindings,
                                   geom_flags );
 
+   case GL_ALPHA12:
+   case GL_ALPHA16:
+      if (screen->is_format_supported( screen, PIPE_FORMAT_A16_UNORM, target,
+                                       sample_count, bindings, geom_flags ))
+         return PIPE_FORMAT_A16_UNORM;
+      /* Pass through. */
    case GL_ALPHA:
    case GL_ALPHA4:
    case GL_ALPHA8:
-   case GL_ALPHA12:
-   case GL_ALPHA16:
    case GL_COMPRESSED_ALPHA:
       if (screen->is_format_supported( screen, PIPE_FORMAT_A8_UNORM, target,
                                        sample_count, bindings, geom_flags ))
@@ -539,12 +579,16 @@ st_choose_format(struct pipe_screen *screen, GLenum internalFormat,
       return default_rgba_format( screen, target, sample_count, bindings,
                                   geom_flags );
 
+   case GL_LUMINANCE12:
+   case GL_LUMINANCE16:
+      if (screen->is_format_supported( screen, PIPE_FORMAT_L16_UNORM, target,
+                                       sample_count, bindings, geom_flags ))
+         return PIPE_FORMAT_L16_UNORM;
+      /* Pass through. */
    case 1:
    case GL_LUMINANCE:
    case GL_LUMINANCE4:
    case GL_LUMINANCE8:
-   case GL_LUMINANCE12:
-   case GL_LUMINANCE16:
    case GL_COMPRESSED_LUMINANCE:
       if (screen->is_format_supported( screen, PIPE_FORMAT_L8_UNORM, target,
                                        sample_count, bindings, geom_flags ))
@@ -552,14 +596,17 @@ st_choose_format(struct pipe_screen *screen, GLenum internalFormat,
       return default_rgba_format( screen, target, sample_count, bindings,
                                   geom_flags );
 
+   case GL_LUMINANCE12_ALPHA4:
+   case GL_LUMINANCE12_ALPHA12:
+   case GL_LUMINANCE16_ALPHA16:
+      if (screen->is_format_supported( screen, PIPE_FORMAT_L16A16_UNORM, target,
+                                       sample_count, bindings, geom_flags ))
+         return PIPE_FORMAT_L16A16_UNORM;
+      /* Pass through. */
    case 2:
    case GL_LUMINANCE_ALPHA:
-   case GL_LUMINANCE4_ALPHA4:
    case GL_LUMINANCE6_ALPHA2:
    case GL_LUMINANCE8_ALPHA8:
-   case GL_LUMINANCE12_ALPHA4:
-   case GL_LUMINANCE12_ALPHA12:
-   case GL_LUMINANCE16_ALPHA16:
    case GL_COMPRESSED_LUMINANCE_ALPHA:
       if (screen->is_format_supported( screen, PIPE_FORMAT_L8A8_UNORM, target,
                                        sample_count, bindings, geom_flags ))
@@ -567,11 +614,25 @@ st_choose_format(struct pipe_screen *screen, GLenum internalFormat,
       return default_rgba_format( screen, target, sample_count, bindings,
                                   geom_flags );
 
+   case GL_LUMINANCE4_ALPHA4:
+      if (screen->is_format_supported( screen, PIPE_FORMAT_L4A4_UNORM, target,
+                                       sample_count, bindings, geom_flags ))
+         return PIPE_FORMAT_L4A4_UNORM;
+      if (screen->is_format_supported( screen, PIPE_FORMAT_L8A8_UNORM, target,
+                                       sample_count, bindings, geom_flags ))
+         return PIPE_FORMAT_L8A8_UNORM;
+      return default_rgba_format( screen, target, sample_count, bindings,
+                                  geom_flags );
+
+   case GL_INTENSITY12:
+   case GL_INTENSITY16:
+      if (screen->is_format_supported( screen, PIPE_FORMAT_I16_UNORM, target,
+                                       sample_count, bindings, geom_flags ))
+         return PIPE_FORMAT_I16_UNORM;
+      /* Pass through. */
    case GL_INTENSITY:
    case GL_INTENSITY4:
    case GL_INTENSITY8:
-   case GL_INTENSITY12:
-   case GL_INTENSITY16:
    case GL_COMPRESSED_INTENSITY:
       if (screen->is_format_supported( screen, PIPE_FORMAT_I8_UNORM, target,
                                        sample_count, bindings, geom_flags ))
@@ -680,10 +741,10 @@ st_choose_format(struct pipe_screen *screen, GLenum internalFormat,
    case GL_DEPTH_COMPONENT:
       {
          static const enum pipe_format formats[] = {
-            PIPE_FORMAT_Z16_UNORM,
             PIPE_FORMAT_Z32_UNORM,
             PIPE_FORMAT_Z24_UNORM_S8_USCALED,
-            PIPE_FORMAT_S8_USCALED_Z24_UNORM
+            PIPE_FORMAT_S8_USCALED_Z24_UNORM,
+            PIPE_FORMAT_Z16_UNORM
          };
          return find_supported_format(screen, formats, Elements(formats),
                                       target, sample_count, bindings, geom_flags);
@@ -717,18 +778,30 @@ st_choose_format(struct pipe_screen *screen, GLenum internalFormat,
 
    case GL_SRGB_EXT:
    case GL_SRGB8_EXT:
-   case GL_COMPRESSED_SRGB_EXT:
-   case GL_COMPRESSED_SRGB_ALPHA_EXT:
    case GL_SRGB_ALPHA_EXT:
    case GL_SRGB8_ALPHA8_EXT:
       return default_srgba_format( screen, target, sample_count, bindings,
                                    geom_flags );
+
+   case GL_COMPRESSED_SRGB_EXT:
    case GL_COMPRESSED_SRGB_S3TC_DXT1_EXT:
-      return PIPE_FORMAT_DXT1_SRGB;
+      if (screen->is_format_supported(screen, PIPE_FORMAT_DXT1_SRGB, target,
+                                      sample_count, bindings, geom_flags))
+         return PIPE_FORMAT_DXT1_SRGB;
+      return default_srgba_format( screen, target, sample_count, bindings,
+                                   geom_flags );
+
    case GL_COMPRESSED_SRGB_ALPHA_S3TC_DXT1_EXT:
       return PIPE_FORMAT_DXT1_SRGBA;
+
+   case GL_COMPRESSED_SRGB_ALPHA_EXT:
    case GL_COMPRESSED_SRGB_ALPHA_S3TC_DXT3_EXT:
-      return PIPE_FORMAT_DXT3_SRGBA;
+      if (screen->is_format_supported(screen, PIPE_FORMAT_DXT3_SRGBA, target,
+                                      sample_count, bindings, geom_flags))
+         return PIPE_FORMAT_DXT3_SRGBA;
+      return default_srgba_format( screen, target, sample_count, bindings,
+                                   geom_flags );
+
    case GL_COMPRESSED_SRGB_ALPHA_S3TC_DXT5_EXT:
       return PIPE_FORMAT_DXT5_SRGBA;
 
@@ -955,8 +1028,13 @@ gl_format
 st_ChooseTextureFormat(struct gl_context *ctx, GLint internalFormat,
                        GLenum format, GLenum type)
 {
+   boolean want_renderable =
+      internalFormat == 3 || internalFormat == 4 ||
+      internalFormat == GL_RGB || internalFormat == GL_RGBA ||
+      internalFormat == GL_RGB8 || internalFormat == GL_RGBA8;
+
    return st_ChooseTextureFormat_renderable(ctx, internalFormat,
-					    format, type, GL_TRUE);
+					    format, type, want_renderable);
 }
 
 /**
diff --git a/src/mesa/state_tracker/st_gen_mipmap.h b/src/mesa/state_tracker/st_gen_mipmap.h
index 3ba091da151..815c6a51638 100644
--- a/src/mesa/state_tracker/st_gen_mipmap.h
+++ b/src/mesa/state_tracker/st_gen_mipmap.h
@@ -30,8 +30,10 @@
 #define ST_GEN_MIPMAP_H
 
 
-#include "main/mtypes.h"
+#include "main/glheader.h"
 
+struct gl_context;
+struct gl_texture_object;
 struct st_context;
 
 extern void
diff --git a/src/mesa/state_tracker/st_manager.c b/src/mesa/state_tracker/st_manager.c
index 0307b48978b..059460a7628 100644
--- a/src/mesa/state_tracker/st_manager.c
+++ b/src/mesa/state_tracker/st_manager.c
@@ -513,7 +513,8 @@ st_context_flush(struct st_context_iface *stctxi, unsigned flags,
 }
 
 static boolean
-st_context_teximage(struct st_context_iface *stctxi, enum st_texture_type target,
+st_context_teximage(struct st_context_iface *stctxi,
+                    enum st_texture_type target,
                     int level, enum pipe_format internal_format,
                     struct pipe_resource *tex, boolean mipmap)
 {
@@ -865,7 +866,8 @@ st_manager_validate_framebuffers(struct st_context *st)
  * Add a color renderbuffer on demand.
  */
 boolean
-st_manager_add_color_renderbuffer(struct st_context *st, struct gl_framebuffer *fb,
+st_manager_add_color_renderbuffer(struct st_context *st,
+                                  struct gl_framebuffer *fb,
                                   gl_buffer_index idx)
 {
    struct st_framebuffer *stfb = st_ws_framebuffer(fb);
diff --git a/src/mesa/state_tracker/st_mesa_to_tgsi.h b/src/mesa/state_tracker/st_mesa_to_tgsi.h
index 9bfd4960b60..0615e52ef62 100644
--- a/src/mesa/state_tracker/st_mesa_to_tgsi.h
+++ b/src/mesa/state_tracker/st_mesa_to_tgsi.h
@@ -29,18 +29,20 @@
 #ifndef ST_MESA_TO_TGSI_H
 #define ST_MESA_TO_TGSI_H
 
-#include "main/mtypes.h"
-
-#include "pipe/p_compiler.h"
-
-struct ureg_program;
-
 #if defined __cplusplus
 extern "C" {
 #endif
 
-struct tgsi_token;
+#include "main/glheader.h"
+
+#include "pipe/p_compiler.h"
+#include "pipe/p_defines.h"
+
+struct gl_context;
 struct gl_program;
+struct tgsi_token;
+struct ureg_program;
+
 
 enum pipe_error
 st_translate_mesa_program(
diff --git a/src/mesa/state_tracker/st_program.c b/src/mesa/state_tracker/st_program.c
index cfdc96b9dbe..cabec14e0e4 100644
--- a/src/mesa/state_tracker/st_program.c
+++ b/src/mesa/state_tracker/st_program.c
@@ -409,6 +409,7 @@ st_translate_fragment_program(struct st_context *st,
 
    assert(!(key->bitmap && key->drawpixels));
 
+#if FEATURE_drawpix
    if (key->bitmap) {
       /* glBitmap drawing */
       struct gl_fragment_program *fp;
@@ -434,6 +435,7 @@ st_translate_fragment_program(struct st_context *st,
       }
       stfp = st_fragment_program(fp);
    }
+#endif
 
    if (!stfp->tgsi.tokens) {
       /* need to translate Mesa instructions to TGSI now */
@@ -444,6 +446,7 @@ st_translate_fragment_program(struct st_context *st,
       enum pipe_error error;
       const GLbitfield inputsRead = stfp->Base.Base.InputsRead;
       struct ureg_program *ureg;
+      GLboolean write_all = GL_FALSE;
 
       ubyte input_semantic_name[PIPE_MAX_SHADER_INPUTS];
       ubyte input_semantic_index[PIPE_MAX_SHADER_INPUTS];
@@ -568,6 +571,8 @@ st_translate_fragment_program(struct st_context *st,
                   /* handled above */
                   assert(0);
                   break;
+               case FRAG_RESULT_COLOR:
+                  write_all = GL_TRUE; /* fallthrough */
                default:
                   assert(attr == FRAG_RESULT_COLOR ||
                          (FRAG_RESULT_DATA0 <= attr && attr < FRAG_RESULT_MAX));
@@ -592,6 +597,8 @@ st_translate_fragment_program(struct st_context *st,
          _mesa_print_program_parameters(st->ctx, &stfp->Base.Base);
          debug_printf("\n");
       }
+      if (write_all == GL_TRUE)
+         ureg_property_fs_color0_writes_all_cbufs(ureg, 1);
 
       error = st_translate_mesa_program(st->ctx,
                                         TGSI_PROCESSOR_FRAGMENT,
diff --git a/src/mesa/swrast/s_aaline.h b/src/mesa/swrast/s_aaline.h
index f7d92c52412..74d5518e179 100644
--- a/src/mesa/swrast/s_aaline.h
+++ b/src/mesa/swrast/s_aaline.h
@@ -28,7 +28,7 @@
 #define S_AALINE_H
 
 
-#include "main/mtypes.h"
+struct gl_context;
 
 
 extern void
diff --git a/src/mesa/swrast/s_aatriangle.h b/src/mesa/swrast/s_aatriangle.h
index 746e456f5f4..e40efb1985b 100644
--- a/src/mesa/swrast/s_aatriangle.h
+++ b/src/mesa/swrast/s_aatriangle.h
@@ -28,7 +28,7 @@
 #define S_AATRIANGLE_H
 
 
-#include "main/mtypes.h"
+struct gl_context;
 
 
 extern void
diff --git a/src/mesa/swrast/s_accum.h b/src/mesa/swrast/s_accum.h
index 071644b64fa..1ad7d329529 100644
--- a/src/mesa/swrast/s_accum.h
+++ b/src/mesa/swrast/s_accum.h
@@ -27,7 +27,8 @@
 #define S_ACCUM_H
 
 
-#include "main/mtypes.h"
+struct gl_context;
+struct gl_renderbuffer;
 
 
 extern void
diff --git a/src/mesa/swrast/s_alpha.h b/src/mesa/swrast/s_alpha.h
index 7cd6d800b29..fca209a4467 100644
--- a/src/mesa/swrast/s_alpha.h
+++ b/src/mesa/swrast/s_alpha.h
@@ -1,4 +1,3 @@
-
 /*
  * Mesa 3-D graphics library
  * Version:  4.1
@@ -28,9 +27,10 @@
 #define S_ALPHA_H
 
 
-#include "main/mtypes.h"
+#include "main/glheader.h"
 #include "s_span.h"
 
+struct gl_context;
 
 extern GLint
 _swrast_alpha_test( const struct gl_context *ctx, SWspan *span );
diff --git a/src/mesa/swrast/s_atifragshader.h b/src/mesa/swrast/s_atifragshader.h
index 39a6e64ed92..10aaaa592c7 100644
--- a/src/mesa/swrast/s_atifragshader.h
+++ b/src/mesa/swrast/s_atifragshader.h
@@ -27,9 +27,9 @@
 #define S_ATIFRAGSHADER_H
 
 
-#include "main/mtypes.h"
 #include "s_span.h"
 
+struct gl_context;
 
 extern void
 _swrast_exec_fragment_shader( struct gl_context *ctx, SWspan *span );
diff --git a/src/mesa/swrast/s_blend.h b/src/mesa/swrast/s_blend.h
index 8b06dd5031e..69cd89e7ac4 100644
--- a/src/mesa/swrast/s_blend.h
+++ b/src/mesa/swrast/s_blend.h
@@ -27,9 +27,12 @@
 #define S_BLEND_H
 
 
-#include "main/mtypes.h"
+#include "main/glheader.h"
 #include "s_span.h"
 
+struct gl_context;
+struct gl_renderbuffer;
+
 
 extern void
 _swrast_blend_span(struct gl_context *ctx, struct gl_renderbuffer *rb, SWspan *span);
diff --git a/src/mesa/swrast/s_depth.h b/src/mesa/swrast/s_depth.h
index e5dae7ef865..44820ac8f76 100644
--- a/src/mesa/swrast/s_depth.h
+++ b/src/mesa/swrast/s_depth.h
@@ -27,9 +27,12 @@
 #define S_DEPTH_H
 
 
-#include "main/mtypes.h"
+#include "main/glheader.h"
 #include "s_span.h"
 
+struct gl_context;
+struct gl_renderbuffer;
+
 
 extern GLuint
 _swrast_depth_test_span( struct gl_context *ctx, SWspan *span);
diff --git a/src/mesa/swrast/s_fog.h b/src/mesa/swrast/s_fog.h
index ebc3513f496..9f93b705084 100644
--- a/src/mesa/swrast/s_fog.h
+++ b/src/mesa/swrast/s_fog.h
@@ -28,9 +28,10 @@
 #define S_FOG_H
 
 
-#include "main/mtypes.h"
+#include "main/glheader.h"
 #include "s_span.h"
 
+struct gl_context;
 
 extern GLfloat
 _swrast_z_to_fogfactor(struct gl_context *ctx, GLfloat z);
diff --git a/src/mesa/swrast/s_fragprog.h b/src/mesa/swrast/s_fragprog.h
index 689d3fc82e4..62a68361769 100644
--- a/src/mesa/swrast/s_fragprog.h
+++ b/src/mesa/swrast/s_fragprog.h
@@ -27,9 +27,10 @@
 #define S_FRAGPROG_H
 
 
-#include "main/mtypes.h"
 #include "s_span.h"
 
+struct gl_context;
+
 
 extern void
 _swrast_exec_fragment_program(struct gl_context *ctx, SWspan *span);
diff --git a/src/mesa/swrast/s_logic.h b/src/mesa/swrast/s_logic.h
index 95c7fe3e150..0a3adfca5ff 100644
--- a/src/mesa/swrast/s_logic.h
+++ b/src/mesa/swrast/s_logic.h
@@ -27,9 +27,11 @@
 #define S_LOGIC_H
 
 
-#include "main/mtypes.h"
 #include "s_span.h"
 
+struct gl_context;
+struct gl_renderbuffer;
+
 extern void
 _swrast_logicop_rgba_span(struct gl_context *ctx, struct gl_renderbuffer *rb,
                           SWspan *span);
diff --git a/src/mesa/swrast/s_masking.h b/src/mesa/swrast/s_masking.h
index 3712c82163b..5124509a049 100644
--- a/src/mesa/swrast/s_masking.h
+++ b/src/mesa/swrast/s_masking.h
@@ -27,9 +27,12 @@
 #define S_MASKING_H
 
 
-#include "main/mtypes.h"
+#include "main/glheader.h"
 #include "s_span.h"
 
+struct gl_context;
+struct gl_renderbuffer;
+
 
 extern void
 _swrast_mask_rgba_span(struct gl_context *ctx, struct gl_renderbuffer *rb,
diff --git a/src/mesa/swrast/s_span.h b/src/mesa/swrast/s_span.h
index 18e275ec8ab..afafbe09acf 100644
--- a/src/mesa/swrast/s_span.h
+++ b/src/mesa/swrast/s_span.h
@@ -28,7 +28,12 @@
 #define S_SPAN_H
 
 
-#include "swrast.h"
+#include "main/config.h"
+#include "main/glheader.h"
+#include "main/mtypes.h"
+
+struct gl_context;
+struct gl_renderbuffer;
 
 
 /**
diff --git a/src/mesa/swrast/s_texcombine.h b/src/mesa/swrast/s_texcombine.h
index 5f47ebecbf1..11049d86b26 100644
--- a/src/mesa/swrast/s_texcombine.h
+++ b/src/mesa/swrast/s_texcombine.h
@@ -27,9 +27,10 @@
 #define S_TEXCOMBINE_H
 
 
-#include "main/mtypes.h"
 #include "s_span.h"
 
+struct gl_context;
+
 extern void
 _swrast_texture_span( struct gl_context *ctx, SWspan *span );
 
diff --git a/src/mesa/swrast/s_texfilter.c b/src/mesa/swrast/s_texfilter.c
index 539d878ddb4..ecc09e095fd 100644
--- a/src/mesa/swrast/s_texfilter.c
+++ b/src/mesa/swrast/s_texfilter.c
@@ -27,7 +27,6 @@
 #include "main/context.h"
 #include "main/colormac.h"
 #include "main/imports.h"
-#include "main/texformat.h"
 
 #include "s_context.h"
 #include "s_texfilter.h"
diff --git a/src/mesa/swrast/s_texfilter.h b/src/mesa/swrast/s_texfilter.h
index 34520f22948..69f2d80003a 100644
--- a/src/mesa/swrast/s_texfilter.h
+++ b/src/mesa/swrast/s_texfilter.h
@@ -27,9 +27,11 @@
 #define S_TEXFILTER_H
 
 
-#include "main/mtypes.h"
 #include "s_context.h"
 
+struct gl_context;
+struct gl_texture_object;
+
 
 extern texture_sample_func
 _swrast_choose_texture_sample_func( struct gl_context *ctx,
diff --git a/src/mesa/swrast/s_triangle.c b/src/mesa/swrast/s_triangle.c
index 85513e1f204..066f18203f1 100644
--- a/src/mesa/swrast/s_triangle.c
+++ b/src/mesa/swrast/s_triangle.c
@@ -34,7 +34,7 @@
 #include "main/colormac.h"
 #include "main/imports.h"
 #include "main/macros.h"
-#include "main/texformat.h"
+#include "main/mtypes.h"
 #include "program/prog_instruction.h"
 
 #include "s_aatriangle.h"
diff --git a/src/mesa/swrast_setup/ss_triangle.h b/src/mesa/swrast_setup/ss_triangle.h
index 05110865daf..a027f48269e 100644
--- a/src/mesa/swrast_setup/ss_triangle.h
+++ b/src/mesa/swrast_setup/ss_triangle.h
@@ -29,7 +29,7 @@
 #ifndef SS_TRIANGLE_H
 #define SS_TRIANGLE_H
 
-#include "main/mtypes.h"
+struct gl_context;
 
 
 void _swsetup_trifuncs_init( struct gl_context *ctx );
diff --git a/src/mesa/swrast_setup/ss_vb.h b/src/mesa/swrast_setup/ss_vb.h
index b8322f35a3d..05e665b5c30 100644
--- a/src/mesa/swrast_setup/ss_vb.h
+++ b/src/mesa/swrast_setup/ss_vb.h
@@ -29,7 +29,7 @@
 #ifndef SS_VB_H
 #define SS_VB_H
 
-#include "main/mtypes.h"
+struct gl_context;
 
 void _swsetup_vb_init( struct gl_context *ctx );
 void _swsetup_choose_rastersetup_func( struct gl_context *ctx );
diff --git a/src/mesa/tnl/t_vertex.h b/src/mesa/tnl/t_vertex.h
index 252f2f7c295..83b0dbcebbd 100644
--- a/src/mesa/tnl/t_vertex.h
+++ b/src/mesa/tnl/t_vertex.h
@@ -28,9 +28,12 @@
 #ifndef _TNL_VERTEX_H
 #define _TNL_VERTEX_H
 
-#include "main/mtypes.h"
+#include "main/glheader.h"
 #include "t_context.h"
 
+struct gl_context;
+struct tnl_clipspace;
+
 /* New mechanism to specify hardware vertices so that tnl can build
  * and manipulate them directly.  
  */
diff --git a/src/mesa/tnl/t_vp_build.c b/src/mesa/tnl/t_vp_build.c
index 421ec88a454..70492a4353d 100644
--- a/src/mesa/tnl/t_vp_build.c
+++ b/src/mesa/tnl/t_vp_build.c
@@ -32,7 +32,7 @@
 
 #include "main/glheader.h"
 #include "main/ffvertex_prog.h"
-#include "main/dd.h"
+#include "main/mtypes.h"
 #include "t_vp_build.h"
 
 
diff --git a/src/mesa/tnl/t_vp_build.h b/src/mesa/tnl/t_vp_build.h
index 1d10ff245d9..e9f6de5a920 100644
--- a/src/mesa/tnl/t_vp_build.h
+++ b/src/mesa/tnl/t_vp_build.h
@@ -27,7 +27,7 @@
 #ifndef T_VP_BUILD_H
 #define T_VP_BUILD_H
 
-#include "main/mtypes.h"
+struct gl_context;
 
 #define TNL_FIXED_FUNCTION_STATE_FLAGS (_NEW_PROGRAM |		\
 					_NEW_LIGHT |		\
diff --git a/src/mesa/tnl/tnl.h b/src/mesa/tnl/tnl.h
index 702efdc5cca..24a0c725008 100644
--- a/src/mesa/tnl/tnl.h
+++ b/src/mesa/tnl/tnl.h
@@ -28,8 +28,11 @@
 #ifndef _TNL_H
 #define _TNL_H
 
-#include "main/mtypes.h"
+#include "main/glheader.h"
 
+struct gl_client_array;
+struct gl_context;
+struct gl_program;
 
 
 /* These are the public-access functions exported from tnl.  (A few
diff --git a/src/mesa/vbo/vbo.h b/src/mesa/vbo/vbo.h
index 79f76655354..e221538bad4 100644
--- a/src/mesa/vbo/vbo.h
+++ b/src/mesa/vbo/vbo.h
@@ -32,7 +32,10 @@
 #ifndef _VBO_H
 #define _VBO_H
 
-#include "main/mtypes.h"
+#include "main/glheader.h"
+
+struct gl_client_array;
+struct gl_context;
 
 struct _mesa_prim {
    GLuint mode:8;
diff --git a/src/mesa/vbo/vbo_context.c b/src/mesa/vbo/vbo_context.c
index 9992cc34739..6656ed89816 100644
--- a/src/mesa/vbo/vbo_context.c
+++ b/src/mesa/vbo/vbo_context.c
@@ -209,7 +209,6 @@ GLboolean _vbo_CreateContext( struct gl_context *ctx )
 
 void _vbo_InvalidateState( struct gl_context *ctx, GLuint new_state )
 {
-   _ae_invalidate_state(ctx, new_state);
    vbo_exec_invalidate_state(ctx, new_state);
 }
 
diff --git a/src/mesa/x86/mmx.h b/src/mesa/x86/mmx.h
index d4bda07eade..74e9979d312 100644
--- a/src/mesa/x86/mmx.h
+++ b/src/mesa/x86/mmx.h
@@ -27,7 +27,9 @@
 #define ASM_MMX_H
 
 #include "main/compiler.h"
-#include "main/mtypes.h"
+#include "main/glheader.h"
+
+struct gl_context;
 
 extern void _ASMAPI
 _mesa_mmx_blend_transparency( struct gl_context *ctx, GLuint n, const GLubyte mask[],