r600g: rework vertex buffer uploads

Only upload the [min_index, max_index] range instead of [0, userbuf_size].
This an important optimization.

Framerate in Lightsmark:
Before: 22 fps
After: 75 fps

The same optimization is already in r300g.

3 files changed, 106 insertions, 84 deletions

diff --git a/src/gallium/drivers/r600/r600_buffer.c b/src/gallium/drivers/r600/r600_buffer.c
index 024bbc29168..6ebe6ab2364 100644
--- a/src/gallium/drivers/r600/r600_buffer.c
+++ b/src/gallium/drivers/r600/r600_buffer.c
@@ -201,39 +201,52 @@ struct u_resource_vtbl r600_buffer_vtbl =
 
 void r600_upload_index_buffer(struct r600_pipe_context *rctx, struct r600_drawl *draw)
 {
-
-	if (r600_is_user_buffer(draw->index_buffer)) {
-		struct r600_resource_buffer *rbuffer = r600_buffer(draw->index_buffer);
-		boolean flushed;
-
-		u_upload_data(rctx->upload_vb, 0,
-			      draw->info.count * draw->index_size,
-			      rbuffer->user_buffer,
-			      &draw->index_buffer_offset,
-			      &draw->index_buffer, &flushed);
-	}
+	struct r600_resource_buffer *rbuffer = r600_buffer(draw->index_buffer);
+	boolean flushed;
+
+	u_upload_data(rctx->upload_vb, 0,
+		      draw->info.count * draw->index_size,
+		      rbuffer->user_buffer,
+		      &draw->index_buffer_offset,
+		      &draw->index_buffer, &flushed);
 }
 
-void r600_upload_user_buffers(struct r600_pipe_context *rctx)
+void r600_upload_user_buffers(struct r600_pipe_context *rctx,
+			      int min_index, int max_index)
 {
-	int i, nr;
-
-	nr = rctx->vertex_elements->count;
-	nr = rctx->nvertex_buffer;
+	int i, nr = rctx->vertex_elements->count;
+	unsigned count = max_index + 1 - min_index;
+	boolean flushed;
+	boolean uploaded[32] = {0};
 
 	for (i = 0; i < nr; i++) {
-		struct pipe_vertex_buffer *vb = &rctx->vertex_buffer[i];
-
-		if (r600_is_user_buffer(vb->buffer)) {
-			struct r600_resource_buffer *rbuffer = r600_buffer(vb->buffer);
-			boolean flushed;
-
-			u_upload_data(rctx->upload_vb, 0,
-				      vb->buffer->width0,
-				      rbuffer->user_buffer,
+		unsigned index = rctx->vertex_elements->elements[i].vertex_buffer_index;
+		struct pipe_vertex_buffer *vb = &rctx->vertex_buffer[index];
+		struct r600_resource_buffer *userbuf = r600_buffer(vb->buffer);
+
+		if (userbuf && userbuf->user_buffer && !uploaded[index]) {
+			unsigned first, size;
+
+			if (vb->stride) {
+			    first = vb->stride * min_index;
+			    size = vb->stride * count;
+			} else {
+			    first = 0;
+			    size = rctx->vertex_elements->hw_format_size[i];
+			}
+
+			u_upload_data(rctx->upload_vb, first, size,
+				      userbuf->user_buffer + first,
 				      &vb->buffer_offset,
-				      &vb->buffer,
+				      &rctx->real_vertex_buffer[index],
 				      &flushed);
+
+			vb->buffer_offset -= first;
+
+			/* vertex_arrays_dirty = TRUE; */
+			uploaded[index] = TRUE;
+		} else {
+			assert(rctx->real_vertex_buffer[index]);
 		}
 	}
 }
diff --git a/src/gallium/drivers/r600/r600_pipe.h b/src/gallium/drivers/r600/r600_pipe.h
index a0164868f34..360ee2af1fc 100644
--- a/src/gallium/drivers/r600/r600_pipe.h
+++ b/src/gallium/drivers/r600/r600_pipe.h
@@ -144,6 +144,7 @@ struct r600_pipe_context {
 	struct pipe_framebuffer_state	framebuffer;
 	struct pipe_index_buffer	index_buffer;
 	struct pipe_vertex_buffer	vertex_buffer[PIPE_MAX_ATTRIBS];
+	struct pipe_resource		*real_vertex_buffer[PIPE_MAX_ATTRIBS];
 	unsigned			nvertex_buffer;
 	unsigned			cb_target_mask;
 	/* for saving when using blitter */
@@ -207,7 +208,8 @@ unsigned r600_buffer_is_referenced_by_cs(struct pipe_context *context,
 struct pipe_resource *r600_buffer_from_handle(struct pipe_screen *screen,
 					      struct winsys_handle *whandle);
 void r600_upload_index_buffer(struct r600_pipe_context *rctx, struct r600_drawl *draw);
-void r600_upload_user_buffers(struct r600_pipe_context *rctx);
+void r600_upload_user_buffers(struct r600_pipe_context *rctx,
+			      int min_index, int max_index);
 
 /* r600_query.c */
 void r600_init_query_functions(struct r600_pipe_context *rctx);
@@ -279,7 +281,6 @@ void r600_delete_ps_shader(struct pipe_context *ctx, void *state);
 void r600_delete_vs_shader(struct pipe_context *ctx, void *state);
 void r600_set_constant_buffer(struct pipe_context *ctx, uint shader, uint index,
 			      struct pipe_resource *buffer);
-void r600_vertex_buffer_update(struct r600_pipe_context *rctx);
 void r600_draw_vbo(struct pipe_context *ctx, const struct pipe_draw_info *info);
 
 /*
diff --git a/src/gallium/drivers/r600/r600_state_common.c b/src/gallium/drivers/r600/r600_state_common.c
index 26e5981e982..409a07bda17 100644
--- a/src/gallium/drivers/r600/r600_state_common.c
+++ b/src/gallium/drivers/r600/r600_state_common.c
@@ -121,20 +121,10 @@ void r600_bind_vertex_elements(struct pipe_context *ctx, void *state)
 	struct r600_pipe_context *rctx = (struct r600_pipe_context *)ctx;
 	struct r600_vertex_element *v = (struct r600_vertex_element*)state;
 
-	/* delete previous translated vertex elements */
-	if (rctx->tran.new_velems) {
-		r600_end_vertex_translate(rctx);
-	}
-
 	rctx->vertex_elements = v;
 	if (v) {
 		rctx->states[v->rstate.id] = &v->rstate;
 		r600_context_pipe_state_set(&rctx->ctx, &v->rstate);
-		r600_vertex_buffer_update(rctx);
-	}
-
-	if (v) {
-//		rctx->vs_rebuild = TRUE;
 	}
 }
 
@@ -175,45 +165,62 @@ void r600_set_vertex_buffers(struct pipe_context *ctx, unsigned count,
 {
 	struct r600_pipe_context *rctx = (struct r600_pipe_context *)ctx;
 	struct pipe_vertex_buffer *vbo;
-	unsigned max_index = (unsigned)-1;
-
-	if (rctx->family >= CHIP_CEDAR) {
-		for (int i = 0; i < rctx->nvertex_buffer; i++) {
-			pipe_resource_reference(&rctx->vertex_buffer[i].buffer, NULL);
-			evergreen_context_pipe_state_set_fs_resource(&rctx->ctx, NULL, i);
-		}
-	} else {
-		for (int i = 0; i < rctx->nvertex_buffer; i++) {
-			pipe_resource_reference(&rctx->vertex_buffer[i].buffer, NULL);
-			r600_context_pipe_state_set_fs_resource(&rctx->ctx, NULL, i);
-		}
-	}
-	memcpy(rctx->vertex_buffer, buffers, sizeof(struct pipe_vertex_buffer) * count);
+	unsigned max_index = ~0;
+	int i;
 
-	for (int i = 0; i < count; i++) {
+	for (i = 0; i < count; i++) {
 		vbo = (struct pipe_vertex_buffer*)&buffers[i];
 
-		rctx->vertex_buffer[i].buffer = NULL;
-		if (buffers[i].buffer == NULL)
+		pipe_resource_reference(&rctx->vertex_buffer[i].buffer, vbo->buffer);
+		pipe_resource_reference(&rctx->real_vertex_buffer[i], NULL);
+
+		if (!vbo->buffer) {
+			/* Zero states. */
+			if (rctx->family >= CHIP_CEDAR) {
+				evergreen_context_pipe_state_set_fs_resource(&rctx->ctx, NULL, i);
+			} else {
+				r600_context_pipe_state_set_fs_resource(&rctx->ctx, NULL, i);
+			}
 			continue;
-		if (r600_is_user_buffer(buffers[i].buffer))
+		}
+
+		if (r600_is_user_buffer(vbo->buffer)) {
 			rctx->any_user_vbs = TRUE;
-		pipe_resource_reference(&rctx->vertex_buffer[i].buffer, buffers[i].buffer);
+			continue;
+		}
+
+		pipe_resource_reference(&rctx->real_vertex_buffer[i], vbo->buffer);
 
 		/* The stride of zero means we will be fetching only the first
 		 * vertex, so don't care about max_index. */
-		if (!vbo->stride)
+		if (!vbo->stride) {
 			continue;
+		}
 
-		if (vbo->max_index == ~0) {
-			vbo->max_index = (vbo->buffer->width0 - vbo->buffer_offset) / vbo->stride;
+		/* Update the maximum index. */
+		{
+		    unsigned vbo_max_index =
+			  (vbo->buffer->width0 - vbo->buffer_offset) / vbo->stride;
+		    max_index = MIN2(max_index, vbo_max_index);
 		}
-		max_index = MIN2(vbo->max_index, max_index);
 	}
+
+	for (; i < rctx->nvertex_buffer; i++) {
+		pipe_resource_reference(&rctx->vertex_buffer[i].buffer, NULL);
+		pipe_resource_reference(&rctx->real_vertex_buffer[i], NULL);
+
+		/* Zero states. */
+		if (rctx->family >= CHIP_CEDAR) {
+			evergreen_context_pipe_state_set_fs_resource(&rctx->ctx, NULL, i);
+		} else {
+			r600_context_pipe_state_set_fs_resource(&rctx->ctx, NULL, i);
+		}
+	}
+
+	memcpy(rctx->vertex_buffer, buffers, sizeof(struct pipe_vertex_buffer) * count);
+
 	rctx->nvertex_buffer = count;
 	rctx->vb_max_index = max_index;
-
-	r600_vertex_buffer_update(rctx);
 }
 
 
@@ -407,30 +414,13 @@ void r600_set_constant_buffer(struct pipe_context *ctx, uint shader, uint index,
 		pipe_resource_reference((struct pipe_resource**)&rbuffer, NULL);
 }
 
-void r600_vertex_buffer_update(struct r600_pipe_context *rctx)
+static void r600_vertex_buffer_update(struct r600_pipe_context *rctx)
 {
 	struct r600_pipe_state *rstate;
 	struct r600_resource *rbuffer;
 	struct pipe_vertex_buffer *vertex_buffer;
 	unsigned i, offset;
 
-	/* we don't update until we know vertex elements */
-	if (rctx->vertex_elements == NULL || !rctx->nvertex_buffer)
-		return;
-
-	if (rctx->vertex_elements->incompatible_layout) {
-		/* translate rebind new vertex elements so
-		 * return once translated
-		 */
-		r600_begin_vertex_translate(rctx);
-		return;
-	}
-
-	if (rctx->any_user_vbs) {
-		r600_upload_user_buffers(rctx);
-		rctx->any_user_vbs = FALSE;
-	}
-
 	if (rctx->vertex_elements->vbuffer_need_offset) {
 		/* one resource per vertex elements */
 		rctx->nvs_resource = rctx->vertex_elements->count;
@@ -449,12 +439,12 @@ void r600_vertex_buffer_update(struct r600_pipe_context *rctx)
 			unsigned vbuffer_index;
 			vbuffer_index = rctx->vertex_elements->elements[i].vertex_buffer_index;
 			vertex_buffer = &rctx->vertex_buffer[vbuffer_index];
-			rbuffer = (struct r600_resource*)vertex_buffer->buffer;
+			rbuffer = (struct r600_resource*)rctx->real_vertex_buffer[vbuffer_index];
 			offset = rctx->vertex_elements->vbuffer_offset[i];
 		} else {
 			/* bind vertex buffer once */
 			vertex_buffer = &rctx->vertex_buffer[i];
-			rbuffer = (struct r600_resource*)vertex_buffer->buffer;
+			rbuffer = (struct r600_resource*)rctx->real_vertex_buffer[i];
 			offset = 0;
 		}
 		if (vertex_buffer == NULL || rbuffer == NULL)
@@ -483,6 +473,16 @@ void r600_draw_vbo(struct pipe_context *ctx, const struct pipe_draw_info *info)
 	struct r600_drawl draw = {};
 	unsigned prim;
 
+	if (rctx->vertex_elements->incompatible_layout) {
+		r600_begin_vertex_translate(rctx);
+	}
+
+	if (rctx->any_user_vbs) {
+		r600_upload_user_buffers(rctx, info->min_index, info->max_index);
+	}
+
+	r600_vertex_buffer_update(rctx);
+
 	draw.info = *info;
 	draw.ctx = ctx;
 	if (info->indexed && rctx->index_buffer.buffer) {
@@ -497,7 +497,10 @@ void r600_draw_vbo(struct pipe_context *ctx, const struct pipe_draw_info *info)
 		pipe_resource_reference(&draw.index_buffer, rctx->index_buffer.buffer);
 		draw.index_buffer_offset = draw.info.start * draw.index_size;
 		draw.info.start = 0;
-		r600_upload_index_buffer(rctx, &draw);
+
+		if (r600_is_user_buffer(draw.index_buffer)) {
+			r600_upload_index_buffer(rctx, &draw);
+		}
 	} else {
 		draw.info.index_bias = info->start;
 	}
@@ -572,4 +575,9 @@ void r600_draw_vbo(struct pipe_context *ctx, const struct pipe_draw_info *info)
 	}
 
 	pipe_resource_reference(&draw.index_buffer, NULL);
+
+	/* delete previous translated vertex elements */
+	if (rctx->tran.new_velems) {
+		r600_end_vertex_translate(rctx);
+	}
 }