r600g: specialized upload manager

Allow important performance increase by doing hw specific implementation
of the upload manager helper. Drop the range flushing that is not hit with
this code (and wasn't with previous neither). Performance improvement are
mostly visible on slow CPU.

Signed-off-by: Jerome Glisse <[email protected]>

8 files changed, 176 insertions, 109 deletions

diff --git a/src/gallium/drivers/r600/Makefile b/src/gallium/drivers/r600/Makefile
index a484f38e9f1..b476b9af3b8 100644
--- a/src/gallium/drivers/r600/Makefile
+++ b/src/gallium/drivers/r600/Makefile
@@ -21,6 +21,7 @@ C_SOURCES = \
 	evergreen_state.c \
 	eg_asm.c \
 	r600_translate.c \
-	r600_state_common.c
+	r600_state_common.c \
+	r600_upload.c
 
 include ../../Makefile.template
diff --git a/src/gallium/drivers/r600/evergreen_state.c b/src/gallium/drivers/r600/evergreen_state.c
index 9e1a5e1f988..a9d4a862c32 100644
--- a/src/gallium/drivers/r600/evergreen_state.c
+++ b/src/gallium/drivers/r600/evergreen_state.c
@@ -1346,7 +1346,7 @@ void evergreen_vertex_buffer_update(struct r600_pipe_context *rctx)
 		r600_pipe_state_add_reg(rstate, R_030000_RESOURCE0_WORD0,
 					offset, 0xFFFFFFFF, rbuffer->bo);
 		r600_pipe_state_add_reg(rstate, R_030004_RESOURCE0_WORD1,
-					rbuffer->size - offset - 1, 0xFFFFFFFF, NULL);
+					rbuffer->bo_size - offset - 1, 0xFFFFFFFF, NULL);
 		r600_pipe_state_add_reg(rstate, R_030008_RESOURCE0_WORD2,
 					S_030008_STRIDE(vertex_buffer->stride),
 					0xFFFFFFFF, NULL);
diff --git a/src/gallium/drivers/r600/r600_buffer.c b/src/gallium/drivers/r600/r600_buffer.c
index 03a61a3213c..7d29f760a5d 100644
--- a/src/gallium/drivers/r600/r600_buffer.c
+++ b/src/gallium/drivers/r600/r600_buffer.c
@@ -29,7 +29,6 @@
 #include <util/u_math.h>
 #include <util/u_inlines.h>
 #include <util/u_memory.h>
-#include <util/u_upload_mgr.h>
 #include "state_tracker/drm_driver.h"
 #include <xf86drm.h>
 #include "radeon_drm.h"
@@ -53,12 +52,13 @@ struct pipe_resource *r600_buffer_create(struct pipe_screen *screen,
 
 	rbuffer->magic = R600_BUFFER_MAGIC;
 	rbuffer->user_buffer = NULL;
-	rbuffer->num_ranges = 0;
 	rbuffer->r.base.b = *templ;
 	pipe_reference_init(&rbuffer->r.base.b.reference, 1);
 	rbuffer->r.base.b.screen = screen;
 	rbuffer->r.base.vtbl = &r600_buffer_vtbl;
 	rbuffer->r.size = rbuffer->r.base.b.width0;
+	rbuffer->r.bo_size = rbuffer->r.size;
+	rbuffer->uploaded = FALSE;
 	bo = r600_bo((struct radeon*)screen->winsys, rbuffer->r.base.b.width0, alignment, rbuffer->r.base.b.bind, rbuffer->r.base.b.usage);
 	if (bo == NULL) {
 		FREE(rbuffer);
@@ -91,9 +91,10 @@ struct pipe_resource *r600_user_buffer_create(struct pipe_screen *screen,
 	rbuffer->r.base.b.depth0 = 1;
 	rbuffer->r.base.b.array_size = 1;
 	rbuffer->r.base.b.flags = 0;
-	rbuffer->num_ranges = 0;
 	rbuffer->r.bo = NULL;
+	rbuffer->r.bo_size = 0;
 	rbuffer->user_buffer = ptr;
+	rbuffer->uploaded = FALSE;
 	return &rbuffer->r.base.b;
 }
 
@@ -102,9 +103,10 @@ static void r600_buffer_destroy(struct pipe_screen *screen,
 {
 	struct r600_resource_buffer *rbuffer = r600_buffer(buf);
 
-	if (rbuffer->r.bo) {
+	if (!rbuffer->uploaded && rbuffer->r.bo) {
 		r600_bo_reference((struct radeon*)screen->winsys, &rbuffer->r.bo, NULL);
 	}
+	rbuffer->r.bo = NULL;
 	FREE(rbuffer);
 }
 
@@ -114,29 +116,10 @@ static void *r600_buffer_transfer_map(struct pipe_context *pipe,
 	struct r600_resource_buffer *rbuffer = r600_buffer(transfer->resource);
 	int write = 0;
 	uint8_t *data;
-	int i;
-	boolean flush = FALSE;
 
 	if (rbuffer->user_buffer)
 		return (uint8_t*)rbuffer->user_buffer + transfer->box.x;
 
-	if (transfer->usage & PIPE_TRANSFER_DISCARD) {
-		for (i = 0; i < rbuffer->num_ranges; i++) {
-			if ((transfer->box.x >= rbuffer->ranges[i].start) &&
-			    (transfer->box.x < rbuffer->ranges[i].end))
-				flush = TRUE;
-
-			if (flush) {
-				r600_bo_reference((struct radeon*)pipe->winsys, &rbuffer->r.bo, NULL);
-				rbuffer->num_ranges = 0;
-				rbuffer->r.bo = r600_bo((struct radeon*)pipe->winsys,
-							rbuffer->r.base.b.width0, 0,
-							rbuffer->r.base.b.bind,
-							rbuffer->r.base.b.usage);
-				break;
-			}
-		}
-	}
 	if (transfer->usage & PIPE_TRANSFER_DONTBLOCK) {
 		/* FIXME */
 	}
@@ -155,36 +138,17 @@ static void r600_buffer_transfer_unmap(struct pipe_context *pipe,
 {
 	struct r600_resource_buffer *rbuffer = r600_buffer(transfer->resource);
 
+	if (rbuffer->user_buffer)
+		return;
+
 	if (rbuffer->r.bo)
 		r600_bo_unmap((struct radeon*)pipe->winsys, rbuffer->r.bo);
 }
 
 static void r600_buffer_transfer_flush_region(struct pipe_context *pipe,
-					      struct pipe_transfer *transfer,
-					      const struct pipe_box *box)
+						struct pipe_transfer *transfer,
+						const struct pipe_box *box)
 {
-	struct r600_resource_buffer *rbuffer = r600_buffer(transfer->resource);
-	unsigned i;
-	unsigned offset = transfer->box.x + box->x;
-	unsigned length = box->width;
-
-	assert(box->x + box->width <= transfer->box.width);
-
-	if (rbuffer->user_buffer)
-		return;
-
-	/* mark the range as used */
-	for(i = 0; i < rbuffer->num_ranges; ++i) {
-		if(offset <= rbuffer->ranges[i].end && rbuffer->ranges[i].start <= (offset+box->width)) {
-			rbuffer->ranges[i].start = MIN2(rbuffer->ranges[i].start, offset);
-			rbuffer->ranges[i].end   = MAX2(rbuffer->ranges[i].end, (offset+length));
-			return;
-		}
-	}
-
-	rbuffer->ranges[rbuffer->num_ranges].start = offset;
-	rbuffer->ranges[rbuffer->num_ranges].end = offset+length;
-	rbuffer->num_ranges++;
 }
 
 unsigned r600_buffer_is_referenced_by_cs(struct pipe_context *context,
@@ -236,29 +200,25 @@ struct u_resource_vtbl r600_buffer_vtbl =
 
 int r600_upload_index_buffer(struct r600_pipe_context *rctx, struct r600_drawl *draw)
 {
-	struct pipe_resource *upload_buffer = NULL;
-	unsigned index_offset = draw->index_buffer_offset;
-	int ret = 0;
-
 	if (r600_buffer_is_user_buffer(draw->index_buffer)) {
-		ret = u_upload_buffer(rctx->upload_ib,
-				      index_offset,
-				      draw->count * draw->index_size,
-				      draw->index_buffer,
-				      &index_offset,
-				      &upload_buffer);
-		if (ret) {
-			goto done;
-		}
-		draw->index_buffer_offset = index_offset;
-
-		/* Transfer ownership. */
-		pipe_resource_reference(&draw->index_buffer, upload_buffer);
-		pipe_resource_reference(&upload_buffer, NULL);
+		struct r600_resource_buffer *rbuffer = r600_buffer(draw->index_buffer);
+		unsigned upload_offset;
+		int ret = 0;
+
+		ret = r600_upload_buffer(rctx->rupload_vb,
+					draw->index_buffer_offset,
+					draw->count * draw->index_size,
+					rbuffer,
+					&upload_offset,
+					&rbuffer->r.bo_size,
+					&rbuffer->r.bo);
+		if (ret)
+			return ret;
+		rbuffer->uploaded = TRUE;
+		draw->index_buffer_offset = upload_offset;
 	}
 
-done:
-	return ret;
+	return 0;
 }
 
 int r600_upload_user_buffers(struct r600_pipe_context *rctx)
@@ -270,23 +230,21 @@ int r600_upload_user_buffers(struct r600_pipe_context *rctx)
 	nr = rctx->nvertex_buffer;
 
 	for (i = 0; i < nr; i++) {
-//		struct pipe_vertex_buffer *vb = &rctx->vertex_buffer[rctx->vertex_elements->elements[i].vertex_buffer_index];
 		struct pipe_vertex_buffer *vb = &rctx->vertex_buffer[i];
 
 		if (r600_buffer_is_user_buffer(vb->buffer)) {
-			struct pipe_resource *upload_buffer = NULL;
-			unsigned offset = 0; /*vb->buffer_offset * 4;*/
-			unsigned size = vb->buffer->width0;
+			struct r600_resource_buffer *rbuffer = r600_buffer(vb->buffer);
 			unsigned upload_offset;
-			ret = u_upload_buffer(rctx->upload_vb,
-					      offset, size,
-					      vb->buffer,
-					      &upload_offset, &upload_buffer);
+
+			ret = r600_upload_buffer(rctx->rupload_vb,
+						0, vb->buffer->width0,
+						rbuffer,
+						&upload_offset,
+						&rbuffer->r.bo_size,
+						&rbuffer->r.bo);
 			if (ret)
 				return ret;
-
-			pipe_resource_reference(&vb->buffer, NULL);
-			vb->buffer = upload_buffer;
+			rbuffer->uploaded = TRUE;
 			vb->buffer_offset = upload_offset;
 		}
 	}
diff --git a/src/gallium/drivers/r600/r600_pipe.c b/src/gallium/drivers/r600/r600_pipe.c
index 6842571044d..72988b946e5 100644
--- a/src/gallium/drivers/r600/r600_pipe.c
+++ b/src/gallium/drivers/r600/r600_pipe.c
@@ -35,7 +35,6 @@
 #include <util/u_pack_color.h>
 #include <util/u_memory.h>
 #include <util/u_inlines.h>
-#include <util/u_upload_mgr.h>
 #include <pipebuffer/pb_buffer.h>
 #include "r600.h"
 #include "r600d.h"
@@ -59,9 +58,6 @@ static void r600_flush(struct pipe_context *ctx, unsigned flags,
 	if (!rctx->ctx.pm4_cdwords)
 		return;
 
-	u_upload_flush(rctx->upload_vb);
-	u_upload_flush(rctx->upload_ib);
-
 #if 0
 	sprintf(dname, "gallium-%08d.bof", dc);
 	if (dc < 20) {
@@ -71,6 +67,8 @@ static void r600_flush(struct pipe_context *ctx, unsigned flags,
 	dc++;
 #endif
 	r600_context_flush(&rctx->ctx);
+
+	r600_upload_flush(rctx->rupload_vb);
 }
 
 static void r600_destroy_context(struct pipe_context *context)
@@ -89,8 +87,7 @@ static void r600_destroy_context(struct pipe_context *context)
 		free(rctx->states[i]);
 	}
 
-	u_upload_destroy(rctx->upload_vb);
-	u_upload_destroy(rctx->upload_ib);
+	r600_upload_destroy(rctx->rupload_vb);
 
 	if (rctx->tran.translate_cache)
 		translate_cache_destroy(rctx->tran.translate_cache);
@@ -165,16 +162,8 @@ static struct pipe_context *r600_create_context(struct pipe_screen *screen, void
 		return NULL;
 	}
 
-	rctx->upload_ib = u_upload_create(&rctx->context, 32 * 1024, 16,
-					  PIPE_BIND_INDEX_BUFFER);
-	if (rctx->upload_ib == NULL) {
-		r600_destroy_context(&rctx->context);
-		return NULL;
-	}
-
-	rctx->upload_vb = u_upload_create(&rctx->context, 128 * 1024, 16,
-					  PIPE_BIND_VERTEX_BUFFER);
-	if (rctx->upload_vb == NULL) {
+	rctx->rupload_vb = r600_upload_create(rctx, 128 * 1024, 16);
+	if (rctx->rupload_vb == NULL) {
 		r600_destroy_context(&rctx->context);
 		return NULL;
 	}
diff --git a/src/gallium/drivers/r600/r600_pipe.h b/src/gallium/drivers/r600/r600_pipe.h
index 485f42166d0..f7f6f63c46f 100644
--- a/src/gallium/drivers/r600/r600_pipe.h
+++ b/src/gallium/drivers/r600/r600_pipe.h
@@ -131,6 +131,8 @@ struct r600_translate_context {
 #define R600_CONSTANT_ARRAY_SIZE 256
 #define R600_RESOURCE_ARRAY_SIZE 160
 
+struct r600_upload;
+
 struct r600_pipe_context {
 	struct pipe_context		context;
 	struct blitter_context		*blitter;
@@ -163,8 +165,7 @@ struct r600_pipe_context {
 	/* shader information */
 	unsigned			sprite_coord_enable;
 	bool				flatshade;
-	struct u_upload_mgr		*upload_vb;
-	struct u_upload_mgr		*upload_ib;
+	struct r600_upload		*rupload_vb;
 	unsigned			any_user_vbs;
 	struct r600_textures_info	ps_samplers;
 	unsigned			vb_max_index;
diff --git a/src/gallium/drivers/r600/r600_resource.h b/src/gallium/drivers/r600/r600_resource.h
index 25aa84682c5..e21916f2af2 100644
--- a/src/gallium/drivers/r600/r600_resource.h
+++ b/src/gallium/drivers/r600/r600_resource.h
@@ -46,6 +46,7 @@ struct r600_resource {
 	struct u_resource		base;
 	struct r600_bo			*bo;
 	u32				size;
+	unsigned			bo_size;
 };
 
 struct r600_resource_texture {
@@ -74,19 +75,12 @@ struct pipe_resource *r600_texture_from_handle(struct pipe_screen *screen,
 						struct winsys_handle *whandle);
 
 #define R600_BUFFER_MAGIC 0xabcd1600
-#define R600_BUFFER_MAX_RANGES 32
-
-struct r600_buffer_range {
-	uint32_t start;
-	uint32_t end;
-};
 
 struct r600_resource_buffer {
 	struct r600_resource r;
 	uint32_t magic;
 	void *user_buffer;
-	struct r600_buffer_range ranges[R600_BUFFER_MAX_RANGES];
-	unsigned num_ranges;
+	bool uploaded;
 };
 
 /* r600_buffer */
@@ -101,7 +95,9 @@ static INLINE struct r600_resource_buffer *r600_buffer(struct pipe_resource *buf
 
 static INLINE boolean r600_buffer_is_user_buffer(struct pipe_resource *buffer)
 {
-    return r600_buffer(buffer)->user_buffer ? TRUE : FALSE;
+	if (r600_buffer(buffer)->uploaded)
+		return FALSE;
+	return r600_buffer(buffer)->user_buffer ? TRUE : FALSE;
 }
 
 int r600_texture_depth_flush(struct pipe_context *ctx,
@@ -127,4 +123,15 @@ struct r600_surface {
 	unsigned aligned_height;
 };
 
+struct r600_pipe_context;
+struct r600_upload *r600_upload_create(struct r600_pipe_context *rctx,
+					unsigned default_size,
+					unsigned alignment);
+void r600_upload_flush(struct r600_upload *upload);
+void r600_upload_destroy(struct r600_upload *upload);
+int r600_upload_buffer(struct r600_upload *upload, unsigned offset,
+			unsigned size, struct r600_resource_buffer *in_buffer,
+			unsigned *out_offset, unsigned *out_size,
+			struct r600_bo **out_buffer);
+
 #endif
diff --git a/src/gallium/drivers/r600/r600_state.c b/src/gallium/drivers/r600/r600_state.c
index 553d786d655..67ea21712f7 100644
--- a/src/gallium/drivers/r600/r600_state.c
+++ b/src/gallium/drivers/r600/r600_state.c
@@ -36,7 +36,6 @@
 #include <util/u_pack_color.h>
 #include <util/u_memory.h>
 #include <util/u_inlines.h>
-#include <util/u_upload_mgr.h>
 #include <util/u_framebuffer.h>
 #include <pipebuffer/pb_buffer.h>
 #include "r600.h"
@@ -187,7 +186,7 @@ void r600_vertex_buffer_update(struct r600_pipe_context *rctx)
 		r600_pipe_state_add_reg(rstate, R_038000_RESOURCE0_WORD0,
 					offset, 0xFFFFFFFF, rbuffer->bo);
 		r600_pipe_state_add_reg(rstate, R_038004_RESOURCE0_WORD1,
-					rbuffer->size - offset - 1, 0xFFFFFFFF, NULL);
+					rbuffer->bo_size - offset - 1, 0xFFFFFFFF, NULL);
 		r600_pipe_state_add_reg(rstate, R_038008_RESOURCE0_WORD2,
 					S_038008_STRIDE(vertex_buffer->stride),
 					0xFFFFFFFF, NULL);
diff --git a/src/gallium/drivers/r600/r600_upload.c b/src/gallium/drivers/r600/r600_upload.c
new file mode 100644
index 00000000000..bea0fd4f0c5
--- /dev/null
+++ b/src/gallium/drivers/r600/r600_upload.c
@@ -0,0 +1,112 @@
+/*
+ * Copyright 2010 Jerome Glisse <[email protected]>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * on the rights to use, copy, modify, merge, publish, distribute, sub
+ * license, and/or sell copies of the Software, and to permit persons to whom
+ * the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
+ * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+ * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
+ * USE OR OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors:
+ *      Jerome Glisse <[email protected]>
+ */
+#include <errno.h>
+#include "util/u_inlines.h"
+#include "util/u_memory.h"
+#include "r600.h"
+#include "r600_pipe.h"
+#include "r600_resource.h"
+
+struct r600_upload {
+	struct r600_pipe_context	*rctx;
+	struct r600_bo			*buffer;
+	char				*ptr;
+	unsigned			size;
+	unsigned			default_size;
+	unsigned			total_alloc_size;
+	unsigned			offset;
+	unsigned			alignment;
+};
+
+struct r600_upload *r600_upload_create(struct r600_pipe_context *rctx,
+					unsigned default_size,
+					unsigned alignment)
+{
+	struct r600_upload *upload = CALLOC_STRUCT(r600_upload);
+
+	if (upload == NULL)
+		return NULL;
+
+	upload->rctx = rctx;
+	upload->size = 0;
+	upload->default_size = default_size;
+	upload->alignment = alignment;
+	upload->ptr = NULL;
+	upload->buffer = NULL;
+	upload->total_alloc_size = 0;
+
+	return upload;
+}
+
+void r600_upload_flush(struct r600_upload *upload)
+{
+	if (upload->buffer) {
+		r600_bo_reference(upload->rctx->radeon, &upload->buffer, NULL);
+	}
+	upload->default_size = MAX2(upload->total_alloc_size, upload->default_size);
+	upload->total_alloc_size = 0;
+	upload->size = 0;
+	upload->ptr = NULL;
+	upload->buffer = NULL;
+}
+
+void r600_upload_destroy(struct r600_upload *upload)
+{
+	r600_upload_flush(upload);
+	FREE(upload);
+}
+
+int r600_upload_buffer(struct r600_upload *upload, unsigned offset,
+			unsigned size, struct r600_resource_buffer *in_buffer,
+			unsigned *out_offset, unsigned *out_size,
+			struct r600_bo **out_buffer)
+{
+	unsigned alloc_size = align(size, upload->alignment);
+	const void *in_ptr = NULL;
+
+	if (upload->offset + alloc_size > upload->size) {
+		if (upload->size) {
+			r600_bo_reference(upload->rctx->radeon, &upload->buffer, NULL);
+		}
+		upload->size = align(MAX2(upload->default_size, alloc_size), 4096);
+		upload->total_alloc_size += upload->size;
+		upload->offset = 0;
+		upload->buffer = r600_bo(upload->rctx->radeon, upload->size, 4096, PIPE_BIND_VERTEX_BUFFER, 0);
+		if (upload->buffer == NULL) {
+			return -ENOMEM;
+		}
+		upload->ptr = r600_bo_map(upload->rctx->radeon, upload->buffer, 0, NULL);
+	}
+
+	in_ptr = in_buffer->user_buffer;
+	memcpy(upload->ptr + upload->offset, in_ptr + offset, size);
+	*out_offset = upload->offset;
+	*out_size = upload->size;
+	*out_buffer = upload->buffer;
+	upload->offset += alloc_size;
+
+	return 0;
+}