radeon: Optimize memory handling for dma operations.

We keep dma buffer objects in list untill they have been unused for many
draw operations. Current limit of having 100 flushes is just guess for
good performance/memory trade off.

Moving WARN_ONCE macro to common context because it is used in multiple drivers.

Signed-off-by: Pauli Nieminen <[email protected]>

16 files changed, 163 insertions, 93 deletions

diff --git a/src/mesa/drivers/dri/r200/r200_state.c b/src/mesa/drivers/dri/r200/r200_state.c
index 5a6fd20d8c2..ffc1a957454 100644
--- a/src/mesa/drivers/dri/r200/r200_state.c
+++ b/src/mesa/drivers/dri/r200/r200_state.c
@@ -2289,8 +2289,11 @@ static GLboolean r200ValidateBuffers(GLcontext *ctx)
 {
    r200ContextPtr rmesa = R200_CONTEXT(ctx);
    struct radeon_renderbuffer *rrb;
+   struct radeon_dma_bo *dma_bo;
    int i, ret;
 
+	if (RADEON_DEBUG & DEBUG_IOCTL)
+		fprintf(stderr, "%s\n", __FUNCTION__);
    radeon_cs_space_reset_bos(rmesa->radeon.cmdbuf.cs);
 
    rrb = radeon_get_colorbuffer(&rmesa->radeon);
@@ -2323,9 +2326,12 @@ static GLboolean r200ValidateBuffers(GLcontext *ctx)
 			   RADEON_GEM_DOMAIN_GTT | RADEON_GEM_DOMAIN_VRAM, 0);
    }
 
-   ret = radeon_cs_space_check_with_bo(rmesa->radeon.cmdbuf.cs, rmesa->radeon.dma.current, RADEON_GEM_DOMAIN_GTT, 0);
-   if (ret)
-       return GL_FALSE;
+   dma_bo = first_elem(&rmesa->radeon.dma.reserved);
+   {
+       ret = radeon_cs_space_check_with_bo(rmesa->radeon.cmdbuf.cs, dma_bo->bo, RADEON_GEM_DOMAIN_GTT, 0);
+       if (ret)
+	   return GL_FALSE;
+   }
    return GL_TRUE;
 }
 
diff --git a/src/mesa/drivers/dri/r200/r200_swtcl.c b/src/mesa/drivers/dri/r200/r200_swtcl.c
index 83e70b586d7..1b238911408 100644
--- a/src/mesa/drivers/dri/r200/r200_swtcl.c
+++ b/src/mesa/drivers/dri/r200/r200_swtcl.c
@@ -39,6 +39,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 #include "main/image.h"
 #include "main/imports.h"
 #include "main/macros.h"
+#include "main/simple_list.h"
 
 #include "swrast/s_context.h"
 #include "swrast/s_fog.h"
@@ -275,7 +276,7 @@ void r200_swtcl_flush(GLcontext *ctx, uint32_t current_offset)
    radeonEmitState(&rmesa->radeon);
    r200EmitVertexAOS( rmesa,
 		      rmesa->radeon.swtcl.vertex_size,
-		      rmesa->radeon.dma.current,
+		      first_elem(&rmesa->radeon.dma.reserved)->bo,
 		      current_offset);
 
 
diff --git a/src/mesa/drivers/dri/r300/r300_context.h b/src/mesa/drivers/dri/r300/r300_context.h
index 65e0d466113..339b3045586 100644
--- a/src/mesa/drivers/dri/r300/r300_context.h
+++ b/src/mesa/drivers/dri/r300/r300_context.h
@@ -51,22 +51,6 @@ typedef struct r300_context r300ContextRec;
 typedef struct r300_context *r300ContextPtr;
 
 
-/* From http://gcc. gnu.org/onlinedocs/gcc-3.2.3/gcc/Variadic-Macros.html .
-   I suppose we could inline this and use macro to fetch out __LINE__ and stuff in case we run into trouble
-   with other compilers ... GLUE!
-*/
-#define WARN_ONCE(a, ...)	{ \
-	static int warn##__LINE__=1; \
-	if(warn##__LINE__){ \
-		fprintf(stderr, "*********************************WARN_ONCE*********************************\n"); \
-		fprintf(stderr, "File %s function %s line %d\n", \
-			__FILE__, __FUNCTION__, __LINE__); \
-		fprintf(stderr,  a, ## __VA_ARGS__);\
-		fprintf(stderr, "***************************************************************************\n"); \
-		warn##__LINE__=0;\
-		} \
-	}
-
 #include "r300_vertprog.h"
 
 
diff --git a/src/mesa/drivers/dri/r300/r300_draw.c b/src/mesa/drivers/dri/r300/r300_draw.c
index d34f33b3289..dbf5384d558 100644
--- a/src/mesa/drivers/dri/r300/r300_draw.c
+++ b/src/mesa/drivers/dri/r300/r300_draw.c
@@ -31,6 +31,7 @@
 #include "main/state.h"
 #include "main/api_validate.h"
 #include "main/enums.h"
+#include "main/simple_list.h"
 
 #include "r300_reg.h"
 #include "r300_context.h"
@@ -510,7 +511,7 @@ static void r300SetVertexFormat(GLcontext *ctx, const struct gl_client_array *ar
 		}
 
 		r300->radeon.tcl.aos_count = vbuf->num_attribs;
-		ret = radeon_cs_space_check_with_bo(r300->radeon.cmdbuf.cs, r300->radeon.dma.current, RADEON_GEM_DOMAIN_GTT, 0);
+		ret = radeon_cs_space_check_with_bo(r300->radeon.cmdbuf.cs, first_elem(&r300->radeon.dma.reserved)->bo, RADEON_GEM_DOMAIN_GTT, 0);
 		if (ret)
 			r300SwitchFallback(ctx, R300_FALLBACK_INVALID_BUFFERS, GL_TRUE);
 	}
diff --git a/src/mesa/drivers/dri/r300/r300_swtcl.c b/src/mesa/drivers/dri/r300/r300_swtcl.c
index a634cb5192d..9d6f7568799 100644
--- a/src/mesa/drivers/dri/r300/r300_swtcl.c
+++ b/src/mesa/drivers/dri/r300/r300_swtcl.c
@@ -39,6 +39,7 @@ USE OR OTHER DEALINGS IN THE SOFTWARE.
 #include "r300_emit.h"
 #include "r300_tex.h"
 #include "r300_render.h"
+#include "main/simple_list.h"
 
 #define EMIT_ATTR( ATTR, STYLE )					\
 do {									\
@@ -617,7 +618,7 @@ void r300_swtcl_flush(GLcontext *ctx, uint32_t current_offset)
     r300_emit_scissor(ctx);
 	r300EmitVertexAOS(rmesa,
 			rmesa->radeon.swtcl.vertex_size,
-			rmesa->radeon.dma.current,
+			first_elem(&rmesa->radeon.dma.reserved)->bo,
 			current_offset);
 
 	r300EmitVbufPrim(rmesa,
diff --git a/src/mesa/drivers/dri/r300/r300_texstate.c b/src/mesa/drivers/dri/r300/r300_texstate.c
index 6f489ace7bf..f030451b282 100644
--- a/src/mesa/drivers/dri/r300/r300_texstate.c
+++ b/src/mesa/drivers/dri/r300/r300_texstate.c
@@ -43,6 +43,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 #include "main/teximage.h"
 #include "main/texobj.h"
 #include "main/enums.h"
+#include "main/simple_list.h"
 
 #include "r300_context.h"
 #include "r300_state.h"
@@ -323,7 +324,7 @@ GLboolean r300ValidateBuffers(GLcontext * ctx)
 							  RADEON_GEM_DOMAIN_GTT | RADEON_GEM_DOMAIN_VRAM, 0);
 	}
 
-	ret = radeon_cs_space_check_with_bo(rmesa->radeon.cmdbuf.cs, rmesa->radeon.dma.current, RADEON_GEM_DOMAIN_GTT, 0);
+	ret = radeon_cs_space_check_with_bo(rmesa->radeon.cmdbuf.cs, first_elem(&rmesa->radeon.dma.reserved)->bo, RADEON_GEM_DOMAIN_GTT, 0);
 	if (ret)
 		return GL_FALSE;
 	return GL_TRUE;
diff --git a/src/mesa/drivers/dri/r600/r600_context.h b/src/mesa/drivers/dri/r600/r600_context.h
index 30ddce682c1..4373254dd68 100644
--- a/src/mesa/drivers/dri/r600/r600_context.h
+++ b/src/mesa/drivers/dri/r600/r600_context.h
@@ -61,22 +61,6 @@ GLboolean r700SendSQConfig(context_t *context);
 
 #include "main/mm.h"
 
-/* From http://gcc. gnu.org/onlinedocs/gcc-3.2.3/gcc/Variadic-Macros.html .
-   I suppose we could inline this and use macro to fetch out __LINE__ and stuff in case we run into trouble
-   with other compilers ... GLUE!
-*/
-#define WARN_ONCE(a, ...)	{ \
-	static int warn##__LINE__=1; \
-	if(warn##__LINE__){ \
-		fprintf(stderr, "*********************************WARN_ONCE*********************************\n"); \
-		fprintf(stderr, "File %s function %s line %d\n", \
-			__FILE__, __FUNCTION__, __LINE__); \
-		fprintf(stderr,  a, ## __VA_ARGS__);\
-		fprintf(stderr, "***************************************************************************\n"); \
-		warn##__LINE__=0;\
-		} \
-	}
-
 /************ DMA BUFFERS **************/
 
 /* The blit width for texture uploads
diff --git a/src/mesa/drivers/dri/r600/r600_texstate.c b/src/mesa/drivers/dri/r600/r600_texstate.c
index ee9b64ee43a..1057d7d8bbf 100644
--- a/src/mesa/drivers/dri/r600/r600_texstate.c
+++ b/src/mesa/drivers/dri/r600/r600_texstate.c
@@ -43,6 +43,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 #include "main/teximage.h"
 #include "main/texobj.h"
 #include "main/enums.h"
+#include "main/simple_list.h"
 
 #include "r600_context.h"
 #include "r700_state.h"
@@ -685,7 +686,7 @@ GLboolean r600ValidateBuffers(GLcontext * ctx)
 							  RADEON_GEM_DOMAIN_GTT | RADEON_GEM_DOMAIN_VRAM, 0);
 	}
 
-	ret = radeon_cs_space_check_with_bo(rmesa->radeon.cmdbuf.cs, rmesa->radeon.dma.current, RADEON_GEM_DOMAIN_GTT, 0);
+	ret = radeon_cs_space_check_with_bo(rmesa->radeon.cmdbuf.cs, first_elem(&rmesa->radeon.dma.reserved)->bo, RADEON_GEM_DOMAIN_GTT, 0);
 	if (ret)
 		return GL_FALSE;
 	return GL_TRUE;
diff --git a/src/mesa/drivers/dri/r600/r700_ioctl.c b/src/mesa/drivers/dri/r600/r700_ioctl.c
index 23cc128d6db..e0e506d665d 100644
--- a/src/mesa/drivers/dri/r600/r700_ioctl.c
+++ b/src/mesa/drivers/dri/r600/r700_ioctl.c
@@ -31,6 +31,7 @@
 #include "main/imports.h"
 #include "main/macros.h"
 #include "main/context.h"
+#include "main/simple_list.h"
 #include "swrast/swrast.h"
 
 #include "radeon_common.h"
@@ -52,7 +53,7 @@ static void r700Flush(GLcontext *ctx)
 	   we have no DMA buffer allocated.
 	   then no point flushing anything at all.
 	*/
-	if (!radeon->dma.flush && !radeon->cmdbuf.cs->cdw && !radeon->dma.current)
+	if (!radeon->dma.flush && !radeon->cmdbuf.cs->cdw && is_empty_list(&radeon->dma.reserved))
 		return;
 
 	if (radeon->dma.flush)
diff --git a/src/mesa/drivers/dri/radeon/radeon_common.c b/src/mesa/drivers/dri/radeon/radeon_common.c
index 0e9a1ae5d81..b5b4fed8fa8 100644
--- a/src/mesa/drivers/dri/radeon/radeon_common.c
+++ b/src/mesa/drivers/dri/radeon/radeon_common.c
@@ -1043,7 +1043,7 @@ void radeonFlush(GLcontext *ctx)
 	   we have no DMA buffer allocated.
 	   then no point flushing anything at all.
 	*/
-	if (!radeon->dma.flush && !radeon->cmdbuf.cs->cdw && !radeon->dma.current)
+	if (!radeon->dma.flush && !radeon->cmdbuf.cs->cdw && is_empty_list(&radeon->dma.reserved))
 		return;
 
 	if (radeon->dma.flush)
@@ -1152,7 +1152,7 @@ int rcommonFlushCmdBuf(radeonContextPtr rmesa, const char *caller)
 {
 	int ret;
 
-	radeonReleaseDmaRegion(rmesa);
+	radeonReleaseDmaRegions(rmesa);
 
 	LOCK_HARDWARE(rmesa);
 	ret = rcommonFlushCmdBufLocked(rmesa, caller);
diff --git a/src/mesa/drivers/dri/radeon/radeon_common_context.c b/src/mesa/drivers/dri/radeon/radeon_common_context.c
index 35622099bf0..13711963e2a 100644
--- a/src/mesa/drivers/dri/radeon/radeon_common_context.c
+++ b/src/mesa/drivers/dri/radeon/radeon_common_context.c
@@ -264,6 +264,7 @@ GLboolean radeonInitContext(radeonContextPtr radeon,
 	}
 
 	make_empty_list(&radeon->query.not_flushed_head);
+	radeon_init_dma(radeon);
 
 	return GL_TRUE;
 }
@@ -309,10 +310,11 @@ void radeonDestroyContext(__DRIcontextPrivate *driContextPriv )
 
 	assert(radeon);
 	if (radeon) {
-		if (radeon->dma.current) {
+		if (!is_empty_list(&radeon->dma.reserved)) {
 			rcommonFlushCmdBuf( radeon, __FUNCTION__ );
 		}
 
+		radeonFreeDmaRegions(radeon);
 		radeonReleaseArrays(radeon->glCtx, ~0);
 		meta_destroy_metaops(&radeon->meta);
 		if (radeon->vtbl.free_context)
diff --git a/src/mesa/drivers/dri/radeon/radeon_common_context.h b/src/mesa/drivers/dri/radeon/radeon_common_context.h
index a9480cd2e4c..39fab66ca76 100644
--- a/src/mesa/drivers/dri/radeon/radeon_common_context.h
+++ b/src/mesa/drivers/dri/radeon/radeon_common_context.h
@@ -18,6 +18,22 @@ struct radeon_context;
 
 #include "radeon_bocs_wrapper.h"
 
+/* From http://gcc. gnu.org/onlinedocs/gcc-3.2.3/gcc/Variadic-Macros.html .
+   I suppose we could inline this and use macro to fetch out __LINE__ and stuff in case we run into trouble
+   with other compilers ... GLUE!
+*/
+#define WARN_ONCE(a, ...)	{ \
+	static int warn##__LINE__=1; \
+	if(warn##__LINE__){ \
+		fprintf(stderr, "*********************************WARN_ONCE*********************************\n"); \
+		fprintf(stderr, "File %s function %s line %d\n", \
+			__FILE__, __FUNCTION__, __LINE__); \
+		fprintf(stderr,  a, ## __VA_ARGS__);\
+		fprintf(stderr, "***************************************************************************\n"); \
+		warn##__LINE__=0;\
+		} \
+	}
+
 /* This union is used to avoid warnings/miscompilation
    with float to uint32_t casts due to strict-aliasing */
 typedef union { GLfloat f; uint32_t ui32; } float_ui32_type;
@@ -281,12 +297,22 @@ struct radeon_aos {
 	int count; /** Number of vertices */
 };
 
+#define DMA_BO_FREE_TIME 100
+
+struct radeon_dma_bo {
+  struct radeon_dma_bo *next, *prev;
+  struct radeon_bo *bo;
+  int expire_counter;
+};
+
 struct radeon_dma {
         /* Active dma region.  Allocations for vertices and retained
          * regions come from here.  Also used for emitting random vertices,
          * these may be flushed by calling flush_current();
          */
-        struct radeon_bo *current; /** Buffer that DMA memory is allocated from */
+	struct radeon_dma_bo free;
+	struct radeon_dma_bo wait;
+	struct radeon_dma_bo reserved;
         int current_used; /** Number of bytes allocated and forgotten about */
         int current_vertexptr; /** End of active vertex region */
 
@@ -296,12 +322,6 @@ struct radeon_dma {
          * performed.
          */
         void (*flush) (GLcontext *);
-
-        /* Number of "in-flight" DMA buffers, i.e. the number of buffers
-         * for which a DISCARD command is currently queued in the command buffer
-.
-         */
-        GLuint nr_released_bufs;
 };
 
 /* radeon_swtcl.c
diff --git a/src/mesa/drivers/dri/radeon/radeon_dma.c b/src/mesa/drivers/dri/radeon/radeon_dma.c
index 5e755c51edb..93bcae2454b 100644
--- a/src/mesa/drivers/dri/radeon/radeon_dma.c
+++ b/src/mesa/drivers/dri/radeon/radeon_dma.c
@@ -31,6 +31,7 @@ USE OR OTHER DEALINGS IN THE SOFTWARE.
 **************************************************************************/
 
 #include "radeon_common.h"
+#include "main/simple_list.h"
 
 #if defined(USE_X86_ASM)
 #define COPY_DWORDS( dst, src, nr )					\
@@ -161,9 +162,15 @@ void rcommon_emit_vector(GLcontext * ctx, struct radeon_aos *aos,
 	}
 }
 
-void radeonRefillCurrentDmaRegion(radeonContextPtr rmesa, int size)
+void radeon_init_dma(radeonContextPtr rmesa)
 {
+   make_empty_list(&rmesa->dma.free);
+   make_empty_list(&rmesa->dma.wait);
+   make_empty_list(&rmesa->dma.reserved);
+}
 
+void radeonRefillCurrentDmaRegion(radeonContextPtr rmesa, int size)
+{
 	size = MAX2(size, MAX_DMA_BUF_SZ);
 
 	if (RADEON_DEBUG & (DEBUG_IOCTL | DEBUG_DMA))
@@ -173,43 +180,41 @@ void radeonRefillCurrentDmaRegion(radeonContextPtr rmesa, int size)
 		rmesa->dma.flush(rmesa->glCtx);
 	}
 
-	if (rmesa->dma.nr_released_bufs > 4) {
-		rcommonFlushCmdBuf(rmesa, __FUNCTION__);
-		rmesa->dma.nr_released_bufs = 0;
-	}
-
-	if (rmesa->dma.current) {
-		radeon_bo_unmap(rmesa->dma.current);
-		radeon_bo_unref(rmesa->dma.current);
-		rmesa->dma.current = 0;
-	}
+	if (is_empty_list(&rmesa->dma.free)) {
+		struct radeon_dma_bo *dma_bo = CALLOC(sizeof(struct radeon_dma_bo));
+		assert(dma_bo);
 
 again_alloc:
-	rmesa->dma.current = radeon_bo_open(rmesa->radeonScreen->bom,
+		dma_bo->bo = radeon_bo_open(rmesa->radeonScreen->bom,
 					    0, size, 4, RADEON_GEM_DOMAIN_GTT,
 					    0);
 
-	if (!rmesa->dma.current) {
-		rcommonFlushCmdBuf(rmesa, __FUNCTION__);
-		rmesa->dma.nr_released_bufs = 0;
-		goto again_alloc;
+		if (!dma_bo->bo) {
+			rcommonFlushCmdBuf(rmesa, __FUNCTION__);
+			goto again_alloc;
+		}
+		insert_at_head(&rmesa->dma.reserved, dma_bo);
+	} else {
+		struct radeon_dma_bo *dma_bo = last_elem(&rmesa->dma.free);
+		assert(dma_bo->bo->cref == 1);
+		remove_from_list(dma_bo);
+		insert_at_head(&rmesa->dma.reserved, dma_bo);
 	}
 
 	rmesa->dma.current_used = 0;
 	rmesa->dma.current_vertexptr = 0;
 	
 	if (radeon_cs_space_check_with_bo(rmesa->cmdbuf.cs,
-					  rmesa->dma.current,
+					  first_elem(&rmesa->dma.reserved)->bo,
 					  RADEON_GEM_DOMAIN_GTT, 0))
 		fprintf(stderr,"failure to revalidate BOs - badness\n");
 
-	if (!rmesa->dma.current) {
+	if (is_empty_list(&rmesa->dma.reserved)) {
         /* Cmd buff have been flushed in radeon_revalidate_bos */
-		rmesa->dma.nr_released_bufs = 0;
 		goto again_alloc;
 	}
 
-	radeon_bo_map(rmesa->dma.current, 1);
+	radeon_bo_map(first_elem(&rmesa->dma.reserved)->bo, 1);
 }
 
 /* Allocates a region from rmesa->dma.current.  If there isn't enough
@@ -230,30 +235,87 @@ void radeonAllocDmaRegion(radeonContextPtr rmesa,
 	alignment--;
 	rmesa->dma.current_used = (rmesa->dma.current_used + alignment) & ~alignment;
 
-	if (!rmesa->dma.current || rmesa->dma.current_used + bytes > rmesa->dma.current->size)
+	if (is_empty_list(&rmesa->dma.reserved)
+		|| rmesa->dma.current_used + bytes > first_elem(&rmesa->dma.reserved)->bo->size)
 		radeonRefillCurrentDmaRegion(rmesa, (bytes + 15) & ~15);
 
 	*poffset = rmesa->dma.current_used;
-	*pbo = rmesa->dma.current;
+	*pbo = first_elem(&rmesa->dma.reserved)->bo;
 	radeon_bo_ref(*pbo);
 
 	/* Always align to at least 16 bytes */
 	rmesa->dma.current_used = (rmesa->dma.current_used + bytes + 15) & ~15;
 	rmesa->dma.current_vertexptr = rmesa->dma.current_used;
 
-	assert(rmesa->dma.current_used <= rmesa->dma.current->size);
+	assert(rmesa->dma.current_used <= first_elem(&rmesa->dma.reserved)->bo->size);
+}
+
+void radeonFreeDmaRegions(radeonContextPtr rmesa)
+{
+	struct radeon_dma_bo *dma_bo;
+	struct radeon_dma_bo *temp;
+	if (RADEON_DEBUG & DEBUG_IOCTL)
+		fprintf(stderr, "%s\n", __FUNCTION__);
+
+	foreach_s(dma_bo, temp, &rmesa->dma.free) {
+		remove_from_list(dma_bo);
+	        radeon_bo_unref(dma_bo->bo);
+		FREE(dma_bo);
+	}
+
+	foreach_s(dma_bo, temp, &rmesa->dma.reserved) {
+		remove_from_list(dma_bo);
+		radeon_bo_unmap(dma_bo->bo);
+	        radeon_bo_unref(dma_bo->bo);
+		FREE(dma_bo);
+	}
 }
 
-void radeonReleaseDmaRegion(radeonContextPtr rmesa)
+void radeonReleaseDmaRegions(radeonContextPtr rmesa)
 {
+	struct radeon_dma_bo *dma_bo;
+	struct radeon_dma_bo *temp;
+	const int expire_at = ++rmesa->dma.free.expire_counter + DMA_BO_FREE_TIME;
+	const int time = rmesa->dma.free.expire_counter;
 	if (RADEON_DEBUG & DEBUG_IOCTL)
-		fprintf(stderr, "%s %p\n", __FUNCTION__, rmesa->dma.current);
-	if (rmesa->dma.current) {
-		rmesa->dma.nr_released_bufs++;
-		radeon_bo_unmap(rmesa->dma.current);
-	        radeon_bo_unref(rmesa->dma.current);
+		fprintf(stderr, "%s\n", __FUNCTION__);
+
+	/* move waiting bos to free list.
+	   wait list provides gpu time to handle data before reuse */
+	foreach_s(dma_bo, temp, &rmesa->dma.wait) {
+		if (dma_bo->expire_counter == time) {
+			WARN_ONCE("Leaking dma buffer object!\n");
+			radeon_bo_unref(dma_bo->bo);
+			remove_from_list(dma_bo);
+			FREE(dma_bo);
+			continue;
+		}
+		if (dma_bo->bo->cref > 1)
+			continue;
+		remove_from_list(dma_bo);
+		dma_bo->expire_counter = expire_at;
+		insert_at_tail(&rmesa->dma.free, dma_bo);
+	}
+
+	/* unmap the last dma region */
+	if (!is_empty_list(&rmesa->dma.reserved))
+		radeon_bo_unmap(first_elem(&rmesa->dma.reserved)->bo);
+	/* move reserved to wait list */
+	foreach_s(dma_bo, temp, &rmesa->dma.reserved) {
+		remove_from_list(dma_bo);
+		dma_bo->expire_counter = expire_at;
+		insert_at_tail(&rmesa->dma.wait, dma_bo);
+	}
+
+	/* free bos that have been unused for some time */
+	foreach_s(dma_bo, temp, &rmesa->dma.free) {
+		if (dma_bo->expire_counter != time)
+			break;
+		remove_from_list(dma_bo);
+	        radeon_bo_unref(dma_bo->bo);
+		FREE(dma_bo);
 	}
-	rmesa->dma.current = NULL;
+
 }
 
 
@@ -266,10 +328,10 @@ void rcommon_flush_last_swtcl_prim( GLcontext *ctx  )
 		
 
 	if (RADEON_DEBUG & DEBUG_IOCTL)
-		fprintf(stderr, "%s %p\n", __FUNCTION__, dma->current);
+		fprintf(stderr, "%s\n", __FUNCTION__);
 	dma->flush = NULL;
 
-	if (dma->current) {
+	if (!is_empty_list(&dma->reserved)) {
 	    GLuint current_offset = dma->current_used;
 
 	    assert (dma->current_used +
@@ -292,7 +354,10 @@ rcommonAllocDmaLowVerts( radeonContextPtr rmesa, int nverts, int vsize )
 	GLuint bytes = vsize * nverts;
 	void *head;
 restart:
-	if (!rmesa->dma.current || rmesa->dma.current_vertexptr + bytes > rmesa->dma.current->size) {
+	if (RADEON_DEBUG & DEBUG_IOCTL)
+		fprintf(stderr, "%s\n", __FUNCTION__);
+	if (is_empty_list(&rmesa->dma.reserved)
+		|| rmesa->dma.current_vertexptr + bytes > first_elem(&rmesa->dma.reserved)->bo->size) {
                 radeonRefillCurrentDmaRegion(rmesa, bytes);
 	}
 
@@ -302,7 +367,7 @@ restart:
 			      rmesa->hw.max_state_size + (20*sizeof(int)),
 			      __FUNCTION__);
 		/* if cmdbuf flushed DMA restart */
-		if (!rmesa->dma.current)
+		if (is_empty_list(&rmesa->dma.reserved))
 			goto restart;
                 rmesa->glCtx->Driver.NeedFlush |= FLUSH_STORED_VERTICES;
                 rmesa->dma.flush = rcommon_flush_last_swtcl_prim;
@@ -314,7 +379,7 @@ restart:
                 rmesa->swtcl.numverts * rmesa->swtcl.vertex_size * 4 ==
                 rmesa->dma.current_vertexptr );
 
-	head = (rmesa->dma.current->ptr + rmesa->dma.current_vertexptr);
+	head = (first_elem(&rmesa->dma.reserved)->bo->ptr + rmesa->dma.current_vertexptr);
 	rmesa->dma.current_vertexptr += bytes;
 	rmesa->swtcl.numverts += nverts;
 	return head;
@@ -324,18 +389,17 @@ void radeonReleaseArrays( GLcontext *ctx, GLuint newinputs )
 {
    radeonContextPtr radeon = RADEON_CONTEXT( ctx );
    int i;
+	if (RADEON_DEBUG & DEBUG_IOCTL)
+		fprintf(stderr, "%s\n", __FUNCTION__);
 
    if (radeon->dma.flush) {
        radeon->dma.flush(radeon->glCtx);
    }
-   if (radeon->tcl.elt_dma_bo) {
-	   radeon_bo_unref(radeon->tcl.elt_dma_bo);
-	   radeon->tcl.elt_dma_bo = NULL;
-   }
    for (i = 0; i < radeon->tcl.aos_count; i++) {
       if (radeon->tcl.aos[i].bo) {
          radeon_bo_unref(radeon->tcl.aos[i].bo);
          radeon->tcl.aos[i].bo = NULL;
+
       }
    }
 }
diff --git a/src/mesa/drivers/dri/radeon/radeon_dma.h b/src/mesa/drivers/dri/radeon/radeon_dma.h
index 55509ed00c1..21f956e3211 100644
--- a/src/mesa/drivers/dri/radeon/radeon_dma.h
+++ b/src/mesa/drivers/dri/radeon/radeon_dma.h
@@ -42,13 +42,15 @@ void rcommon_emit_vector(GLcontext * ctx, struct radeon_aos *aos,
 			 const GLvoid * data, int size, int stride, int count);
 
 void radeonRefillCurrentDmaRegion(radeonContextPtr rmesa, int size);
+void radeon_init_dma(radeonContextPtr rmesa);
 void radeonAllocDmaRegion(radeonContextPtr rmesa,
 			  struct radeon_bo **pbo, int *poffset,
 			  int bytes, int alignment);
-void radeonReleaseDmaRegion(radeonContextPtr rmesa);
+void radeonReleaseDmaRegions(radeonContextPtr rmesa);
 
 void rcommon_flush_last_swtcl_prim(GLcontext *ctx);
 
 void *rcommonAllocDmaLowVerts(radeonContextPtr rmesa, int nverts, int vsize);
+void radeonFreeDmaRegions(radeonContextPtr rmesa);
 void radeonReleaseArrays( GLcontext *ctx, GLuint newinputs );
 #endif
diff --git a/src/mesa/drivers/dri/radeon/radeon_state.c b/src/mesa/drivers/dri/radeon/radeon_state.c
index 0d1728b747a..56f82bdb0b6 100644
--- a/src/mesa/drivers/dri/radeon/radeon_state.c
+++ b/src/mesa/drivers/dri/radeon/radeon_state.c
@@ -40,6 +40,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 #include "main/state.h"
 #include "main/context.h"
 #include "main/framebuffer.h"
+#include "main/simple_list.h"
 
 #include "vbo/vbo.h"
 #include "tnl/tnl.h"
@@ -2099,7 +2100,7 @@ static GLboolean r100ValidateBuffers(GLcontext *ctx)
 			   RADEON_GEM_DOMAIN_GTT | RADEON_GEM_DOMAIN_VRAM, 0);
    }
 
-   ret = radeon_cs_space_check_with_bo(rmesa->radeon.cmdbuf.cs, rmesa->radeon.dma.current, RADEON_GEM_DOMAIN_GTT, 0);
+   ret = radeon_cs_space_check_with_bo(rmesa->radeon.cmdbuf.cs, first_elem(&rmesa->radeon.dma.reserved)->bo, RADEON_GEM_DOMAIN_GTT, 0);
    if (ret)
        return GL_FALSE;
    return GL_TRUE;
diff --git a/src/mesa/drivers/dri/radeon/radeon_swtcl.c b/src/mesa/drivers/dri/radeon/radeon_swtcl.c
index e31f045991c..58b3be9391b 100644
--- a/src/mesa/drivers/dri/radeon/radeon_swtcl.c
+++ b/src/mesa/drivers/dri/radeon/radeon_swtcl.c
@@ -38,6 +38,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 #include "main/enums.h"
 #include "main/imports.h"
 #include "main/macros.h"
+#include "main/simple_list.h"
 
 #include "swrast_setup/swrast_setup.h"
 #include "math/m_translate.h"
@@ -291,7 +292,7 @@ void r100_swtcl_flush(GLcontext *ctx, uint32_t current_offset)
    radeonEmitState(&rmesa->radeon);
    radeonEmitVertexAOS( rmesa,
 			rmesa->radeon.swtcl.vertex_size,
-			rmesa->radeon.dma.current,
+			first_elem(&rmesa->radeon.dma.reserved)->bo,
 			current_offset);