nvc0: buffer suballocation with a primitive slab allocator

1 files changed, 167 insertions, 36 deletions

diff --git a/src/gallium/drivers/nvc0/nvc0_buffer.c b/src/gallium/drivers/nvc0/nvc0_buffer.c
index 873016f0d5f..93d7f5d3033 100644
--- a/src/gallium/drivers/nvc0/nvc0_buffer.c
+++ b/src/gallium/drivers/nvc0/nvc0_buffer.c
@@ -11,46 +11,116 @@
 #include "nvc0_context.h"
 #include "nvc0_resource.h"
 
+#define NVC0_BUFFER_STATUS_USER_MEMORY 0xff
+
+static INLINE boolean
+nvc0_buffer_allocate(struct nvc0_screen *screen, struct nvc0_resource *buf,
+                     unsigned domain)
+{
+   if (domain == NOUVEAU_BO_VRAM) {
+      buf->mm = nvc0_mm_allocate(screen->mm_VRAM, buf->base.width0, &buf->bo,
+                                 &buf->offset);
+      if (!buf->bo)
+         return nvc0_buffer_allocate(screen, buf, NOUVEAU_BO_GART);
+   } else
+   if (domain == NOUVEAU_BO_GART) {
+      buf->mm = nvc0_mm_allocate(screen->mm_GART, buf->base.width0, &buf->bo,
+                                 &buf->offset);
+      if (!buf->bo)
+         return FALSE;
+   } else {
+      assert(!domain);
+      if (!buf->data)
+         buf->data = MALLOC(buf->base.width0);
+      if (!buf->data)
+         return FALSE;
+   }
+   buf->domain = domain;
+   return TRUE;
+}
+
+static INLINE void
+release_allocation(struct nvc0_mm_allocation **mm, struct nvc0_fence *fence)
+{
+   (*mm)->next = fence->buffers;
+   fence->buffers = (*mm);
+   (*mm) = NULL;
+}
+
 static void
 nvc0_buffer_destroy(struct pipe_screen *pscreen,
                     struct pipe_resource *presource)
 {
+   struct nvc0_screen *screen = nvc0_screen(pscreen);
    struct nvc0_resource *res = nvc0_resource(presource);
 
-   if (res->bo)
-      nouveau_screen_bo_release(pscreen, res->bo);
+   nouveau_bo_ref(NULL, &res->bo);
 
-   if (res->data)
+   if (res->mm)
+      release_allocation(&res->mm, screen->fence.current);
+
+   if (res->status != NVC0_BUFFER_STATUS_USER_MEMORY && res->data)
       FREE(res->data);
 
    FREE(res);
 }
 
+static INLINE uint32_t
+nouveau_buffer_rw_flags(unsigned pipe)
+{
+   uint32_t flags = 0;
+
+   if (pipe & PIPE_TRANSFER_READ)
+      flags = NOUVEAU_BO_RD;
+   if (pipe & PIPE_TRANSFER_WRITE)
+      flags |= NOUVEAU_BO_WR;
+
+   return flags;
+}
+
 static void *
 nvc0_buffer_transfer_map(struct pipe_context *pipe,
                          struct pipe_transfer *transfer)
 {
    struct nvc0_resource *res = nvc0_resource(transfer->resource);
+   struct nvc0_fence *fence;
    uint8_t *map;
-   uint32_t flags;
+   int ret;
+   uint32_t flags = nouveau_buffer_rw_flags(transfer->usage);
 
-   if (res->base.bind & PIPE_BIND_VERTEX_BUFFER)
+   if ((res->base.bind & (PIPE_BIND_VERTEX_BUFFER | PIPE_BIND_INDEX_BUFFER)) &&
+       (flags & NOUVEAU_BO_WR))
       nvc0_context(pipe)->vbo_dirty = TRUE;
 
-// #ifdef NOUVEAU_USERPSACE_MM
-   if (res->base.bind & PIPE_BIND_CONSTANT_BUFFER)
+   if (res->domain == 0)
       return res->data + transfer->box.x;
-// #endif
-   flags = nouveau_screen_transfer_flags(transfer->usage);
 
-   map = nouveau_screen_bo_map_range(pipe->screen,
-                                     res->bo,
-                                     transfer->box.x, transfer->box.width,
-                                     flags);
-   if (!map)
+   if (res->domain == NOUVEAU_BO_VRAM) {
+      NOUVEAU_ERR("transfers to/from VRAM buffers are not allowed\n");
+      /* if this happens, migrate back to GART */
       return NULL;
+   }
+
+   if (res->score > -1024)
+      --res->score;
+
+   ret = nouveau_bo_map(res->bo, flags | NOUVEAU_BO_NOSYNC);
+   if (ret)
+      return NULL;
+   map = res->bo->map;
+   nouveau_bo_unmap(res->bo);
+
+   fence = (flags == NOUVEAU_BO_RD) ? res->fence_wr : res->fence;
 
-   return map + transfer->box.x;
+   if (fence) {
+      if (nvc0_fence_wait(fence) == FALSE)
+         NOUVEAU_ERR("failed to fence buffer\n");
+
+      nvc0_fence_reference(&res->fence, NULL);
+      nvc0_fence_reference(&res->fence_wr, NULL);
+   }
+
+   return map + transfer->box.x + res->offset;
 }
 
 
@@ -62,13 +132,12 @@ nvc0_buffer_transfer_flush_region(struct pipe_context *pipe,
 {
    struct nvc0_resource *res = nvc0_resource(transfer->resource);
 
-#ifdef NOUVEAU_USERPSACE_MM
    if (!res->bo)
       return;
-#endif
+
    nouveau_screen_bo_map_flush_range(pipe->screen,
                                      res->bo,
-                                     transfer->box.x + box->x,
+                                     res->offset + transfer->box.x + box->x,
                                      box->width);
 }
 
@@ -78,11 +147,10 @@ nvc0_buffer_transfer_unmap(struct pipe_context *pipe,
 {
    struct nvc0_resource *res = nvc0_resource(transfer->resource);
 
-// #ifdef NOUVEAU_USERPSACE_MM
    if (res->data)
       return;
-// #endif
-   nouveau_screen_bo_unmap(pipe->screen, res->bo);
+
+   /* nouveau_screen_bo_unmap(pipe->screen, res->bo); */
 }
 
 const struct u_resource_vtbl nvc0_buffer_vtbl =
@@ -102,7 +170,9 @@ struct pipe_resource *
 nvc0_buffer_create(struct pipe_screen *pscreen,
                    const struct pipe_resource *templ)
 {
+   struct nvc0_screen *screen = nvc0_screen(pscreen);
    struct nvc0_resource *buffer;
+   boolean ret;
 
    buffer = CALLOC_STRUCT(nvc0_resource);
    if (!buffer)
@@ -114,14 +184,11 @@ nvc0_buffer_create(struct pipe_screen *pscreen,
    buffer->base.screen = pscreen;
 
    if (buffer->base.bind & PIPE_BIND_CONSTANT_BUFFER)
-      buffer->data = MALLOC(buffer->base.width0);
-
-   buffer->bo = nouveau_screen_bo_new(pscreen,
-                                      16,
-                                      buffer->base.usage,
-                                      buffer->base.bind,
-                                      buffer->base.width0);
-   if (buffer->bo == NULL)
+      ret = nvc0_buffer_allocate(screen, buffer, 0);
+   else
+      ret = nvc0_buffer_allocate(screen, buffer, NOUVEAU_BO_GART);
+
+   if (ret == FALSE)
       goto fail;
 
    return &buffer->base;
@@ -154,13 +221,77 @@ nvc0_user_buffer_create(struct pipe_screen *pscreen,
    buffer->base.height0 = 1;
    buffer->base.depth0 = 1;
 
-   buffer->bo = nouveau_screen_bo_user(pscreen, ptr, bytes);
-   if (!buffer->bo)
-      goto fail;
-	
+   buffer->data = ptr;
+   buffer->status = NVC0_BUFFER_STATUS_USER_MEMORY;
+
    return &buffer->base;
+}
 
-fail:
-   FREE(buffer);
-   return NULL;
+/* Migrate a linear buffer (vertex, index, constants) USER -> GART -> VRAM. */
+boolean
+nvc0_buffer_migrate(struct nvc0_context *nvc0,
+                    struct nvc0_resource *buf, unsigned domain)
+{
+   struct nvc0_screen *screen = nvc0_screen(buf->base.screen);
+   struct nouveau_bo *bo;
+   unsigned size = buf->base.width0;
+   int ret;
+
+   if (domain == NOUVEAU_BO_GART && buf->domain == 0) {
+      if (!nvc0_buffer_allocate(screen, buf, domain))
+         return FALSE;
+      ret = nouveau_bo_map(buf->bo, NOUVEAU_BO_WR | NOUVEAU_BO_NOSYNC);
+      if (ret)
+         return ret;
+      memcpy((uint8_t *)buf->bo->map + buf->offset, buf->data, size);
+      nouveau_bo_unmap(buf->bo);
+   } else
+   if (domain == NOUVEAU_BO_VRAM && buf->domain == NOUVEAU_BO_GART) {
+      struct nvc0_mm_allocation *mm = buf->mm;
+
+      bo = buf->bo;
+      buf->bo = NULL;
+      buf->mm = NULL;
+      nvc0_buffer_allocate(screen, buf, domain);
+
+      nvc0_m2mf_copy_linear(nvc0, buf->bo, 0, NOUVEAU_BO_VRAM,
+                            bo, 0, NOUVEAU_BO_GART, buf->base.width0);
+
+      release_allocation(&mm, screen->fence.current);
+      nouveau_bo_ref(NULL, &bo);
+   } else
+   if (domain == NOUVEAU_BO_VRAM && buf->domain == 0) {
+      /* should use a scratch buffer instead here */
+      if (!nvc0_buffer_migrate(nvc0, buf, NOUVEAU_BO_GART))
+         return FALSE;
+      return nvc0_buffer_migrate(nvc0, buf, NOUVEAU_BO_VRAM);
+   } else
+      return -1;
+
+   buf->domain = domain;
+
+   return TRUE;
+}
+
+/* Migrate data from glVertexAttribPointer(non-VBO) user buffers to GART.
+ * MUST NOT FLUSH THE PUSH BUFFER, we could be in the middle of a method.
+ */
+boolean
+nvc0_migrate_vertices(struct nvc0_resource *buf, unsigned base, unsigned size)
+{
+   struct nvc0_screen *screen = nvc0_screen(buf->base.screen);
+   int ret;
+
+   assert(buf->data && !buf->domain);
+
+   if (!nvc0_buffer_allocate(screen, buf, NOUVEAU_BO_GART))
+      return FALSE;
+   ret = nouveau_bo_map_range(buf->bo, base + buf->offset, size,
+                              NOUVEAU_BO_WR | NOUVEAU_BO_NOSYNC);
+   if (ret)
+      return FALSE;
+   memcpy(buf->bo->map, buf->data + base, size);
+   nouveau_bo_unmap(buf->bo);
+
+   return TRUE;
 }