Merge remote branch 'origin/nvc0'

1 files changed, 465 insertions, 0 deletions

diff --git a/src/gallium/drivers/nvc0/nvc0_buffer.c b/src/gallium/drivers/nvc0/nvc0_buffer.c
new file mode 100644
index 00000000000..8021e4348b2
--- /dev/null
+++ b/src/gallium/drivers/nvc0/nvc0_buffer.c
@@ -0,0 +1,465 @@
+
+#include "util/u_inlines.h"
+#include "util/u_memory.h"
+#include "util/u_math.h"
+
+#define NOUVEAU_NVC0
+#include "nouveau/nouveau_screen.h"
+#include "nouveau/nouveau_winsys.h"
+#undef NOUVEAU_NVC0
+
+#include "nvc0_context.h"
+#include "nvc0_resource.h"
+
+struct nvc0_transfer {
+   struct pipe_transfer base;
+};
+
+static INLINE struct nvc0_transfer *
+nvc0_transfer(struct pipe_transfer *transfer)
+{
+   return (struct nvc0_transfer *)transfer;
+}
+
+static INLINE boolean
+nvc0_buffer_allocate(struct nvc0_screen *screen, struct nvc0_resource *buf,
+                     unsigned domain)
+{
+   if (domain == NOUVEAU_BO_VRAM) {
+      buf->mm = nvc0_mm_allocate(screen->mm_VRAM, buf->base.width0, &buf->bo,
+                                 &buf->offset);
+      if (!buf->bo)
+         return nvc0_buffer_allocate(screen, buf, NOUVEAU_BO_GART);
+   } else
+   if (domain == NOUVEAU_BO_GART) {
+      buf->mm = nvc0_mm_allocate(screen->mm_GART, buf->base.width0, &buf->bo,
+                                 &buf->offset);
+      if (!buf->bo)
+         return FALSE;
+   }
+   if (domain != NOUVEAU_BO_GART) {
+      if (!buf->data) {
+         buf->data = MALLOC(buf->base.width0);
+         if (!buf->data)
+            return FALSE;
+      }
+   }
+   buf->domain = domain;
+   return TRUE;
+}
+
+static INLINE void
+release_allocation(struct nvc0_mm_allocation **mm, struct nvc0_fence *fence)
+{
+   (*mm)->next = fence->buffers;
+   fence->buffers = (*mm);
+   (*mm) = NULL;
+}
+
+static void
+nvc0_buffer_destroy(struct pipe_screen *pscreen,
+                    struct pipe_resource *presource)
+{
+   struct nvc0_screen *screen = nvc0_screen(pscreen);
+   struct nvc0_resource *res = nvc0_resource(presource);
+
+   nouveau_bo_ref(NULL, &res->bo);
+
+   if (res->mm)
+      release_allocation(&res->mm, screen->fence.current);
+
+   if (res->data && !(res->status & NVC0_BUFFER_STATUS_USER_MEMORY))
+      FREE(res->data);
+
+   FREE(res);
+}
+
+/* Maybe just migrate to GART right away if we actually need to do this. */
+boolean
+nvc0_buffer_download(struct nvc0_context *nvc0, struct nvc0_resource *buf,
+                     unsigned start, unsigned size)
+{
+   struct nvc0_mm_allocation *mm;
+   struct nouveau_bo *bounce = NULL;
+   uint32_t offset;
+
+   assert(buf->domain == NOUVEAU_BO_VRAM);
+
+   mm = nvc0_mm_allocate(nvc0->screen->mm_GART, size, &bounce, &offset);
+   if (!bounce)
+      return FALSE;
+
+   nvc0_m2mf_copy_linear(nvc0, bounce, offset, NOUVEAU_BO_GART,
+                         buf->bo, buf->offset + start, NOUVEAU_BO_VRAM,
+                         size);
+
+   if (nouveau_bo_map_range(bounce, offset, size, NOUVEAU_BO_RD))
+      return FALSE;
+   memcpy(buf->data + start, bounce->map, size);
+   nouveau_bo_unmap(bounce);
+
+   buf->status &= ~NVC0_BUFFER_STATUS_DIRTY;
+
+   nouveau_bo_ref(NULL, &bounce);
+   if (mm)
+      nvc0_mm_free(mm);
+   return TRUE;
+}
+
+static boolean
+nvc0_buffer_upload(struct nvc0_context *nvc0, struct nvc0_resource *buf,
+                   unsigned start, unsigned size)
+{
+   struct nvc0_mm_allocation *mm;
+   struct nouveau_bo *bounce = NULL;
+   uint32_t offset;
+
+   mm = nvc0_mm_allocate(nvc0->screen->mm_GART, size, &bounce, &offset);
+   if (!bounce)
+      return FALSE;
+
+   nouveau_bo_map_range(bounce, offset, size,
+                        NOUVEAU_BO_WR | NOUVEAU_BO_NOSYNC);
+   memcpy(bounce->map, buf->data + start, size);
+   nouveau_bo_unmap(bounce);
+
+   nvc0_m2mf_copy_linear(nvc0, buf->bo, buf->offset + start, NOUVEAU_BO_VRAM,
+                         bounce, offset, NOUVEAU_BO_GART, size);
+
+   nouveau_bo_ref(NULL, &bounce);
+   if (mm)
+      release_allocation(&mm, nvc0->screen->fence.current);
+
+   if (start == 0 && size == buf->base.width0)
+      buf->status &= ~NVC0_BUFFER_STATUS_DIRTY;
+   return TRUE;
+}
+
+static struct pipe_transfer *
+nvc0_buffer_transfer_get(struct pipe_context *pipe,
+                         struct pipe_resource *resource,
+                         unsigned level,
+                         unsigned usage,
+                         const struct pipe_box *box)
+{
+   struct nvc0_resource *buf = nvc0_resource(resource);
+   struct nvc0_transfer *xfr = CALLOC_STRUCT(nvc0_transfer);
+   if (!xfr)
+      return NULL;
+
+   xfr->base.resource = resource;
+   xfr->base.box.x = box->x;
+   xfr->base.box.width = box->width;
+   xfr->base.usage = usage;
+
+   if (buf->domain == NOUVEAU_BO_VRAM) {
+      if (usage & PIPE_TRANSFER_READ) {
+         if (buf->status & NVC0_BUFFER_STATUS_DIRTY)
+            nvc0_buffer_download(nvc0_context(pipe), buf, 0, buf->base.width0);
+      }
+   }
+
+   return &xfr->base;
+}
+
+static void
+nvc0_buffer_transfer_destroy(struct pipe_context *pipe,
+                             struct pipe_transfer *transfer)
+{
+   struct nvc0_resource *buf = nvc0_resource(transfer->resource);
+   struct nvc0_transfer *xfr = nvc0_transfer(transfer);
+
+   if (xfr->base.usage & PIPE_TRANSFER_WRITE) {
+      /* writing is worse */
+      nvc0_buffer_adjust_score(nvc0_context(pipe), buf, -5000);
+
+      if (buf->domain == NOUVEAU_BO_VRAM) {
+         nvc0_buffer_upload(nvc0_context(pipe), buf,
+                            transfer->box.x, transfer->box.width);
+      }
+
+      if (buf->domain != 0 && (buf->base.bind & (PIPE_BIND_VERTEX_BUFFER |
+                                                 PIPE_BIND_INDEX_BUFFER)))
+         nvc0_context(pipe)->vbo_dirty = TRUE;
+   }
+
+   FREE(xfr);
+}
+
+static INLINE boolean
+nvc0_buffer_sync(struct nvc0_resource *buf, unsigned rw)
+{
+   if (rw == PIPE_TRANSFER_READ) {
+      if (!buf->fence_wr)
+         return TRUE;
+      if (!nvc0_fence_wait(buf->fence_wr))
+         return FALSE;
+   } else {
+      if (!buf->fence)
+         return TRUE;
+      if (!nvc0_fence_wait(buf->fence))
+         return FALSE;
+
+      nvc0_fence_reference(&buf->fence, NULL);
+   }
+   nvc0_fence_reference(&buf->fence_wr, NULL);
+
+   return TRUE;
+}
+
+static INLINE boolean
+nvc0_buffer_busy(struct nvc0_resource *buf, unsigned rw)
+{
+   if (rw == PIPE_TRANSFER_READ)
+      return (buf->fence_wr && !nvc0_fence_signalled(buf->fence_wr));
+   else
+      return (buf->fence && !nvc0_fence_signalled(buf->fence));
+}
+
+static void *
+nvc0_buffer_transfer_map(struct pipe_context *pipe,
+                         struct pipe_transfer *transfer)
+{
+   struct nvc0_transfer *xfr = nvc0_transfer(transfer);
+   struct nvc0_resource *buf = nvc0_resource(transfer->resource);
+   struct nouveau_bo *bo = buf->bo;
+   uint8_t *map;
+   int ret;
+   uint32_t offset = xfr->base.box.x;
+   uint32_t flags;
+
+   nvc0_buffer_adjust_score(nvc0_context(pipe), buf, -250);
+
+   if (buf->domain != NOUVEAU_BO_GART)
+      return buf->data + offset;
+
+   if (buf->mm)
+      flags = NOUVEAU_BO_NOSYNC | NOUVEAU_BO_RDWR;
+   else
+      flags = nouveau_screen_transfer_flags(xfr->base.usage);
+
+   offset += buf->offset;
+
+   ret = nouveau_bo_map_range(buf->bo, offset, xfr->base.box.width, flags);
+   if (ret)
+      return NULL;
+   map = bo->map;
+
+   /* Unmap right now. Since multiple buffers can share a single nouveau_bo,
+    * not doing so might make future maps fail or trigger "reloc while mapped"
+    * errors. For now, mappings to userspace are guaranteed to be persistent.
+    */
+   nouveau_bo_unmap(bo);
+
+   if (buf->mm) {
+      if (xfr->base.usage & PIPE_TRANSFER_DONTBLOCK) {
+         if (nvc0_buffer_busy(buf, xfr->base.usage & PIPE_TRANSFER_READ_WRITE))
+            return NULL;
+      } else
+      if (!(xfr->base.usage & PIPE_TRANSFER_UNSYNCHRONIZED)) {
+         nvc0_buffer_sync(buf, xfr->base.usage & PIPE_TRANSFER_READ_WRITE);
+      }
+   }
+   return map;
+}
+
+
+
+static void
+nvc0_buffer_transfer_flush_region(struct pipe_context *pipe,
+                                  struct pipe_transfer *transfer,
+                                  const struct pipe_box *box)
+{
+   struct nvc0_resource *res = nvc0_resource(transfer->resource);
+   struct nouveau_bo *bo = res->bo;
+   unsigned offset = res->offset + transfer->box.x + box->x;
+
+   /* not using non-snoop system memory yet, no need for cflush */
+   if (1)
+      return;
+
+   /* XXX: maybe need to upload for VRAM buffers here */
+
+   nouveau_screen_bo_map_flush_range(pipe->screen, bo, offset, box->width);
+}
+
+static void
+nvc0_buffer_transfer_unmap(struct pipe_context *pipe,
+                           struct pipe_transfer *transfer)
+{
+   /* we've called nouveau_bo_unmap right after map */
+}
+
+const struct u_resource_vtbl nvc0_buffer_vtbl =
+{
+   u_default_resource_get_handle,     /* get_handle */
+   nvc0_buffer_destroy,               /* resource_destroy */
+   NULL,                              /* is_resource_referenced */
+   nvc0_buffer_transfer_get,          /* get_transfer */
+   nvc0_buffer_transfer_destroy,      /* transfer_destroy */
+   nvc0_buffer_transfer_map,          /* transfer_map */
+   nvc0_buffer_transfer_flush_region, /* transfer_flush_region */
+   nvc0_buffer_transfer_unmap,        /* transfer_unmap */
+   u_default_transfer_inline_write    /* transfer_inline_write */
+};
+
+struct pipe_resource *
+nvc0_buffer_create(struct pipe_screen *pscreen,
+                   const struct pipe_resource *templ)
+{
+   struct nvc0_screen *screen = nvc0_screen(pscreen);
+   struct nvc0_resource *buffer;
+   boolean ret;
+
+   buffer = CALLOC_STRUCT(nvc0_resource);
+   if (!buffer)
+      return NULL;
+
+   buffer->base = *templ;
+   buffer->vtbl = &nvc0_buffer_vtbl;
+   pipe_reference_init(&buffer->base.reference, 1);
+   buffer->base.screen = pscreen;
+
+   if (buffer->base.bind & PIPE_BIND_CONSTANT_BUFFER)
+      ret = nvc0_buffer_allocate(screen, buffer, 0);
+   else
+      ret = nvc0_buffer_allocate(screen, buffer, NOUVEAU_BO_GART);
+
+   if (ret == FALSE)
+      goto fail;
+
+   return &buffer->base;
+
+fail:
+   FREE(buffer);
+   return NULL;
+}
+
+
+struct pipe_resource *
+nvc0_user_buffer_create(struct pipe_screen *pscreen,
+                        void *ptr,
+                        unsigned bytes,
+                        unsigned bind)
+{
+   struct nvc0_resource *buffer;
+
+   buffer = CALLOC_STRUCT(nvc0_resource);
+   if (!buffer)
+      return NULL;
+
+   pipe_reference_init(&buffer->base.reference, 1);
+   buffer->vtbl = &nvc0_buffer_vtbl;
+   buffer->base.screen = pscreen;
+   buffer->base.format = PIPE_FORMAT_R8_UNORM;
+   buffer->base.usage = PIPE_USAGE_IMMUTABLE;
+   buffer->base.bind = bind;
+   buffer->base.width0 = bytes;
+   buffer->base.height0 = 1;
+   buffer->base.depth0 = 1;
+
+   buffer->data = ptr;
+   buffer->status = NVC0_BUFFER_STATUS_USER_MEMORY;
+
+   return &buffer->base;
+}
+
+static INLINE boolean
+nvc0_buffer_fetch_data(struct nvc0_resource *buf,
+                       struct nouveau_bo *bo, unsigned offset, unsigned size)
+{
+   if (!buf->data) {
+      buf->data = MALLOC(size);
+      if (!buf->data)
+         return FALSE;
+   }
+   if (nouveau_bo_map_range(bo, offset, size, NOUVEAU_BO_RD))
+      return FALSE;
+   memcpy(buf->data, bo->map, size);
+   nouveau_bo_unmap(bo);
+
+   return TRUE;
+}
+
+/* Migrate a linear buffer (vertex, index, constants) USER -> GART -> VRAM. */
+boolean
+nvc0_buffer_migrate(struct nvc0_context *nvc0,
+                    struct nvc0_resource *buf, unsigned domain)
+{
+   struct nvc0_screen *screen = nvc0_screen(buf->base.screen);
+   struct nouveau_bo *bo;
+   unsigned size = buf->base.width0;
+   unsigned offset;
+   int ret;
+
+   assert(domain != buf->domain);
+
+   if (domain == NOUVEAU_BO_GART && buf->domain == 0) {
+      if (!nvc0_buffer_allocate(screen, buf, domain))
+         return FALSE;
+      ret = nouveau_bo_map_range(buf->bo, buf->offset, size, NOUVEAU_BO_WR |
+                                 NOUVEAU_BO_NOSYNC);
+      if (ret)
+         return ret;
+      memcpy(buf->bo->map, buf->data, size);
+      nouveau_bo_unmap(buf->bo);
+      FREE(buf->data);
+   } else
+   if (domain != 0 && buf->domain != 0) {
+      struct nvc0_mm_allocation *mm = buf->mm;
+
+      if (domain == NOUVEAU_BO_VRAM) {
+         /* keep a system memory copy of our data in case we hit a fallback */
+         if (!nvc0_buffer_fetch_data(buf, buf->bo, buf->offset, size))
+            return FALSE;
+         debug_printf("migrating %u KiB to VRAM\n", size / 1024);
+      }
+
+      offset = buf->offset;
+      bo = buf->bo;
+      buf->bo = NULL;
+      buf->mm = NULL;
+      nvc0_buffer_allocate(screen, buf, domain);
+
+      nvc0_m2mf_copy_linear(nvc0, buf->bo, buf->offset, domain,
+                            bo, offset, buf->domain, buf->base.width0);
+
+      nouveau_bo_ref(NULL, &bo);
+      if (mm)
+         release_allocation(&mm, screen->fence.current);
+   } else
+   if (domain == NOUVEAU_BO_VRAM && buf->domain == 0) {
+      if (!nvc0_buffer_allocate(screen, buf, NOUVEAU_BO_VRAM))
+         return FALSE;
+      if (!nvc0_buffer_upload(nvc0, buf, 0, buf->base.width0))
+         return FALSE;
+   } else
+      return FALSE;
+
+   buf->domain = domain;
+
+   return TRUE;
+}
+
+/* Migrate data from glVertexAttribPointer(non-VBO) user buffers to GART.
+ * MUST NOT FLUSH THE PUSH BUFFER, we could be in the middle of a method.
+ */
+boolean
+nvc0_migrate_vertices(struct nvc0_resource *buf, unsigned base, unsigned size)
+{
+   struct nvc0_screen *screen = nvc0_screen(buf->base.screen);
+   int ret;
+
+   assert(buf->data && !buf->domain);
+
+   if (!nvc0_buffer_allocate(screen, buf, NOUVEAU_BO_GART))
+      return FALSE;
+   ret = nouveau_bo_map_range(buf->bo, base + buf->offset, size,
+                              NOUVEAU_BO_WR | NOUVEAU_BO_NOSYNC);
+   if (ret)
+      return FALSE;
+   memcpy(buf->bo->map, buf->data + base, size);
+   nouveau_bo_unmap(buf->bo);
+
+   return TRUE;
+}