diff options
Diffstat (limited to 'src/gallium/drivers/nouveau/nouveau_buffer.c')
| -rw-r--r-- | src/gallium/drivers/nouveau/nouveau_buffer.c | 488 |
1 files changed, 488 insertions, 0 deletions
diff --git a/src/gallium/drivers/nouveau/nouveau_buffer.c b/src/gallium/drivers/nouveau/nouveau_buffer.c new file mode 100644 index 00000000000..d0cc29104b8 --- /dev/null +++ b/src/gallium/drivers/nouveau/nouveau_buffer.c @@ -0,0 +1,488 @@ + +#include "util/u_inlines.h" +#include "util/u_memory.h" +#include "util/u_math.h" + +#include "nouveau_screen.h" +#include "nouveau_context.h" +#include "nouveau_winsys.h" +#include "nouveau_fence.h" +#include "nouveau_buffer.h" +#include "nouveau_mm.h" + +struct nouveau_transfer { + struct pipe_transfer base; +}; + +static INLINE struct nouveau_transfer * +nouveau_transfer(struct pipe_transfer *transfer) +{ + return (struct nouveau_transfer *)transfer; +} + +static INLINE boolean +nouveau_buffer_allocate(struct nouveau_screen *screen, + struct nv04_resource *buf, unsigned domain) +{ + if (domain == NOUVEAU_BO_VRAM) { + buf->mm = nouveau_mm_allocate(screen->mm_VRAM, buf->base.width0, + &buf->bo, &buf->offset); + if (!buf->bo) + return nouveau_buffer_allocate(screen, buf, NOUVEAU_BO_GART); + } else + if (domain == NOUVEAU_BO_GART) { + buf->mm = nouveau_mm_allocate(screen->mm_GART, buf->base.width0, + &buf->bo, &buf->offset); + if (!buf->bo) + return FALSE; + } + if (domain != NOUVEAU_BO_GART) { + if (!buf->data) { + buf->data = MALLOC(buf->base.width0); + if (!buf->data) + return FALSE; + } + } + buf->domain = domain; + return TRUE; +} + +static INLINE void +release_allocation(struct nouveau_mm_allocation **mm, + struct nouveau_fence *fence) +{ + nouveau_fence_work(fence, nouveau_mm_free_work, *mm); + (*mm) = NULL; +} + +INLINE void +nouveau_buffer_release_gpu_storage(struct nv04_resource *buf) +{ + nouveau_bo_ref(NULL, &buf->bo); + + if (buf->mm) + release_allocation(&buf->mm, buf->fence); + + buf->domain = 0; +} + +static INLINE boolean +nouveau_buffer_reallocate(struct nouveau_screen *screen, + struct nv04_resource *buf, unsigned domain) +{ + nouveau_buffer_release_gpu_storage(buf); + + return nouveau_buffer_allocate(screen, buf, domain); +} + +static void +nouveau_buffer_destroy(struct pipe_screen *pscreen, + struct pipe_resource *presource) +{ + struct nv04_resource *res = nv04_resource(presource); + + nouveau_buffer_release_gpu_storage(res); + + if (res->data && !(res->status & NOUVEAU_BUFFER_STATUS_USER_MEMORY)) + FREE(res->data); + + FREE(res); +} + +/* Maybe just migrate to GART right away if we actually need to do this. */ +boolean +nouveau_buffer_download(struct nouveau_context *nv, struct nv04_resource *buf, + unsigned start, unsigned size) +{ + struct nouveau_mm_allocation *mm; + struct nouveau_bo *bounce = NULL; + uint32_t offset; + + assert(buf->domain == NOUVEAU_BO_VRAM); + + mm = nouveau_mm_allocate(nv->screen->mm_GART, size, &bounce, &offset); + if (!bounce) + return FALSE; + + nv->copy_data(nv, bounce, offset, NOUVEAU_BO_GART, + buf->bo, buf->offset + start, NOUVEAU_BO_VRAM, size); + + if (nouveau_bo_map_range(bounce, offset, size, NOUVEAU_BO_RD)) + return FALSE; + memcpy(buf->data + start, bounce->map, size); + nouveau_bo_unmap(bounce); + + buf->status &= ~NOUVEAU_BUFFER_STATUS_GPU_WRITING; + + nouveau_bo_ref(NULL, &bounce); + if (mm) + nouveau_mm_free(mm); + return TRUE; +} + +static boolean +nouveau_buffer_upload(struct nouveau_context *nv, struct nv04_resource *buf, + unsigned start, unsigned size) +{ + struct nouveau_mm_allocation *mm; + struct nouveau_bo *bounce = NULL; + uint32_t offset; + + if (size <= 192) { + nv->push_data(nv, buf->bo, buf->offset + start, buf->domain, + size, buf->data + start); + return TRUE; + } + + mm = nouveau_mm_allocate(nv->screen->mm_GART, size, &bounce, &offset); + if (!bounce) + return FALSE; + + nouveau_bo_map_range(bounce, offset, size, + NOUVEAU_BO_WR | NOUVEAU_BO_NOSYNC); + memcpy(bounce->map, buf->data + start, size); + nouveau_bo_unmap(bounce); + + nv->copy_data(nv, buf->bo, buf->offset + start, NOUVEAU_BO_VRAM, + bounce, offset, NOUVEAU_BO_GART, size); + + nouveau_bo_ref(NULL, &bounce); + if (mm) + release_allocation(&mm, nv->screen->fence.current); + + if (start == 0 && size == buf->base.width0) + buf->status &= ~NOUVEAU_BUFFER_STATUS_GPU_WRITING; + return TRUE; +} + +static struct pipe_transfer * +nouveau_buffer_transfer_get(struct pipe_context *pipe, + struct pipe_resource *resource, + unsigned level, unsigned usage, + const struct pipe_box *box) +{ + struct nv04_resource *buf = nv04_resource(resource); + struct nouveau_context *nv = nouveau_context(pipe); + struct nouveau_transfer *xfr = CALLOC_STRUCT(nouveau_transfer); + if (!xfr) + return NULL; + + xfr->base.resource = resource; + xfr->base.box.x = box->x; + xfr->base.box.width = box->width; + xfr->base.usage = usage; + + if (buf->domain == NOUVEAU_BO_VRAM) { + if (usage & PIPE_TRANSFER_READ) { + if (buf->status & NOUVEAU_BUFFER_STATUS_GPU_WRITING) + nouveau_buffer_download(nv, buf, 0, buf->base.width0); + } + } + + return &xfr->base; +} + +static void +nouveau_buffer_transfer_destroy(struct pipe_context *pipe, + struct pipe_transfer *transfer) +{ + struct nv04_resource *buf = nv04_resource(transfer->resource); + struct nouveau_transfer *xfr = nouveau_transfer(transfer); + struct nouveau_context *nv = nouveau_context(pipe); + + if (xfr->base.usage & PIPE_TRANSFER_WRITE) { + /* writing is worse */ + nouveau_buffer_adjust_score(nv, buf, -5000); + + if (buf->domain == NOUVEAU_BO_VRAM) { + nouveau_buffer_upload(nv, buf, transfer->box.x, transfer->box.width); + } + + if (buf->domain != 0 && (buf->base.bind & (PIPE_BIND_VERTEX_BUFFER | + PIPE_BIND_INDEX_BUFFER))) + nouveau_context(pipe)->vbo_dirty = TRUE; + } + + FREE(xfr); +} + +static INLINE boolean +nouveau_buffer_sync(struct nv04_resource *buf, unsigned rw) +{ + if (rw == PIPE_TRANSFER_READ) { + if (!buf->fence_wr) + return TRUE; + if (!nouveau_fence_wait(buf->fence_wr)) + return FALSE; + } else { + if (!buf->fence) + return TRUE; + if (!nouveau_fence_wait(buf->fence)) + return FALSE; + + nouveau_fence_ref(NULL, &buf->fence); + } + nouveau_fence_ref(NULL, &buf->fence_wr); + + return TRUE; +} + +static INLINE boolean +nouveau_buffer_busy(struct nv04_resource *buf, unsigned rw) +{ + if (rw == PIPE_TRANSFER_READ) + return (buf->fence_wr && !nouveau_fence_signalled(buf->fence_wr)); + else + return (buf->fence && !nouveau_fence_signalled(buf->fence)); +} + +static void * +nouveau_buffer_transfer_map(struct pipe_context *pipe, + struct pipe_transfer *transfer) +{ + struct nouveau_transfer *xfr = nouveau_transfer(transfer); + struct nv04_resource *buf = nv04_resource(transfer->resource); + struct nouveau_bo *bo = buf->bo; + uint8_t *map; + int ret; + uint32_t offset = xfr->base.box.x; + uint32_t flags; + + nouveau_buffer_adjust_score(nouveau_context(pipe), buf, -250); + + if (buf->domain != NOUVEAU_BO_GART) + return buf->data + offset; + + if (buf->mm) + flags = NOUVEAU_BO_NOSYNC | NOUVEAU_BO_RDWR; + else + flags = nouveau_screen_transfer_flags(xfr->base.usage); + + offset += buf->offset; + + ret = nouveau_bo_map_range(buf->bo, offset, xfr->base.box.width, flags); + if (ret) + return NULL; + map = bo->map; + + /* Unmap right now. Since multiple buffers can share a single nouveau_bo, + * not doing so might make future maps fail or trigger "reloc while mapped" + * errors. For now, mappings to userspace are guaranteed to be persistent. + */ + nouveau_bo_unmap(bo); + + if (buf->mm) { + if (xfr->base.usage & PIPE_TRANSFER_DONTBLOCK) { + if (nouveau_buffer_busy(buf, xfr->base.usage & PIPE_TRANSFER_READ_WRITE)) + return NULL; + } else + if (!(xfr->base.usage & PIPE_TRANSFER_UNSYNCHRONIZED)) { + nouveau_buffer_sync(buf, xfr->base.usage & PIPE_TRANSFER_READ_WRITE); + } + } + return map; +} + + + +static void +nouveau_buffer_transfer_flush_region(struct pipe_context *pipe, + struct pipe_transfer *transfer, + const struct pipe_box *box) +{ + struct nv04_resource *res = nv04_resource(transfer->resource); + struct nouveau_bo *bo = res->bo; + unsigned offset = res->offset + transfer->box.x + box->x; + + /* not using non-snoop system memory yet, no need for cflush */ + if (1) + return; + + /* XXX: maybe need to upload for VRAM buffers here */ + + nouveau_screen_bo_map_flush_range(pipe->screen, bo, offset, box->width); +} + +static void +nouveau_buffer_transfer_unmap(struct pipe_context *pipe, + struct pipe_transfer *transfer) +{ + /* we've called nouveau_bo_unmap right after map */ +} + +const struct u_resource_vtbl nouveau_buffer_vtbl = +{ + u_default_resource_get_handle, /* get_handle */ + nouveau_buffer_destroy, /* resource_destroy */ + NULL, /* is_resource_referenced */ + nouveau_buffer_transfer_get, /* get_transfer */ + nouveau_buffer_transfer_destroy, /* transfer_destroy */ + nouveau_buffer_transfer_map, /* transfer_map */ + nouveau_buffer_transfer_flush_region, /* transfer_flush_region */ + nouveau_buffer_transfer_unmap, /* transfer_unmap */ + u_default_transfer_inline_write /* transfer_inline_write */ +}; + +struct pipe_resource * +nouveau_buffer_create(struct pipe_screen *pscreen, + const struct pipe_resource *templ) +{ + struct nouveau_screen *screen = nouveau_screen(pscreen); + struct nv04_resource *buffer; + boolean ret; + + buffer = CALLOC_STRUCT(nv04_resource); + if (!buffer) + return NULL; + + buffer->base = *templ; + buffer->vtbl = &nouveau_buffer_vtbl; + pipe_reference_init(&buffer->base.reference, 1); + buffer->base.screen = pscreen; + + if ((buffer->base.bind & screen->sysmem_bindings) == screen->sysmem_bindings) + ret = nouveau_buffer_allocate(screen, buffer, 0); + else + ret = nouveau_buffer_allocate(screen, buffer, NOUVEAU_BO_GART); + + if (ret == FALSE) + goto fail; + + return &buffer->base; + +fail: + FREE(buffer); + return NULL; +} + + +struct pipe_resource * +nouveau_user_buffer_create(struct pipe_screen *pscreen, void *ptr, + unsigned bytes, unsigned bind) +{ + struct nv04_resource *buffer; + + buffer = CALLOC_STRUCT(nv04_resource); + if (!buffer) + return NULL; + + pipe_reference_init(&buffer->base.reference, 1); + buffer->vtbl = &nouveau_buffer_vtbl; + buffer->base.screen = pscreen; + buffer->base.format = PIPE_FORMAT_R8_UNORM; + buffer->base.usage = PIPE_USAGE_IMMUTABLE; + buffer->base.bind = bind; + buffer->base.width0 = bytes; + buffer->base.height0 = 1; + buffer->base.depth0 = 1; + + buffer->data = ptr; + buffer->status = NOUVEAU_BUFFER_STATUS_USER_MEMORY; + + return &buffer->base; +} + +/* Like download, but for GART buffers. Merge ? */ +static INLINE boolean +nouveau_buffer_data_fetch(struct nv04_resource *buf, struct nouveau_bo *bo, + unsigned offset, unsigned size) +{ + if (!buf->data) { + buf->data = MALLOC(size); + if (!buf->data) + return FALSE; + } + if (nouveau_bo_map_range(bo, offset, size, NOUVEAU_BO_RD)) + return FALSE; + memcpy(buf->data, bo->map, size); + nouveau_bo_unmap(bo); + + return TRUE; +} + +/* Migrate a linear buffer (vertex, index, constants) USER -> GART -> VRAM. */ +boolean +nouveau_buffer_migrate(struct nouveau_context *nv, + struct nv04_resource *buf, const unsigned new_domain) +{ + struct nouveau_screen *screen = nv->screen; + struct nouveau_bo *bo; + const unsigned old_domain = buf->domain; + unsigned size = buf->base.width0; + unsigned offset; + int ret; + + assert(new_domain != old_domain); + + if (new_domain == NOUVEAU_BO_GART && old_domain == 0) { + if (!nouveau_buffer_allocate(screen, buf, new_domain)) + return FALSE; + ret = nouveau_bo_map_range(buf->bo, buf->offset, size, NOUVEAU_BO_WR | + NOUVEAU_BO_NOSYNC); + if (ret) + return ret; + memcpy(buf->bo->map, buf->data, size); + nouveau_bo_unmap(buf->bo); + FREE(buf->data); + } else + if (old_domain != 0 && new_domain != 0) { + struct nouveau_mm_allocation *mm = buf->mm; + + if (new_domain == NOUVEAU_BO_VRAM) { + /* keep a system memory copy of our data in case we hit a fallback */ + if (!nouveau_buffer_data_fetch(buf, buf->bo, buf->offset, size)) + return FALSE; + debug_printf("migrating %u KiB to VRAM\n", size / 1024); + } + + offset = buf->offset; + bo = buf->bo; + buf->bo = NULL; + buf->mm = NULL; + nouveau_buffer_allocate(screen, buf, new_domain); + + nv->copy_data(nv, buf->bo, buf->offset, new_domain, + bo, offset, old_domain, buf->base.width0); + + nouveau_bo_ref(NULL, &bo); + if (mm) + release_allocation(&mm, screen->fence.current); + } else + if (new_domain == NOUVEAU_BO_VRAM && old_domain == 0) { + if (!nouveau_buffer_allocate(screen, buf, NOUVEAU_BO_VRAM)) + return FALSE; + if (!nouveau_buffer_upload(nv, buf, 0, buf->base.width0)) + return FALSE; + } else + return FALSE; + + assert(buf->domain == new_domain); + return TRUE; +} + +/* Migrate data from glVertexAttribPointer(non-VBO) user buffers to GART. + * We'd like to only allocate @size bytes here, but then we'd have to rebase + * the vertex indices ... + */ +boolean +nouveau_user_buffer_upload(struct nv04_resource *buf, + unsigned base, unsigned size) +{ + struct nouveau_screen *screen = nouveau_screen(buf->base.screen); + int ret; + + assert(buf->status & NOUVEAU_BUFFER_STATUS_USER_MEMORY); + + buf->base.width0 = base + size; + if (!nouveau_buffer_reallocate(screen, buf, NOUVEAU_BO_GART)) + return FALSE; + + ret = nouveau_bo_map_range(buf->bo, buf->offset + base, size, + NOUVEAU_BO_WR | NOUVEAU_BO_NOSYNC); + if (ret) + return FALSE; + memcpy(buf->bo->map, buf->data + base, size); + nouveau_bo_unmap(buf->bo); + + return TRUE; +} |