r600g: add support for virtual address space on cayman v11

Virtual address space put the userspace in charge of their GPU
address space. It's up to userspace to bind bo into the virtual
address space. Command stream can them be executed using the
IB_VM chunck.

This patch add support for this configuration. It doesn't remove
the 64K ib size limit thought this limit can be extanded up to
1M for IB_VM chunk.

v2: fix rendering
v3: fix rendering when using index buffer
v4: make vm conditional on kernel support add basic va management
v5: catch the case when we already have va for a bo
v6: agd5f: update on top of ioctl changes
v7: agd5f: further ioctl updates
v8: indentation cleanup + fix non cayman
v9: rebase against lastest mesa + improvement from Marek & Michel
v10: fix cut/paste bug
v11: don't rely on updated radeon_drm.h

Signed-off-by: Jerome Glisse <[email protected]>
Signed-off-by: Alex Deucher <[email protected]>

6 files changed, 273 insertions, 10 deletions

diff --git a/src/gallium/winsys/radeon/drm/radeon_drm_bo.c b/src/gallium/winsys/radeon/drm/radeon_drm_bo.c
index d4746ffc535..de2906faef9 100644
--- a/src/gallium/winsys/radeon/drm/radeon_drm_bo.c
+++ b/src/gallium/winsys/radeon/drm/radeon_drm_bo.c
@@ -30,6 +30,7 @@
 #include "util/u_hash_table.h"
 #include "util/u_memory.h"
 #include "util/u_simple_list.h"
+#include "util/u_double_list.h"
 #include "os/os_thread.h"
 #include "os/os_mman.h"
 
@@ -39,6 +40,11 @@
 #include <xf86drm.h>
 #include <errno.h>
 
+/*
+ * this are copy from radeon_drm, once an updated libdrm is released
+ * we should bump configure.ac requirement for it and remove the following
+ * field
+ */
 #define RADEON_BO_FLAGS_MACRO_TILE  1
 #define RADEON_BO_FLAGS_MICRO_TILE  2
 #define RADEON_BO_FLAGS_MICRO_TILE_SQUARE 0x20
@@ -57,6 +63,33 @@ struct drm_radeon_gem_wait {
 
 #endif
 
+#ifndef RADEON_VA_MAP
+
+#define RADEON_VA_MAP               1
+#define RADEON_VA_UNMAP             2
+
+#define RADEON_VA_RESULT_OK         0
+#define RADEON_VA_RESULT_ERROR      1
+#define RADEON_VA_RESULT_VA_EXIST   2
+
+#define RADEON_VM_PAGE_VALID        (1 << 0)
+#define RADEON_VM_PAGE_READABLE     (1 << 1)
+#define RADEON_VM_PAGE_WRITEABLE    (1 << 2)
+#define RADEON_VM_PAGE_SYSTEM       (1 << 3)
+#define RADEON_VM_PAGE_SNOOPED      (1 << 4)
+
+struct drm_radeon_gem_va {
+    uint32_t    handle;
+    uint32_t    operation;
+    uint32_t    vm_id;
+    uint32_t    flags;
+    uint64_t    offset;
+};
+
+#define DRM_RADEON_GEM_VA   0x2b
+#endif
+
+
 
 extern const struct pb_vtbl radeon_bo_vtbl;
 
@@ -67,6 +100,12 @@ static INLINE struct radeon_bo *radeon_bo(struct pb_buffer *bo)
     return (struct radeon_bo *)bo;
 }
 
+struct radeon_bo_va_hole {
+    struct list_head list;
+    uint64_t         offset;
+    uint64_t         size;
+};
+
 struct radeon_bomgr {
     /* Base class. */
     struct pb_manager base;
@@ -77,6 +116,12 @@ struct radeon_bomgr {
     /* List of buffer handles and its mutex. */
     struct util_hash_table *bo_handles;
     pipe_mutex bo_handles_mutex;
+    pipe_mutex bo_va_mutex;
+
+    /* is virtual address supported */
+    bool va;
+    unsigned va_offset;
+    struct list_head va_holes;
 };
 
 static INLINE struct radeon_bomgr *radeon_bomgr(struct pb_manager *mgr)
@@ -151,9 +196,94 @@ static boolean radeon_bo_is_busy(struct pb_buffer *_buf,
     }
 }
 
+static uint64_t radeon_bomgr_find_va(struct radeon_bomgr *mgr, uint64_t size)
+{
+    struct radeon_bo_va_hole *hole, *n;
+    uint64_t offset = 0;
+
+    pipe_mutex_lock(mgr->bo_va_mutex);
+    /* first look for a hole */
+    LIST_FOR_EACH_ENTRY_SAFE(hole, n, &mgr->va_holes, list) {
+        if (hole->size == size) {
+            offset = hole->offset;
+            list_del(&hole->list);
+            FREE(hole);
+            pipe_mutex_unlock(mgr->bo_va_mutex);
+            return offset;
+        }
+        if (hole->size > size) {
+            offset = hole->offset;
+            hole->size -= size;
+            hole->offset += size;
+            pipe_mutex_unlock(mgr->bo_va_mutex);
+            return offset;
+        }
+    }
+
+    offset = mgr->va_offset;
+    mgr->va_offset += size;
+    pipe_mutex_unlock(mgr->bo_va_mutex);
+    return offset;
+}
+
+static void radeon_bomgr_force_va(struct radeon_bomgr *mgr, uint64_t va, uint64_t size)
+{
+    pipe_mutex_lock(mgr->bo_va_mutex);
+    if (va >= mgr->va_offset) {
+        if (va > mgr->va_offset) {
+            struct radeon_bo_va_hole *hole;
+            hole = CALLOC_STRUCT(radeon_bo_va_hole);
+            if (hole) {
+                hole->size = va - mgr->va_offset;
+                hole->offset = mgr->va_offset;
+                list_add(&hole->list, &mgr->va_holes);
+            }
+        }
+        mgr->va_offset = va + size;
+    } else {
+        struct radeon_bo_va_hole *hole, *n;
+        uint64_t stmp, etmp;
+
+        /* free all holes that fall into the range
+         * NOTE that we might lose virtual address space
+         */
+        LIST_FOR_EACH_ENTRY_SAFE(hole, n, &mgr->va_holes, list) {
+            stmp = hole->offset;
+            etmp = stmp + hole->size;
+            if (va >= stmp && va < etmp) {
+                list_del(&hole->list);
+                FREE(hole);
+            }
+        }
+    }
+    pipe_mutex_unlock(mgr->bo_va_mutex);
+}
+
+static void radeon_bomgr_free_va(struct radeon_bomgr *mgr, uint64_t va, uint64_t size)
+{
+    pipe_mutex_lock(mgr->bo_va_mutex);
+    if ((va + size) == mgr->va_offset) {
+        mgr->va_offset = va;
+    } else {
+        struct radeon_bo_va_hole *hole;
+
+        /* FIXME on allocation failure we just lose virtual address space
+         * maybe print a warning
+         */
+        hole = CALLOC_STRUCT(radeon_bo_va_hole);
+        if (hole) {
+            hole->size = size;
+            hole->offset = va;
+            list_add(&hole->list, &mgr->va_holes);
+        }
+    }
+    pipe_mutex_unlock(mgr->bo_va_mutex);
+}
+
 static void radeon_bo_destroy(struct pb_buffer *_buf)
 {
     struct radeon_bo *bo = radeon_bo(_buf);
+    struct radeon_bomgr *mgr = bo->mgr;
     struct drm_gem_close args;
 
     memset(&args, 0, sizeof(args));
@@ -168,6 +298,10 @@ static void radeon_bo_destroy(struct pb_buffer *_buf)
     if (bo->ptr)
         os_munmap(bo->ptr, bo->base.size);
 
+    if (mgr->va) {
+        radeon_bomgr_free_va(mgr, bo->va, bo->va_size);
+    }
+
     /* Close object. */
     args.handle = bo->handle;
     drmIoctl(bo->rws->fd, DRM_IOCTL_GEM_CLOSE, &args);
@@ -343,6 +477,7 @@ static struct pb_buffer *radeon_bomgr_create_bo(struct pb_manager *_mgr,
     struct radeon_bo *bo;
     struct drm_radeon_gem_create args;
     struct radeon_bo_desc *rdesc = (struct radeon_bo_desc*)desc;
+    int r;
 
     memset(&args, 0, sizeof(args));
 
@@ -375,8 +510,38 @@ static struct pb_buffer *radeon_bomgr_create_bo(struct pb_manager *_mgr,
     bo->mgr = mgr;
     bo->rws = mgr->rws;
     bo->handle = args.handle;
+    bo->va = 0;
     pipe_mutex_init(bo->map_mutex);
 
+    if (mgr->va) {
+        struct drm_radeon_gem_va va;
+
+        bo->va_size = align(size,  4096);
+        bo->va = radeon_bomgr_find_va(mgr, bo->va_size);
+
+        va.handle = bo->handle;
+        va.vm_id = 0;
+        va.operation = RADEON_VA_MAP;
+        va.flags = RADEON_VM_PAGE_READABLE |
+                   RADEON_VM_PAGE_WRITEABLE |
+                   RADEON_VM_PAGE_SNOOPED;
+        va.offset = bo->va;
+        r = drmCommandWriteRead(rws->fd, DRM_RADEON_GEM_VA, &va, sizeof(va));
+        if (r && va.operation == RADEON_VA_RESULT_ERROR) {
+            fprintf(stderr, "radeon: Failed to allocate a buffer:\n");
+            fprintf(stderr, "radeon:    size      : %d bytes\n", size);
+            fprintf(stderr, "radeon:    alignment : %d bytes\n", desc->alignment);
+            fprintf(stderr, "radeon:    domains   : %d\n", args.initial_domain);
+            radeon_bo_destroy(&bo->base);
+            return NULL;
+        }
+        if (va.operation == RADEON_VA_RESULT_VA_EXIST) {
+            radeon_bomgr_free_va(mgr, bo->va, bo->va_size);
+            bo->va = va.offset;
+            radeon_bomgr_force_va(mgr, bo->va, bo->va_size);
+        }
+    }
+
     return &bo->base;
 }
 
@@ -407,6 +572,7 @@ static void radeon_bomgr_destroy(struct pb_manager *_mgr)
     struct radeon_bomgr *mgr = radeon_bomgr(_mgr);
     util_hash_table_destroy(mgr->bo_handles);
     pipe_mutex_destroy(mgr->bo_handles_mutex);
+    pipe_mutex_destroy(mgr->bo_va_mutex);
     FREE(mgr);
 }
 
@@ -438,6 +604,12 @@ struct pb_manager *radeon_bomgr_create(struct radeon_drm_winsys *rws)
     mgr->rws = rws;
     mgr->bo_handles = util_hash_table_create(handle_hash, handle_compare);
     pipe_mutex_init(mgr->bo_handles_mutex);
+    pipe_mutex_init(mgr->bo_va_mutex);
+
+    mgr->va = rws->info.r600_virtual_address;
+    mgr->va_offset = rws->info.r600_va_start;
+    list_inithead(&mgr->va_holes);
+
     return &mgr->base;
 }
 
@@ -560,6 +732,7 @@ static struct pb_buffer *radeon_winsys_bo_from_handle(struct radeon_winsys *rws,
     struct radeon_bo *bo;
     struct radeon_bomgr *mgr = radeon_bomgr(ws->kman);
     struct drm_gem_open open_arg = {};
+    int r;
 
     memset(&open_arg, 0, sizeof(open_arg));
 
@@ -603,6 +776,7 @@ static struct pb_buffer *radeon_winsys_bo_from_handle(struct radeon_winsys *rws,
     bo->base.vtbl = &radeon_bo_vtbl;
     bo->mgr = mgr;
     bo->rws = mgr->rws;
+    bo->va = 0;
     pipe_mutex_init(bo->map_mutex);
 
     util_hash_table_set(mgr->bo_handles, (void*)(uintptr_t)whandle->handle, bo);
@@ -613,6 +787,33 @@ done:
     if (stride)
         *stride = whandle->stride;
 
+    if (mgr->va) {
+        struct drm_radeon_gem_va va;
+
+        bo->va_size = ((bo->base.size + 4095) & ~4095);
+        bo->va = radeon_bomgr_find_va(mgr, bo->va_size);
+
+        va.handle = bo->handle;
+        va.operation = RADEON_VA_MAP;
+        va.vm_id = 0;
+        va.offset = bo->va;
+        va.flags = RADEON_VM_PAGE_READABLE |
+                   RADEON_VM_PAGE_WRITEABLE |
+                   RADEON_VM_PAGE_SNOOPED;
+        va.offset = bo->va;
+        r = drmCommandWriteRead(ws->fd, DRM_RADEON_GEM_VA, &va, sizeof(va));
+        if (r && va.operation == RADEON_VA_RESULT_ERROR) {
+            fprintf(stderr, "radeon: Failed to assign virtual address space\n");
+            radeon_bo_destroy(&bo->base);
+            return NULL;
+        }
+        if (va.operation == RADEON_VA_RESULT_VA_EXIST) {
+            radeon_bomgr_free_va(mgr, bo->va, bo->va_size);
+            bo->va = va.offset;
+            radeon_bomgr_force_va(mgr, bo->va, bo->va_size);
+        }
+    }
+
     return (struct pb_buffer*)bo;
 
 fail:
@@ -649,6 +850,13 @@ static boolean radeon_winsys_bo_get_handle(struct pb_buffer *buffer,
     return TRUE;
 }
 
+static uint64_t radeon_winsys_bo_va(struct pb_buffer *buffer)
+{
+    struct radeon_bo *bo = get_radeon_bo(buffer);
+
+    return bo->va;
+}
+
 void radeon_bomgr_init_functions(struct radeon_drm_winsys *ws)
 {
     ws->base.buffer_get_cs_handle = radeon_drm_get_cs_handle;
@@ -661,4 +869,5 @@ void radeon_bomgr_init_functions(struct radeon_drm_winsys *ws)
     ws->base.buffer_create = radeon_winsys_bo_create;
     ws->base.buffer_from_handle = radeon_winsys_bo_from_handle;
     ws->base.buffer_get_handle = radeon_winsys_bo_get_handle;
+    ws->base.buffer_get_virtual_address = radeon_winsys_bo_va;
 }
diff --git a/src/gallium/winsys/radeon/drm/radeon_drm_bo.h b/src/gallium/winsys/radeon/drm/radeon_drm_bo.h
index 35d25e87eb3..21cfe995510 100644
--- a/src/gallium/winsys/radeon/drm/radeon_drm_bo.h
+++ b/src/gallium/winsys/radeon/drm/radeon_drm_bo.h
@@ -59,6 +59,8 @@ struct radeon_bo {
 
     uint32_t handle;
     uint32_t name;
+    uint64_t va;
+    uint64_t va_size;
 
     /* how many command streams is this bo referenced in? */
     int num_cs_references;
diff --git a/src/gallium/winsys/radeon/drm/radeon_drm_cs.c b/src/gallium/winsys/radeon/drm/radeon_drm_cs.c
index e6109afd7ea..cf8f25bc5b4 100644
--- a/src/gallium/winsys/radeon/drm/radeon_drm_cs.c
+++ b/src/gallium/winsys/radeon/drm/radeon_drm_cs.c
@@ -71,18 +71,32 @@
 #include <stdint.h>
 #include <xf86drm.h>
 
+/*
+ * this are copy from radeon_drm, once an updated libdrm is released
+ * we should bump configure.ac requirement for it and remove the following
+ * field
+ */
 #ifndef RADEON_CHUNK_ID_FLAGS
-#define RADEON_CHUNK_ID_FLAGS	0x03
+#define RADEON_CHUNK_ID_FLAGS       0x03
 
 /* The first dword of RADEON_CHUNK_ID_FLAGS is a uint32 of these flags: */
 #define RADEON_CS_KEEP_TILING_FLAGS 0x01
 #endif
 
+#ifndef RADEON_CS_USE_VM
+#define RADEON_CS_USE_VM            0x02
+/* The second dword of RADEON_CHUNK_ID_FLAGS is a uint32 that sets the ring type */
+#define RADEON_CS_RING_GFX          0
+#define RADEON_CS_RING_COMPUTE      1
+#endif
+
+
 #define RELOC_DWORDS (sizeof(struct drm_radeon_cs_reloc) / sizeof(uint32_t))
 
-static boolean radeon_init_cs_context(struct radeon_cs_context *csc, int fd)
+static boolean radeon_init_cs_context(struct radeon_cs_context *csc,
+                                      struct radeon_drm_winsys *ws)
 {
-    csc->fd = fd;
+    csc->fd = ws->fd;
     csc->nrelocs = 512;
     csc->relocs_bo = (struct radeon_bo**)
                      CALLOC(1, csc->nrelocs * sizeof(struct radeon_bo*));
@@ -157,11 +171,11 @@ static struct radeon_winsys_cs *radeon_drm_cs_create(struct radeon_winsys *rws)
 
     cs->ws = ws;
 
-    if (!radeon_init_cs_context(&cs->csc1, cs->ws->fd)) {
+    if (!radeon_init_cs_context(&cs->csc1, cs->ws)) {
         FREE(cs);
         return NULL;
     }
-    if (!radeon_init_cs_context(&cs->csc2, cs->ws->fd)) {
+    if (!radeon_init_cs_context(&cs->csc2, cs->ws)) {
         radeon_destroy_cs_context(&cs->csc1);
         FREE(cs);
         return NULL;
@@ -440,11 +454,15 @@ static void radeon_drm_cs_flush(struct radeon_winsys_cs *rcs, unsigned flags)
             p_atomic_inc(&cs->cst->relocs_bo[i]->num_active_ioctls);
         }
 
+        cs->cst->flags = 0;
+        cs->cst->cs.num_chunks = 2;
         if (flags & RADEON_FLUSH_KEEP_TILING_FLAGS) {
+            cs->cst->flags |= RADEON_CS_KEEP_TILING_FLAGS;
             cs->cst->cs.num_chunks = 3;
-            cs->cst->flags = RADEON_CS_KEEP_TILING_FLAGS;
-        } else {
-            cs->cst->cs.num_chunks = 2;
+        }
+        if (cs->ws->info.r600_virtual_address) {
+            cs->cst->cs.num_chunks = 3;
+            cs->cst->flags |= RADEON_CS_USE_VM;
         }
 
         if (cs->thread &&
diff --git a/src/gallium/winsys/radeon/drm/radeon_drm_cs.h b/src/gallium/winsys/radeon/drm/radeon_drm_cs.h
index 904000d6933..05b9a487645 100644
--- a/src/gallium/winsys/radeon/drm/radeon_drm_cs.h
+++ b/src/gallium/winsys/radeon/drm/radeon_drm_cs.h
@@ -51,8 +51,8 @@ struct radeon_cs_context {
     struct drm_radeon_cs_reloc  *relocs_hashlist[256];
     unsigned                    reloc_indices_hashlist[256];
 
-    unsigned used_vram;
-    unsigned used_gart;
+    unsigned                    used_vram;
+    unsigned                    used_gart;
 };
 
 struct radeon_drm_cs {
diff --git a/src/gallium/winsys/radeon/drm/radeon_drm_winsys.c b/src/gallium/winsys/radeon/drm/radeon_drm_winsys.c
index f337411e223..051a390ed22 100644
--- a/src/gallium/winsys/radeon/drm/radeon_drm_winsys.c
+++ b/src/gallium/winsys/radeon/drm/radeon_drm_winsys.c
@@ -41,6 +41,11 @@
 #include <xf86drm.h>
 #include <stdio.h>
 
+/*
+ * this are copy from radeon_drm, once an updated libdrm is released
+ * we should bump configure.ac requirement for it and remove the following
+ * field
+ */
 #ifndef RADEON_INFO_TILING_CONFIG
 #define RADEON_INFO_TILING_CONFIG 6
 #endif
@@ -69,6 +74,14 @@
 #define RADEON_INFO_BACKEND_MAP 0xd
 #endif
 
+#ifndef RADEON_INFO_VA_START
+/* virtual address start, va < start are reserved by the kernel */
+#define RADEON_INFO_VA_START        0x0e
+/* maximum size of ib using the virtual memory cs */
+#define RADEON_INFO_IB_VM_MAX_SIZE  0x0f
+#endif
+
+
 /* Enable/disable feature access for one command stream.
  * If enable == TRUE, return TRUE on success.
  * Otherwise, return FALSE.
@@ -265,6 +278,16 @@ static boolean do_winsys_init(struct radeon_drm_winsys *ws)
                                       &ws->info.r600_backend_map))
                 ws->info.r600_backend_map_valid = TRUE;
         }
+        ws->info.r600_virtual_address = FALSE;
+        if (ws->info.drm_minor >= 13) {
+            ws->info.r600_virtual_address = TRUE;
+            if (!radeon_get_drm_value(ws->fd, RADEON_INFO_VA_START, NULL,
+                                      &ws->info.r600_va_start))
+                ws->info.r600_virtual_address = FALSE;
+            if (!radeon_get_drm_value(ws->fd, RADEON_INFO_IB_VM_MAX_SIZE, NULL,
+                                      &ws->info.r600_ib_vm_max_size))
+                ws->info.r600_virtual_address = FALSE;
+        }
     }
 
     return TRUE;
diff --git a/src/gallium/winsys/radeon/drm/radeon_winsys.h b/src/gallium/winsys/radeon/drm/radeon_winsys.h
index 59c1aad3308..d33eaa7059c 100644
--- a/src/gallium/winsys/radeon/drm/radeon_winsys.h
+++ b/src/gallium/winsys/radeon/drm/radeon_winsys.h
@@ -103,6 +103,9 @@ struct radeon_info {
     uint32_t r600_num_tile_pipes;
     uint32_t r600_backend_map;
     boolean r600_backend_map_valid;
+    boolean r600_virtual_address;
+    uint32_t r600_va_start;
+    uint32_t r600_ib_vm_max_size;
 };
 
 enum radeon_feature_id {
@@ -250,6 +253,14 @@ struct radeon_winsys {
                                  unsigned stride,
                                  struct winsys_handle *whandle);
 
+    /**
+     * Return the virtual address of a buffer.
+     *
+     * \param buf       A winsys buffer object
+     * \return          virtual address
+     */
+    uint64_t (*buffer_get_virtual_address)(struct pb_buffer *buf);
+
     /**************************************************************************
      * Command submission.
      *