iris: Use coherent allocation for PIPE_RESOURCE_STAGING

On !llc machines (Atoms), reading from a linear buffers is slow and so
copying from one resource into the linear staging buffer is still slow.
However, we can tell the GPU to snoop the CPU cache when reading from and
writing to the staging buffer eliminating the slow uncached reads.

Reviewed-by: Kenneth Graunke <[email protected]>

3 files changed, 24 insertions, 1 deletions

diff --git a/src/gallium/drivers/iris/iris_bufmgr.c b/src/gallium/drivers/iris/iris_bufmgr.c
index 92ede934056..b20297fd94a 100644
--- a/src/gallium/drivers/iris/iris_bufmgr.c
+++ b/src/gallium/drivers/iris/iris_bufmgr.c
@@ -521,6 +521,12 @@ bo_alloc_internal(struct iris_bufmgr *bufmgr,
    if (flags & BO_ALLOC_ZEROED)
       zeroed = true;
 
+   if ((flags & BO_ALLOC_COHERENT) && !bufmgr->has_llc) {
+      bo_size = MAX2(ALIGN(size, page_size), page_size);
+      bucket = NULL;
+      goto skip_cache;
+   }
+
    /* Round the allocated size up to a power of two number of pages. */
    bucket = bucket_for_size(bufmgr, size);
 
@@ -579,6 +585,7 @@ retry:
          bo->gtt_offset = 0ull;
       }
    } else {
+skip_cache:
       bo = bo_calloc();
       if (!bo)
          goto err;
@@ -644,6 +651,17 @@ retry:
 
    mtx_unlock(&bufmgr->lock);
 
+   if ((flags & BO_ALLOC_COHERENT) && !bo->cache_coherent) {
+      struct drm_i915_gem_caching arg = {
+         .handle = bo->gem_handle,
+         .caching = 1,
+      };
+      if (drm_ioctl(bufmgr->fd, DRM_IOCTL_I915_GEM_SET_CACHING, &arg) == 0) {
+         bo->cache_coherent = true;
+         bo->reusable = false;
+      }
+   }
+
    DBG("bo_create: buf %d (%s) %llub\n", bo->gem_handle, bo->name,
        (unsigned long long) size);
 
diff --git a/src/gallium/drivers/iris/iris_bufmgr.h b/src/gallium/drivers/iris/iris_bufmgr.h
index c8fd6af7216..a8a65088036 100644
--- a/src/gallium/drivers/iris/iris_bufmgr.h
+++ b/src/gallium/drivers/iris/iris_bufmgr.h
@@ -191,6 +191,7 @@ struct iris_bo {
 };
 
 #define BO_ALLOC_ZEROED     (1<<0)
+#define BO_ALLOC_COHERENT   (1<<1)
 
 /**
  * Allocate a buffer object.
diff --git a/src/gallium/drivers/iris/iris_resource.c b/src/gallium/drivers/iris/iris_resource.c
index 5040637ee77..155c8786d27 100644
--- a/src/gallium/drivers/iris/iris_resource.c
+++ b/src/gallium/drivers/iris/iris_resource.c
@@ -592,6 +592,10 @@ iris_resource_create_with_modifiers(struct pipe_screen *pscreen,
    const char *name = "miptree";
    enum iris_memory_zone memzone = IRIS_MEMZONE_OTHER;
 
+   unsigned int flags = 0;
+   if (templ->usage == PIPE_USAGE_STAGING)
+      flags |= BO_ALLOC_COHERENT;
+
    /* These are for u_upload_mgr buffers only */
    assert(!(templ->flags & (IRIS_RESOURCE_FLAG_SHADER_MEMZONE |
                             IRIS_RESOURCE_FLAG_SURFACE_MEMZONE |
@@ -600,7 +604,7 @@ iris_resource_create_with_modifiers(struct pipe_screen *pscreen,
    res->bo = iris_bo_alloc_tiled(screen->bufmgr, name, res->surf.size_B,
                                  memzone,
                                  isl_tiling_to_i915_tiling(res->surf.tiling),
-                                 res->surf.row_pitch_B, 0);
+                                 res->surf.row_pitch_B, flags);
 
    if (!res->bo)
       goto fail;